Revert "Merge pull request #16715 from adamemerson/wip-I-Object!"

This reverts commit 669453138d89e0f797a1bd37f38a2d68e6aac366, reversing
changes made to 36f5fcbb97eb2b1bceb526331eb3464f460fc701.

Signed-off-by: Sage Weil <sage@redhat.com>

- conflicts due to code rearrangement in 14b0db908f652032c358e419ffa90f5676698d0e

120 files changed, 2177 insertions, 10233 deletions

diff --git a/.gitignore b/.gitignore
index 2600c2c614e..a04d59a632e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -73,6 +73,3 @@ GTAGS
 /src/pybind/mgr/dashboard/frontend/src/environments/environment.ts
 /src/pybind/mgr/dashboard/frontend/src/environments/environment.prod.ts
 /src/pybind/mgr/dashboard/frontend/src/locale/messages.xlf
-
-# Python building things where it shouldn't
-/src/python-common/build/
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index ca1631c2e20..5b89d039374 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -297,7 +297,6 @@ if(NOT fmt_FOUND)
   message(STATUS "Could not find fmt, will build it")
   add_subdirectory(fmt)
 endif()
-include_directories(SYSTEM "${CMAKE_SOURCE_DIR}/src/fmt/include")
 
 if(WITH_SEASTAR)
   find_package(c-ares 1.13.0 QUIET)
@@ -332,13 +331,11 @@ set(libcommon_files
   ${CMAKE_BINARY_DIR}/src/include/ceph_ver.h
   ceph_ver.c
   xxHash/xxhash.c
-  common/error_code.cc
   log/Log.cc
   mon/MonCap.cc
   mon/MonClient.cc
   mon/MonMap.cc
   mon/MonSub.cc
-  mon/error_code.cc
   mgr/MgrClient.cc
   mon/PGMap.cc
   mgr/ServiceMap.cc
@@ -347,12 +344,10 @@ set(libcommon_files
   osd/OSDMap.cc
   osd/OSDMapMapping.cc
   osd/osd_types.cc
-  osd/error_code.cc
   osd/PGPeeringEvent.cc
   osd/OpRequest.cc
   osdc/Striper.cc
   osdc/Objecter.cc
-  osdc/error_code.cc
   librbd/Features.cc
   ${mds_files})
 set_source_files_properties(ceph_ver.c
@@ -480,7 +475,6 @@ option(WITH_LIBRADOSSTRIPER "build with libradosstriper support" ON)
 
 add_subdirectory(include)
 add_subdirectory(librados)
-add_subdirectory(RADOS)
 
 if(WITH_LIBRADOSSTRIPER)
   add_subdirectory(libradosstriper)
diff --git a/src/RADOS/CMakeLists.txt b/src/RADOS/CMakeLists.txt
deleted file mode 100644
index c8d4f1bb2e0..00000000000
--- a/src/RADOS/CMakeLists.txt
+++ /dev/null
@@ -1,37 +0,0 @@
-add_library(RADOS_objs OBJECT
-  RADOSImpl.cc)
-add_library(RADOS_api_obj OBJECT
-  RADOS.cc)
-
-add_library(libRADOS STATIC
-  $<TARGET_OBJECTS:RADOS_api_obj>
-  $<TARGET_OBJECTS:RADOS_objs>)
-target_link_libraries(libRADOS PRIVATE
-  osdc ceph-common cls_lock_client fmt::fmt
-  ${BLKID_LIBRARIES} ${CRYPTO_LIBS} ${EXTRALIBS})
-
-# if(ENABLE_SHARED)
-#   add_library(libRADOS ${CEPH_SHARED}
-#     $<TARGET_OBJECTS:RADOS_api_obj>
-#     $<TARGET_OBJECTS:RADOS_objs>
-#     $<TARGET_OBJECTS:common_buffer_obj>)
-#   set_target_properties(libRADOS PROPERTIES
-#     OUTPUT_NAME RADOS
-#     VERSION 0.0.1
-#     SOVERSION 1
-#     CXX_VISIBILITY_PRESET hidden
-#     VISIBILITY_INLINES_HIDDEN ON)
-#   if(NOT APPLE)
-#     set_property(TARGET libRADOS APPEND_STRING PROPERTY
-#       LINK_FLAGS " -Wl,--exclude-libs,ALL")
-#   endif()
-# else(ENABLE_SHARED)
-#   add_library(libRADOS STATIC
-#     $<TARGET_OBJECTS:RADOS_api_obj>
-#     $<TARGET_OBJECTS:RADOS_objs>)
-# endif(ENABLE_SHARED)
-# target_link_libraries(libRADOS PRIVATE
-#   osdc ceph-common cls_lock_client
-#   ${BLKID_LIBRARIES} ${CRYPTO_LIBS} ${EXTRALIBS})
-# target_link_libraries(libRADOS ${rados_libs})
-# install(TARGETS libRADOS DESTINATION ${CMAKE_INSTALL_LIBDIR})
diff --git a/src/RADOS/RADOS.cc b/src/RADOS/RADOS.cc
deleted file mode 100644
index e0a900fa8e2..00000000000
--- a/src/RADOS/RADOS.cc
+++ /dev/null
@@ -1,1475 +0,0 @@
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2018 Red Hat <contact@redhat.com>
- *
- * This is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License version 2.1, as published by the Free Software
- * Foundation.  See file COPYING.
- *
- */
-
-#define BOOST_BIND_NO_PLACEHOLDERS
-
-#include <optional>
-#include <string_view>
-
-#include <boost/intrusive_ptr.hpp>
-
-#include <fmt/format.h>
-
-#include "include/ceph_fs.h"
-#include "common/ceph_context.h"
-#include "common/ceph_argparse.h"
-#include "common/common_init.h"
-#include "common/hobject.h"
-
-#include "global/global_init.h"
-
-#include "osd/osd_types.h"
-#include "osdc/error_code.h"
-
-#include "RADOS/RADOSImpl.h"
-#include "include/RADOS/RADOS.hpp"
-
-namespace bc = boost::container;
-namespace bs = boost::system;
-namespace ca = ceph::async;
-namespace cb = ceph::buffer;
-
-namespace RADOS {
-// Object
-
-Object::Object(std::string_view s) {
-  static_assert(impl_size >= sizeof(object_t));
-  new (&impl) object_t(s);
-}
-
-Object::Object(std::string&& s) {
-  static_assert(impl_size >= sizeof(object_t));
-  new (&impl) object_t(std::move(s));
-}
-
-Object::Object(const std::string& s) {
-  static_assert(impl_size >= sizeof(object_t));
-  new (&impl) object_t(s);
-}
-
-Object::~Object() {
-  reinterpret_cast<object_t*>(&impl)->~object_t();
-}
-
-Object::Object(const Object& o) {
-  static_assert(impl_size >= sizeof(object_t));
-  new (&impl) object_t(*reinterpret_cast<const object_t*>(&o.impl));
-}
-Object& Object::operator =(const Object& o) {
-  *reinterpret_cast<object_t*>(&impl) =
-    *reinterpret_cast<const object_t*>(&o.impl);
-  return *this;
-}
-Object::Object(Object&& o) {
-  static_assert(impl_size >= sizeof(object_t));
-  new (&impl) object_t(std::move(*reinterpret_cast<object_t*>(&o.impl)));
-}
-Object& Object::operator =(Object&& o) {
-  *reinterpret_cast<object_t*>(&impl) =
-    std::move(*reinterpret_cast<object_t*>(&o.impl));
-  return *this;
-}
-
-Object::operator std::string_view() const {
-  return std::string_view(reinterpret_cast<const object_t*>(&impl)->name);
-}
-
-bool operator <(const Object& lhs, const Object& rhs) {
-  return (*reinterpret_cast<const object_t*>(&lhs.impl) <
-	  *reinterpret_cast<const object_t*>(&rhs.impl));
-}
-bool operator <=(const Object& lhs, const Object& rhs) {
-  return (*reinterpret_cast<const object_t*>(&lhs.impl) <=
-	  *reinterpret_cast<const object_t*>(&rhs.impl));
-}
-bool operator >=(const Object& lhs, const Object& rhs) {
-  return (*reinterpret_cast<const object_t*>(&lhs.impl) >=
-	  *reinterpret_cast<const object_t*>(&rhs.impl));
-}
-bool operator >(const Object& lhs, const Object& rhs) {
-  return (*reinterpret_cast<const object_t*>(&lhs.impl) >
-	  *reinterpret_cast<const object_t*>(&rhs.impl));
-}
-
-bool operator ==(const Object& lhs, const Object& rhs) {
-  return (*reinterpret_cast<const object_t*>(&lhs.impl) ==
-	  *reinterpret_cast<const object_t*>(&rhs.impl));
-}
-bool operator !=(const Object& lhs, const Object& rhs) {
-  return (*reinterpret_cast<const object_t*>(&lhs.impl) !=
-	  *reinterpret_cast<const object_t*>(&rhs.impl));
-}
-
-std::ostream& operator <<(std::ostream& m, const Object& o) {
-  return (m << *reinterpret_cast<const object_t*>(&o.impl));
-}
-
-// IOContext
-
-struct IOContextImpl {
-  object_locator_t oloc;
-  snapid_t snap_seq = CEPH_NOSNAP;
-  SnapContext snapc;
-};
-
-IOContext::IOContext() {
-  static_assert(impl_size >= sizeof(IOContextImpl));
-  new (&impl) IOContextImpl();
-}
-
-IOContext::IOContext(std::int64_t _pool) : IOContext() {
-  pool(_pool);
-}
-
-IOContext::IOContext(std::int64_t _pool, std::string_view _ns)
-  : IOContext() {
-  pool(_pool);
-  ns(_ns);
-}
-
-IOContext::IOContext(std::int64_t _pool, std::string&& _ns)
-  : IOContext() {
-  pool(_pool);
-  ns(std::move(_ns));
-}
-
-IOContext::~IOContext() {
-  reinterpret_cast<IOContextImpl*>(&impl)->~IOContextImpl();
-}
-
-IOContext::IOContext(const IOContext& rhs) {
-  static_assert(impl_size >= sizeof(IOContextImpl));
-  new (&impl) IOContextImpl(*reinterpret_cast<const IOContextImpl*>(&rhs.impl));
-}
-
-IOContext& IOContext::operator =(const IOContext& rhs) {
-  *reinterpret_cast<IOContextImpl*>(&impl) =
-    *reinterpret_cast<const IOContextImpl*>(&rhs.impl);
-  return *this;
-}
-
-IOContext::IOContext(IOContext&& rhs) {
-  static_assert(impl_size >= sizeof(IOContextImpl));
-  new (&impl) IOContextImpl(
-    std::move(*reinterpret_cast<IOContextImpl*>(&rhs.impl)));
-}
-
-IOContext& IOContext::operator =(IOContext&& rhs) {
-  *reinterpret_cast<IOContextImpl*>(&impl) =
-    std::move(*reinterpret_cast<IOContextImpl*>(&rhs.impl));
-  return *this;
-}
-
-std::int64_t IOContext::pool() const {
-  return reinterpret_cast<const IOContextImpl*>(&impl)->oloc.pool;
-}
-
-void IOContext::pool(std::int64_t _pool) {
-  reinterpret_cast<IOContextImpl*>(&impl)->oloc.pool = _pool;
-}
-
-std::string_view IOContext::ns() const {
-  return reinterpret_cast<const IOContextImpl*>(&impl)->oloc.nspace;
-}
-
-void IOContext::ns(std::string_view _ns) {
-  reinterpret_cast<IOContextImpl*>(&impl)->oloc.nspace = _ns;
-}
-
-void IOContext::ns(std::string&& _ns) {
-  reinterpret_cast<IOContextImpl*>(&impl)->oloc.nspace = std::move(_ns);
-}
-
-std::optional<std::string_view> IOContext::key() const {
-  auto& oloc = reinterpret_cast<const IOContextImpl*>(&impl)->oloc;
-  if (oloc.key.empty())
-    return std::nullopt;
-  else
-    return std::string_view(oloc.key);
-}
-
-void IOContext::key(std::string_view _key) {
-  auto& oloc = reinterpret_cast<IOContextImpl*>(&impl)->oloc;
-  if (_key.empty()) {
-    throw bs::system_error(EINVAL,
-			   bs::system_category(),
-			   "An empty key is no key at all.");
-  } else {
-    oloc.hash = -1;
-    oloc.key = _key;
-  }
-}
-
-void IOContext::key(std::string&&_key) {
-  auto& oloc = reinterpret_cast<IOContextImpl*>(&impl)->oloc;
-  if (_key.empty()) {
-    throw bs::system_error(EINVAL,
-			   bs::system_category(),
-			   "An empty key is no key at all.");
-  } else {
-    oloc.hash = -1;
-    oloc.key = std::move(_key);
-  }
-}
-
-void IOContext::clear_key() {
-  auto& oloc = reinterpret_cast<IOContextImpl*>(&impl)->oloc;
-  oloc.hash = -1;
-  oloc.key.clear();
-}
-
-std::optional<std::int64_t> IOContext::hash() const {
-  auto& oloc = reinterpret_cast<const IOContextImpl*>(&impl)->oloc;
-  if (oloc.hash < 0)
-    return std::nullopt;
-  else
-    return oloc.hash;
-}
-
-void IOContext::hash(std::int64_t _hash) {
-  auto& oloc = reinterpret_cast<IOContextImpl*>(&impl)->oloc;
-  if (_hash < 0) {
-    throw bs::system_error(EINVAL,
-			   bs::system_category(),
-			   "A negative hash is no hash at all.");
-  } else {
-    oloc.hash = _hash;
-    oloc.key.clear();
-  }
-}
-
-void IOContext::clear_hash() {
-  auto& oloc = reinterpret_cast<IOContextImpl*>(&impl)->oloc;
-  oloc.hash = -1;
-  oloc.key.clear();
-}
-
-
-std::optional<std::uint64_t> IOContext::read_snap() const {
-  auto& snap_seq = reinterpret_cast<const IOContextImpl*>(&impl)->snap_seq;
-  if (snap_seq == CEPH_NOSNAP)
-    return std::nullopt;
-  else
-    return snap_seq;
-}
-void IOContext::read_snap(std::optional<std::uint64_t> _snapid) {
-  auto& snap_seq = reinterpret_cast<IOContextImpl*>(&impl)->snap_seq;
-  snap_seq = _snapid.value_or(CEPH_NOSNAP);
-}
-
-std::optional<
-  std::pair<std::uint64_t,
-	    std::vector<std::uint64_t>>> IOContext::write_snap_context() const {
-  auto& snapc = reinterpret_cast<const IOContextImpl*>(&impl)->snapc;
-  if (snapc.empty()) {
-    return std::nullopt;
-  } else {
-    std::vector<uint64_t> v(snapc.snaps.begin(), snapc.snaps.end());
-    return std::make_optional(std::make_pair(uint64_t(snapc.seq), v));
-  }
-}
-
-void IOContext::write_snap_context(
-  std::optional<std::pair<std::uint64_t, std::vector<std::uint64_t>>> _snapc) {
-  auto& snapc = reinterpret_cast<IOContextImpl*>(&impl)->snapc;
-  if (!_snapc) {
-    snapc.clear();
-  } else {
-    SnapContext n(_snapc->first, { _snapc->second.begin(), _snapc->second.end()});
-    if (!n.is_valid()) {
-      throw bs::system_error(EINVAL,
-			     bs::system_category(),
-			     "Invalid snap context.");
-
-    } else {
-      snapc = n;
-    }
-  }
-}
-
-// Op
-
-struct OpImpl {
-  ObjectOperation op;
-  std::optional<ceph::real_time> mtime;
-
-  OpImpl() = default;
-
-  OpImpl(const OpImpl& rhs) = delete;
-  OpImpl(OpImpl&& rhs) = default;
-
-  OpImpl& operator =(const OpImpl& rhs) = delete;
-  OpImpl& operator =(OpImpl&& rhs) = default;
-};
-
-Op::Op() {
-  static_assert(Op::impl_size >= sizeof(OpImpl));
-  new (&impl) OpImpl;
-}
-
-Op::Op(Op&& rhs) {
-  new (&impl) OpImpl(std::move(*reinterpret_cast<OpImpl*>(&rhs.impl)));
-}
-Op& Op::operator =(Op&& rhs) {
-  reinterpret_cast<OpImpl*>(&impl)->~OpImpl();
-  new (&impl) OpImpl(std::move(*reinterpret_cast<OpImpl*>(&rhs.impl)));
-  return *this;
-}
-Op::~Op() {
-  reinterpret_cast<OpImpl*>(&impl)->~OpImpl();
-}
-
-void Op::set_excl() {
-  reinterpret_cast<OpImpl*>(&impl)->op.set_last_op_flags(CEPH_OSD_OP_FLAG_EXCL);
-}
-void Op::set_failok() {
-  reinterpret_cast<OpImpl*>(&impl)->op.set_last_op_flags(
-    CEPH_OSD_OP_FLAG_FAILOK);
-}
-void Op::set_fadvise_random() {
-  reinterpret_cast<OpImpl*>(&impl)->op.set_last_op_flags(
-    CEPH_OSD_OP_FLAG_FADVISE_RANDOM);
-}
-void Op::set_fadvise_sequential() {
-  reinterpret_cast<OpImpl*>(&impl)->op.set_last_op_flags(
-    CEPH_OSD_OP_FLAG_FADVISE_SEQUENTIAL);
-}
-void Op::set_fadvise_willneed() {
-  reinterpret_cast<OpImpl*>(&impl)->op.set_last_op_flags(
-    CEPH_OSD_OP_FLAG_FADVISE_WILLNEED);
-}
-void Op::set_fadvise_dontneed() {
-  reinterpret_cast<OpImpl*>(&impl)->op.set_last_op_flags(
-    CEPH_OSD_OP_FLAG_FADVISE_DONTNEED);
-}
-void Op::set_fadvise_nocache() {
-  reinterpret_cast<OpImpl*>(&impl)->op.set_last_op_flags(
-    CEPH_OSD_OP_FLAG_FADVISE_NOCACHE);
-}
-
-void Op::cmpext(uint64_t off, bufferlist&& cmp_bl, std::size_t* s) {
-  reinterpret_cast<OpImpl*>(&impl)->op.cmpext(off, std::move(cmp_bl), nullptr,
-					      s);
-}
-void Op::cmpxattr(std::string_view name, cmpxattr_op op, const bufferlist& val) {
-  reinterpret_cast<OpImpl*>(&impl)->
-    op.cmpxattr(name, std::uint8_t(op), CEPH_OSD_CMPXATTR_MODE_STRING, val);
-}
-void Op::cmpxattr(std::string_view name, cmpxattr_op op, std::uint64_t val) {
-  bufferlist bl;
-  encode(val, bl);
-  reinterpret_cast<OpImpl*>(&impl)->
-    op.cmpxattr(name, std::uint8_t(op), CEPH_OSD_CMPXATTR_MODE_U64, bl);
-}
-
-void Op::assert_version(uint64_t ver) {
-  reinterpret_cast<OpImpl*>(&impl)->op.assert_version(ver);
-}
-void Op::assert_exists() {
-  reinterpret_cast<OpImpl*>(&impl)->op.stat(
-    nullptr,
-    static_cast<ceph::real_time*>(nullptr),
-    static_cast<bs::error_code*>(nullptr));
-}
-void Op::cmp_omap(const bc::flat_map<
-		  std::string, std::pair<cb::list,
-		  int>>& assertions) {
-  reinterpret_cast<OpImpl*>(&impl)->op.omap_cmp(assertions, nullptr);
-}
-
-std::size_t Op::size() const {
-  return reinterpret_cast<const OpImpl*>(&impl)->op.size();
-}
-
-std::ostream& operator <<(std::ostream& m, const Op& o) {
-  return m << reinterpret_cast<const OpImpl*>(&o.impl)->op;
-}
-
-
-// ---
-
-// ReadOp / WriteOp
-
-void ReadOp::read(size_t off, uint64_t len, cb::list* out,
-		  bs::error_code* ec) {
-  reinterpret_cast<OpImpl*>(&impl)->op.read(off, len, ec, out);
-}
-
-void ReadOp::get_xattr(std::string_view name, cb::list* out,
-		       bs::error_code* ec) {
-  reinterpret_cast<OpImpl*>(&impl)->op.getxattr(name, ec, out);
-}
-
-void ReadOp::get_omap_header(cb::list* out,
-			     bs::error_code* ec) {
-  reinterpret_cast<OpImpl*>(&impl)->op.omap_get_header(ec, out);
-}
-
-void ReadOp::sparse_read(uint64_t off, uint64_t len, cb::list* out,
-			 std::vector<std::pair<std::uint64_t,
-			 std::uint64_t>>* extents,
-			 bs::error_code* ec) {
-  reinterpret_cast<OpImpl*>(&impl)->op.sparse_read(off, len, ec, extents, out);
-}
-
-void ReadOp::stat(std::uint64_t* size, ceph::real_time* mtime,
-		  bs::error_code* ec) {
-  reinterpret_cast<OpImpl*>(&impl)->op.stat(size, mtime, ec);
-}
-
-void ReadOp::get_omap_keys(std::optional<std::string_view> start_after,
-			   std::uint64_t max_return,
-			   bc::flat_set<std::string>* keys,
-			   bool* done,
-			   bs::error_code* ec) {
-  reinterpret_cast<OpImpl*>(&impl)->op.omap_get_keys(start_after, max_return,
-						     ec, keys, done);
-}
-
-void ReadOp::get_xattrs(bc::flat_map<std::string,
-			cb::list>* kv,
-			bs::error_code* ec) {
-  reinterpret_cast<OpImpl*>(&impl)->op.getxattrs(ec, kv);
-}
-
-void ReadOp::get_omap_vals(std::optional<std::string_view> start_after,
-			   std::optional<std::string_view> filter_prefix,
-			   uint64_t max_return,
-			   bc::flat_map<std::string,
-			   cb::list>* kv,
-			   bool* done,
-			   bs::error_code* ec) {
-  reinterpret_cast<OpImpl*>(&impl)->op.omap_get_vals(start_after, filter_prefix,
-						     max_return, ec, kv, done);
-}
-
-void ReadOp::get_omap_vals_by_keys(
-  const bc::flat_set<std::string>& keys,
-  bc::flat_map<std::string, cb::list>* kv,
-  bs::error_code* ec) {
-  reinterpret_cast<OpImpl*>(&impl)->op.omap_get_vals_by_keys(keys, ec, kv);
-}
-
-void ReadOp::list_watchers(std::vector<obj_watch_t>* watchers,
-			   bs::error_code* ec) {
-  reinterpret_cast<OpImpl*>(&impl)-> op.list_watchers(watchers, ec);
-}
-
-void ReadOp::list_snaps(librados::snap_set_t* snaps,
-			bs::error_code* ec) {
-  reinterpret_cast<OpImpl*>(&impl)->op.list_snaps(snaps, nullptr, ec);
-}
-
-void ReadOp::exec(std::string_view cls, std::string_view method,
-		  const bufferlist& inbl,
-		  cb::list* out,
-		  bs::error_code* ec) {
-  reinterpret_cast<OpImpl*>(&impl)->op.call(cls, method, inbl, ec, out);
-}
-
-void ReadOp::exec(std::string_view cls, std::string_view method,
-		  const bufferlist& inbl,
-		  fu2::unique_function<void (bs::error_code,
-					     const cb::list&) &&> f) {
-  reinterpret_cast<OpImpl*>(&impl)->op.call(cls, method, inbl, std::move(f));
-}
-
-// WriteOp
-
-void WriteOp::set_mtime(ceph::real_time t) {
-  auto o = reinterpret_cast<OpImpl*>(&impl);
-  o->mtime = t;
-}
-
-void WriteOp::create(bool exclusive) {
-  reinterpret_cast<OpImpl*>(&impl)->op.create(exclusive);
-}
-
-void WriteOp::write(uint64_t off, bufferlist&& bl) {
-  reinterpret_cast<OpImpl*>(&impl)->op.write(off, bl);
-}
-
-void WriteOp::write_full(bufferlist&& bl) {
-  reinterpret_cast<OpImpl*>(&impl)->op.write_full(bl);
-}
-
-void WriteOp::writesame(uint64_t off, uint64_t write_len, bufferlist&& bl) {
-  reinterpret_cast<OpImpl*>(&impl)->op.writesame(off, write_len, bl);
-}
-
-void WriteOp::append(bufferlist&& bl) {
-  reinterpret_cast<OpImpl*>(&impl)->op.append(bl);
-}
-
-void WriteOp::remove() {
-  reinterpret_cast<OpImpl*>(&impl)->op.remove();
-}
-
-void WriteOp::truncate(uint64_t off) {
-  reinterpret_cast<OpImpl*>(&impl)->op.truncate(off);
-}
-
-void WriteOp::zero(uint64_t off, uint64_t len) {
-  reinterpret_cast<OpImpl*>(&impl)->op.zero(off, len);
-}
-
-void WriteOp::rmxattr(std::string_view name) {
-  reinterpret_cast<OpImpl*>(&impl)->op.rmxattr(name);
-}
-
-void WriteOp::setxattr(std::string_view name,
-                       bufferlist&& bl) {
-  reinterpret_cast<OpImpl*>(&impl)->op.setxattr(name, bl);
-}
-
-void WriteOp::rollback(uint64_t snapid) {
-  reinterpret_cast<OpImpl*>(&impl)->op.rollback(snapid);
-}
-
-void WriteOp::set_omap(
-  const bc::flat_map<std::string, cb::list>& map) {
-  reinterpret_cast<OpImpl*>(&impl)->op.omap_set(map);
-}
-
-void WriteOp::set_omap_header(bufferlist&& bl) {
-  reinterpret_cast<OpImpl*>(&impl)->op.omap_set_header(bl);
-}
-
-void WriteOp::clear_omap() {
-  reinterpret_cast<OpImpl*>(&impl)->op.omap_clear();
-}
-
-void WriteOp::rm_omap_keys(
-  const bc::flat_set<std::string>& to_rm) {
-  reinterpret_cast<OpImpl*>(&impl)->op.omap_rm_keys(to_rm);
-}
-
-void WriteOp::set_alloc_hint(uint64_t expected_object_size,
-			     uint64_t expected_write_size,
-			     alloc_hint::alloc_hint_t flags) {
-  reinterpret_cast<OpImpl*>(&impl)->op.set_alloc_hint(expected_object_size,
-						      expected_write_size,
-						      flags);
-}
-
-void WriteOp::exec(std::string_view cls, std::string_view method,
-		   const bufferlist& inbl, bs::error_code* ec) {
-  reinterpret_cast<OpImpl*>(&impl)->op.call(cls, method, inbl, ec);
-}
-
-// RADOS
-
-RADOS::Builder& RADOS::Builder::add_conf_file(std::string_view f) {
-  if (conf_files)
-    *conf_files += (", " + std::string(f));
-  else
-    conf_files = std::string(f);
-  return *this;
-}
-
-void RADOS::Builder::build(boost::asio::io_context& ioctx,
-			   std::unique_ptr<BuildComp> c) {
-  constexpr auto env = CODE_ENVIRONMENT_LIBRARY;
-  CephInitParameters ci(env);
-  if (name)
-    ci.name.set(CEPH_ENTITY_TYPE_CLIENT, *name);
-  else
-    ci.name.set(CEPH_ENTITY_TYPE_CLIENT, "admin");
-  uint32_t flags = 0;
-  if (no_default_conf)
-    flags |= CINIT_FLAG_NO_DEFAULT_CONFIG_FILE;
-  if (no_mon_conf)
-    flags |= CINIT_FLAG_NO_MON_CONFIG;
-
-  CephContext *cct = common_preinit(ci, env, flags);
-  if (cluster)
-    cct->_conf->cluster = *cluster;
-
-  if (no_mon_conf)
-    cct->_conf->no_mon_config = true;
-
-  // TODO: Come up with proper error codes here. Maybe augment the
-  // functions with a default bs::error_code* parameter to
-  // pass back.
-  {
-    std::ostringstream ss;
-    auto r = cct->_conf.parse_config_files(conf_files ? conf_files->data() : nullptr,
-					   &ss, flags);
-    if (r < 0)
-      c->dispatch(std::move(c), ceph::to_error_code(r), RADOS{nullptr});
-  }
-
-  cct->_conf.parse_env(cct->get_module_type());
-
-  for (const auto& [n, v] : configs) {
-    std::stringstream ss;
-    auto r = cct->_conf.set_val(n, v, &ss);
-    if (r < 0)
-      c->dispatch(std::move(c), ceph::to_error_code(-EINVAL), RADOS{nullptr});
-  }
-
-  if (!no_mon_conf) {
-    MonClient mc_bootstrap(cct, ioctx);
-    // TODO This function should return an error code.
-    auto err = mc_bootstrap.get_monmap_and_config();
-    if (err < 0)
-      c->dispatch(std::move(c), ceph::to_error_code(err), RADOS{nullptr});
-  }
-  if (!cct->_log->is_started()) {
-    cct->_log->start();
-  }
-  common_init_finish(cct);
-
-  RADOS::make_with_cct(cct, ioctx, std::move(c));
-}
-
-void RADOS::make_with_cct(CephContext* cct,
-			  boost::asio::io_context& ioctx,
-			  std::unique_ptr<BuildComp> c) {
-  try {
-    auto r = new detail::RADOS(ioctx, cct);
-    r->objecter->wait_for_osd_map(
-      [c = std::move(c), r = std::unique_ptr<detail::RADOS>(r)]() mutable {
-	c->dispatch(std::move(c), bs::error_code{},
-		    RADOS{std::move(r)});
-      });
-  } catch (const bs::system_error& err) {
-    c->dispatch(std::move(c), err.code(), RADOS{nullptr});
-  }
-}
-
-
-RADOS::RADOS() = default;
-
-RADOS::RADOS(std::unique_ptr<detail::RADOS> impl)
-  : impl(std::move(impl)) {}
-
-RADOS::RADOS(RADOS&&) = default;
-RADOS& RADOS::operator =(RADOS&&) = default;
-
-RADOS::~RADOS() = default;
-
-RADOS::executor_type RADOS::get_executor() {
-  return impl->ioctx.get_executor();
-}
-
-void RADOS::execute(const Object& o, const IOContext& _ioc, ReadOp&& _op,
-		    cb::list* bl,
-		    std::unique_ptr<ReadOp::Completion> c, version_t* objver) {
-  auto oid = reinterpret_cast<const object_t*>(&o.impl);
-  auto ioc = reinterpret_cast<const IOContextImpl*>(&_ioc.impl);
-  auto op = reinterpret_cast<OpImpl*>(&_op.impl);
-  auto flags = 0; // Should be in Op.
-
-  impl->objecter->read(
-    *oid, ioc->oloc, std::move(op->op), ioc->snap_seq, bl, flags,
-    std::move(c), objver);
-}
-
-void RADOS::execute(const Object& o, const IOContext& _ioc, WriteOp&& _op,
-		    std::unique_ptr<WriteOp::Completion> c, version_t* objver) {
-  auto oid = reinterpret_cast<const object_t*>(&o.impl);
-  auto ioc = reinterpret_cast<const IOContextImpl*>(&_ioc.impl);
-  auto op = reinterpret_cast<OpImpl*>(&_op.impl);
-  auto flags = 0; // Should be in Op.
-  ceph::real_time mtime;
-  if (op->mtime)
-    mtime = *op->mtime;
-  else
-    mtime = ceph::real_clock::now();
-
-  impl->objecter->mutate(
-    *oid, ioc->oloc, std::move(op->op), ioc->snapc,
-    mtime, flags,
-    std::move(c), objver);
-}
-
-void RADOS::execute(const Object& o, std::int64_t pool, ReadOp&& _op,
-		    cb::list* bl,
-		    std::unique_ptr<ReadOp::Completion> c,
-		    std::optional<std::string_view> ns,
-		    std::optional<std::string_view> key,
-		    version_t* objver) {
-  auto oid = reinterpret_cast<const object_t*>(&o.impl);
-  auto op = reinterpret_cast<OpImpl*>(&_op.impl);
-  auto flags = 0; // Should be in Op.
-  object_locator_t oloc;
-  oloc.pool = pool;
-  if (ns)
-    oloc.nspace = *ns;
-  if (key)
-    oloc.key = *key;
-
-  impl->objecter->read(
-    *oid, oloc, std::move(op->op), CEPH_NOSNAP, bl, flags,
-    std::move(c), objver);
-}
-
-void RADOS::execute(const Object& o, std::int64_t pool, WriteOp&& _op,
-		    std::unique_ptr<WriteOp::Completion> c,
-		    std::optional<std::string_view> ns,
-		    std::optional<std::string_view> key,
-		    version_t* objver) {
-  auto oid = reinterpret_cast<const object_t*>(&o.impl);
-  auto op = reinterpret_cast<OpImpl*>(&_op.impl);
-  auto flags = 0; // Should be in Op.
-  object_locator_t oloc;
-  oloc.pool = pool;
-  if (ns)
-    oloc.nspace = *ns;
-  if (key)
-    oloc.key = *key;
-
-  ceph::real_time mtime;
-  if (op->mtime)
-    mtime = *op->mtime;
-  else
-    mtime = ceph::real_clock::now();
-
-  impl->objecter->mutate(
-    *oid, oloc, std::move(op->op), {},
-    mtime, flags,
-    std::move(c), objver);
-}
-
-boost::uuids::uuid RADOS::get_fsid() const noexcept {
-  return impl->monclient.get_fsid().uuid;
-}
-
-
-void RADOS::lookup_pool(std::string_view name,
-			std::unique_ptr<LookupPoolComp> c)
-{
-  // I kind of want to make lookup_pg_pool return
-  // std::optional<int64_t> since it can only return one error code.
-  int64_t ret = impl->objecter->with_osdmap(
-    std::mem_fn(&OSDMap::lookup_pg_pool_name),
-    name);
-  if (ret < 0) {
-    impl->objecter->wait_for_latest_osdmap(
-      [name = std::string(name), c = std::move(c),
-       objecter = impl->objecter.get()]
-      (bs::error_code ec) mutable {
-	int64_t ret =
-	  objecter->with_osdmap(std::mem_fn(&OSDMap::lookup_pg_pool_name),
-				name);
-	if (ret < 0)
-	  ca::dispatch(std::move(c), osdc_errc::pool_dne,
-		       std::int64_t(0));
-	else
-	  ca::dispatch(std::move(c), bs::error_code{}, ret);
-      });
-  } else if (ret < 0) {
-    ca::dispatch(std::move(c), osdc_errc::pool_dne,
-		 std::int64_t(0));
-  } else {
-    ca::dispatch(std::move(c), bs::error_code{}, ret);
-  }
-}
-
-
-std::optional<uint64_t> RADOS::get_pool_alignment(int64_t pool_id)
-{
-  return impl->objecter->with_osdmap(
-    [pool_id](const OSDMap &o) -> std::optional<uint64_t> {
-      if (!o.have_pg_pool(pool_id)) {
-	throw bs::system_error(
-	  ENOENT, bs::system_category(),
-	  "Cannot find pool in OSDMap.");
-      } else if (o.get_pg_pool(pool_id)->requires_aligned_append()) {
-	return o.get_pg_pool(pool_id)->required_alignment();
-      } else {
-	return std::nullopt;
-      }
-    });
-}
-
-void RADOS::list_pools(std::unique_ptr<LSPoolsComp> c) {
-  impl->objecter->with_osdmap(
-    [&](OSDMap& o) {
-      std::vector<std::pair<std::int64_t, std::string>> v;
-      for (auto p : o.get_pools())
-	v.push_back(std::make_pair(p.first, o.get_pool_name(p.first)));
-      ca::dispatch(std::move(c), std::move(v));
-    });
-}
-
-void RADOS::create_pool_snap(std::int64_t pool,
-			     std::string_view snapName,
-			     std::unique_ptr<SimpleOpComp> c)
-{
-  impl->objecter->create_pool_snap(
-    pool, snapName,
-    Objecter::PoolOp::OpComp::create(
-      get_executor(),
-      [c = std::move(c)](bs::error_code e, const bufferlist&) mutable {
-	ca::dispatch(std::move(c), e);
-      }));
-}
-
-void RADOS::allocate_selfmanaged_snap(int64_t pool,
-				      std::unique_ptr<SMSnapComp> c) {
-  impl->objecter->allocate_selfmanaged_snap(
-    pool,
-    ca::Completion<void(bs::error_code, snapid_t)>::create(
-      get_executor(),
-      [c = std::move(c)](bs::error_code e, snapid_t snap) mutable {
-	ca::dispatch(std::move(c), e, snap);
-      }));
-}
-
-void RADOS::delete_pool_snap(std::int64_t pool,
-			     std::string_view snapName,
-			     std::unique_ptr<SimpleOpComp> c)
-{
-  impl->objecter->delete_pool_snap(
-    pool, snapName,
-    Objecter::PoolOp::OpComp::create(
-      get_executor(),
-      [c = std::move(c)](bs::error_code e, const bufferlist&) mutable {
-	ca::dispatch(std::move(c), e);
-      }));
-}
-
-void RADOS::delete_selfmanaged_snap(std::int64_t pool,
-				    snapid_t snap,
-				    std::unique_ptr<SimpleOpComp> c)
-{
-  impl->objecter->delete_selfmanaged_snap(
-    pool, snap,
-    Objecter::PoolOp::OpComp::create(
-      get_executor(),
-      [c = std::move(c)](bs::error_code e, const bufferlist&) mutable {
-	ca::dispatch(std::move(c), e);
-      }));
-}
-
-void RADOS::create_pool(std::string_view name,
-			std::optional<int> crush_rule,
-			std::unique_ptr<SimpleOpComp> c)
-{
-  impl->objecter->create_pool(
-    name,
-    Objecter::PoolOp::OpComp::create(
-      get_executor(),
-      [c = std::move(c)](bs::error_code e, const bufferlist&) mutable {
-	ca::dispatch(std::move(c), e);
-      }),
-      crush_rule.value_or(-1));
-}
-
-void RADOS::delete_pool(std::string_view name,
-			std::unique_ptr<SimpleOpComp> c)
-{
-  impl->objecter->delete_pool(
-    name,
-    Objecter::PoolOp::OpComp::create(
-      get_executor(),
-      [c = std::move(c)](bs::error_code e, const bufferlist&) mutable {
-	ca::dispatch(std::move(c), e);
-      }));
-}
-
-void RADOS::delete_pool(std::int64_t pool,
-			std::unique_ptr<SimpleOpComp> c)
-{
-  impl->objecter->delete_pool(
-    pool,
-    Objecter::PoolOp::OpComp::create(
-      get_executor(),
-      [c = std::move(c)](bs::error_code e, const bufferlist&) mutable {
-	ca::dispatch(std::move(c), e);
-      }));
-}
-
-void RADOS::stat_pools(const std::vector<std::string>& pools,
-		       std::unique_ptr<PoolStatComp> c) {
-  impl->objecter->get_pool_stats(
-    pools,
-    [c = std::move(c)]
-    (bs::error_code ec,
-     bc::flat_map<std::string, pool_stat_t> s,
-     bool p) mutable {
-      ca::dispatch(std::move(c), ec, std::move(s), p);
-    });
-}
-
-void RADOS::stat_fs(std::optional<std::int64_t> _pool,
-		    std::unique_ptr<StatFSComp> c) {
-  boost::optional<int64_t> pool;
-  if (_pool)
-    pool = *pool;
-  impl->objecter->get_fs_stats(pool, std::move(c));
-}
-
-// --- Watch/Notify
-
-void RADOS::watch(const Object& o, const IOContext& _ioc,
-		  std::optional<std::chrono::seconds> timeout, WatchCB&& cb,
-		  std::unique_ptr<WatchComp> c) {
-  auto oid = reinterpret_cast<const object_t*>(&o.impl);
-  auto ioc = reinterpret_cast<const IOContextImpl*>(&_ioc.impl);
-
-  ObjectOperation op;
-
-  auto linger_op = impl->objecter->linger_register(*oid, ioc->oloc, 0);
-  uint64_t cookie = linger_op->get_cookie();
-  linger_op->handle = std::move(cb);
-  op.watch(cookie, CEPH_OSD_WATCH_OP_WATCH, timeout.value_or(0s).count());
-  bufferlist bl;
-  impl->objecter->linger_watch(
-    linger_op, op, ioc->snapc, ceph::real_clock::now(), bl,
-    Objecter::LingerOp::OpComp::create(
-      get_executor(),
-      [c = std::move(c), cookie](bs::error_code e, cb::list) mutable {
-	ca::dispatch(std::move(c), e, cookie);
-      }), nullptr);
-}
-
-void RADOS::watch(const Object& o, std::int64_t pool,
-		  std::optional<std::chrono::seconds> timeout, WatchCB&& cb,
-		  std::unique_ptr<WatchComp> c,
-		  std::optional<std::string_view> ns,
-		  std::optional<std::string_view> key) {
-  auto oid = reinterpret_cast<const object_t*>(&o.impl);
-  object_locator_t oloc;
-  oloc.pool = pool;
-  if (ns)
-    oloc.nspace = *ns;
-  if (key)
-    oloc.key = *key;
-
-  ObjectOperation op;
-
-  Objecter::LingerOp *linger_op = impl->objecter->linger_register(*oid, oloc, 0);
-  uint64_t cookie = linger_op->get_cookie();
-  linger_op->handle = std::move(cb);
-  op.watch(cookie, CEPH_OSD_WATCH_OP_WATCH, timeout.value_or(0s).count());
-  bufferlist bl;
-  impl->objecter->linger_watch(
-    linger_op, op, {}, ceph::real_clock::now(), bl,
-    Objecter::LingerOp::OpComp::create(
-      get_executor(),
-      [c = std::move(c), cookie](bs::error_code e, bufferlist) mutable {
-	ca::dispatch(std::move(c), e, cookie);
-      }), nullptr);
-}
-
-void RADOS::notify_ack(const Object& o,
-		       const IOContext& _ioc,
-		       uint64_t notify_id,
-		       uint64_t cookie,
-		       bufferlist&& bl,
-		       std::unique_ptr<SimpleOpComp> c)
-{
-  auto oid = reinterpret_cast<const object_t*>(&o.impl);
-  auto ioc = reinterpret_cast<const IOContextImpl*>(&_ioc.impl);
-
-  ObjectOperation op;
-  op.notify_ack(notify_id, cookie, bl);
-
-  impl->objecter->read(*oid, ioc->oloc, std::move(op), ioc->snap_seq,
-		       nullptr, 0, std::move(c));
-}
-
-void RADOS::notify_ack(const Object& o,
-		       std::int64_t pool,
-		       uint64_t notify_id,
-		       uint64_t cookie,
-		       bufferlist&& bl,
-		       std::unique_ptr<SimpleOpComp> c,
-		       std::optional<std::string_view> ns,
-		       std::optional<std::string_view> key) {
-  auto oid = reinterpret_cast<const object_t*>(&o.impl);
-  object_locator_t oloc;
-  oloc.pool = pool;
-  if (ns)
-    oloc.nspace = *ns;
-  if (key)
-    oloc.key = *key;
-
-  ObjectOperation op;
-  op.notify_ack(notify_id, cookie, bl);
-  impl->objecter->read(*oid, oloc, std::move(op), CEPH_NOSNAP, nullptr, 0,
-		       std::move(c));
-}
-
-tl::expected<ceph::timespan, bs::error_code> RADOS::watch_check(uint64_t cookie)
-{
-  Objecter::LingerOp *linger_op = reinterpret_cast<Objecter::LingerOp*>(cookie);
-  return impl->objecter->linger_check(linger_op);
-}
-
-void RADOS::unwatch(uint64_t cookie, const IOContext& _ioc,
-		    std::unique_ptr<SimpleOpComp> c)
-{
-  auto ioc = reinterpret_cast<const IOContextImpl*>(&_ioc.impl);
-
-  Objecter::LingerOp *linger_op = reinterpret_cast<Objecter::LingerOp*>(cookie);
-
-  ObjectOperation op;
-  op.watch(cookie, CEPH_OSD_WATCH_OP_UNWATCH);
-  impl->objecter->mutate(linger_op->target.base_oid, ioc->oloc, std::move(op),
-			 ioc->snapc, ceph::real_clock::now(), 0,
-			 Objecter::Op::OpComp::create(
-			   get_executor(),
-			   [objecter = impl->objecter.get(),
-			    linger_op, c = std::move(c)]
-			   (bs::error_code ec) mutable {
-			     objecter->linger_cancel(linger_op);
-			     ca::dispatch(std::move(c), ec);
-			   }));
-}
-
-void RADOS::unwatch(uint64_t cookie, std::int64_t pool,
-		    std::unique_ptr<SimpleOpComp> c,
-		    std::optional<std::string_view> ns,
-		    std::optional<std::string_view> key)
-{
-  object_locator_t oloc;
-  oloc.pool = pool;
-  if (ns)
-    oloc.nspace = *ns;
-  if (key)
-    oloc.key = *key;
-
-  Objecter::LingerOp *linger_op = reinterpret_cast<Objecter::LingerOp*>(cookie);
-
-  ObjectOperation op;
-  op.watch(cookie, CEPH_OSD_WATCH_OP_UNWATCH);
-  impl->objecter->mutate(linger_op->target.base_oid, oloc, std::move(op),
-			 {}, ceph::real_clock::now(), 0,
-			 Objecter::Op::OpComp::create(
-			   get_executor(),
-			   [objecter = impl->objecter.get(),
-			    linger_op, c = std::move(c)]
-			   (bs::error_code ec) mutable {
-			     objecter->linger_cancel(linger_op);
-			     ca::dispatch(std::move(c), ec);
-			   }));
-}
-
-void RADOS::flush_watch(std::unique_ptr<VoidOpComp> c)
-{
-  impl->objecter->linger_callback_flush([c = std::move(c)]() mutable {
-					  ca::post(std::move(c));
-					});
-}
-
-struct NotifyHandler : std::enable_shared_from_this<NotifyHandler> {
-  boost::asio::io_context& ioc;
-  boost::asio::io_context::strand strand;
-  Objecter* objecter;
-  Objecter::LingerOp* op;
-  std::unique_ptr<RADOS::NotifyComp> c;
-
-  bool acked = false;
-  bool finished = false;
-  bs::error_code res;
-  bufferlist rbl;
-
-  NotifyHandler(boost::asio::io_context& ioc,
-		Objecter* objecter,
-		Objecter::LingerOp* op,
-		std::unique_ptr<RADOS::NotifyComp> c)
-    : ioc(ioc), strand(ioc), objecter(objecter), op(op), c(std::move(c)) {}
-
-  // Use bind or a lambda to pass this in.
-  void handle_ack(bs::error_code ec,
-		  bufferlist&&) {
-    boost::asio::post(
-      strand,
-      [this, ec, p = shared_from_this()]() mutable {
-	acked = true;
-	maybe_cleanup(ec);
-      });
-  }
-
-  // Notify finish callback. It can actually own the object's storage.
-
-  void operator()(bs::error_code ec,
-		  bufferlist&& bl) {
-    boost::asio::post(
-      strand,
-      [this, ec, p = shared_from_this()]() mutable {
-	finished = true;
-	maybe_cleanup(ec);
-      });
-  }
-
-  // Should be called from strand.
-  void maybe_cleanup(bs::error_code ec) {
-    if (!res && ec)
-      res = ec;
-    if ((acked && finished) || res) {
-      objecter->linger_cancel(op);
-      ceph_assert(c);
-      ca::dispatch(std::move(c), res, std::move(rbl));
-    }
-  }
-};
-
-void RADOS::notify(const Object& o, const IOContext& _ioc, bufferlist&& bl,
-		   std::optional<std::chrono::milliseconds> timeout,
-		   std::unique_ptr<NotifyComp> c)
-{
-  auto oid = reinterpret_cast<const object_t*>(&o.impl);
-  auto ioc = reinterpret_cast<const IOContextImpl*>(&_ioc.impl);
-  auto linger_op = impl->objecter->linger_register(*oid, ioc->oloc, 0);
-
-  auto cb = std::make_shared<NotifyHandler>(impl->ioctx, impl->objecter.get(),
-                                            linger_op, std::move(c));
-  linger_op->on_notify_finish =
-    Objecter::LingerOp::OpComp::create(
-      get_executor(),
-      [cb](bs::error_code ec, ceph::bufferlist bl) mutable {
-	(*cb)(ec, std::move(bl));
-      });
-  ObjectOperation rd;
-  bufferlist inbl;
-  rd.notify(
-    linger_op->get_cookie(), 1,
-    timeout ? timeout->count() : impl->cct->_conf->client_notify_timeout,
-    bl, &inbl);
-
-  impl->objecter->linger_notify(
-    linger_op, rd, ioc->snap_seq, inbl,
-    Objecter::LingerOp::OpComp::create(
-      get_executor(),
-      [cb](bs::error_code ec, ceph::bufferlist bl) mutable {
-	cb->handle_ack(ec, std::move(bl));
-      }), nullptr);
-}
-
-void RADOS::notify(const Object& o, std::int64_t pool, bufferlist&& bl,
-		   std::optional<std::chrono::milliseconds> timeout,
-		   std::unique_ptr<NotifyComp> c,
-		   std::optional<std::string_view> ns,
-		   std::optional<std::string_view> key)
-{
-  auto oid = reinterpret_cast<const object_t*>(&o.impl);
-  object_locator_t oloc;
-  oloc.pool = pool;
-  if (ns)
-    oloc.nspace = *ns;
-  if (key)
-    oloc.key = *key;
-  auto linger_op = impl->objecter->linger_register(*oid, oloc, 0);
-
-  auto cb = std::make_shared<NotifyHandler>(impl->ioctx, impl->objecter.get(),
-                                            linger_op, std::move(c));
-  linger_op->on_notify_finish =
-    Objecter::LingerOp::OpComp::create(
-      get_executor(),
-      [cb](bs::error_code ec, ceph::bufferlist&& bl) mutable {
-	(*cb)(ec, std::move(bl));
-      });
-  ObjectOperation rd;
-  bufferlist inbl;
-  rd.notify(
-    linger_op->get_cookie(), 1,
-    timeout ? timeout->count() : impl->cct->_conf->client_notify_timeout,
-    bl, &inbl);
-
-  impl->objecter->linger_notify(
-    linger_op, rd, CEPH_NOSNAP, inbl,
-    Objecter::LingerOp::OpComp::create(
-      get_executor(),
-      [cb](bs::error_code ec, bufferlist&& bl) mutable {
-	cb->handle_ack(ec, std::move(bl));
-      }), nullptr);
-}
-
-// Enumeration
-
-Cursor::Cursor() {
-  static_assert(impl_size >= sizeof(hobject_t));
-  new (&impl) hobject_t();
-};
-
-Cursor::Cursor(end_magic_t) {
-  static_assert(impl_size >= sizeof(hobject_t));
-  new (&impl) hobject_t(hobject_t::get_max());
-}
-
-Cursor::Cursor(void* p) {
-  static_assert(impl_size >= sizeof(hobject_t));
-  new (&impl) hobject_t(std::move(*reinterpret_cast<hobject_t*>(p)));
-}
-
-Cursor Cursor::begin() {
-  Cursor e;
-  return e;
-}
-
-Cursor Cursor::end() {
-  Cursor e(end_magic_t{});
-  return e;
-}
-
-Cursor::Cursor(const Cursor& rhs) {
-  static_assert(impl_size >= sizeof(hobject_t));
-  new (&impl) hobject_t(*reinterpret_cast<const hobject_t*>(&rhs.impl));
-}
-
-Cursor& Cursor::operator =(const Cursor& rhs) {
-  static_assert(impl_size >= sizeof(hobject_t));
-  reinterpret_cast<hobject_t*>(&impl)->~hobject_t();
-  new (&impl) hobject_t(*reinterpret_cast<const hobject_t*>(&rhs.impl));
-  return *this;
-}
-
-Cursor::Cursor(Cursor&& rhs) {
-  static_assert(impl_size >= sizeof(hobject_t));
-  new (&impl) hobject_t(std::move(*reinterpret_cast<hobject_t*>(&rhs.impl)));
-}
-
-Cursor& Cursor::operator =(Cursor&& rhs) {
-  static_assert(impl_size >= sizeof(hobject_t));
-  reinterpret_cast<hobject_t*>(&impl)->~hobject_t();
-  new (&impl) hobject_t(std::move(*reinterpret_cast<hobject_t*>(&rhs.impl)));
-  return *this;
-}
-Cursor::~Cursor() {
-  reinterpret_cast<hobject_t*>(&impl)->~hobject_t();
-}
-
-bool operator ==(const Cursor& lhs, const Cursor& rhs) {
-  return (*reinterpret_cast<const hobject_t*>(&lhs.impl) ==
-	  *reinterpret_cast<const hobject_t*>(&rhs.impl));
-}
-
-bool operator !=(const Cursor& lhs, const Cursor& rhs) {
-  return (*reinterpret_cast<const hobject_t*>(&lhs.impl) !=
-	  *reinterpret_cast<const hobject_t*>(&rhs.impl));
-}
-
-bool operator <(const Cursor& lhs, const Cursor& rhs) {
-  return (*reinterpret_cast<const hobject_t*>(&lhs.impl) <
-	  *reinterpret_cast<const hobject_t*>(&rhs.impl));
-}
-
-bool operator <=(const Cursor& lhs, const Cursor& rhs) {
-  return (*reinterpret_cast<const hobject_t*>(&lhs.impl) <=
-	  *reinterpret_cast<const hobject_t*>(&rhs.impl));
-}
-
-bool operator >=(const Cursor& lhs, const Cursor& rhs) {
-  return (*reinterpret_cast<const hobject_t*>(&lhs.impl) >=
-	  *reinterpret_cast<const hobject_t*>(&rhs.impl));
-}
-
-bool operator >(const Cursor& lhs, const Cursor& rhs) {
-  return (*reinterpret_cast<const hobject_t*>(&lhs.impl) >
-	  *reinterpret_cast<const hobject_t*>(&rhs.impl));
-}
-
-std::string Cursor::to_str() const {
-  using namespace std::literals;
-  auto& h = *reinterpret_cast<const hobject_t*>(&impl);
-
-  return h.is_max() ? "MAX"s : h.to_str();
-}
-
-std::optional<Cursor>
-Cursor::from_str(const std::string& s) {
-  Cursor e;
-  auto& h = *reinterpret_cast<hobject_t*>(&e.impl);
-  if (!h.parse(s))
-    return std::nullopt;
-
-  return e;
-}
-
-void RADOS::enumerate_objects(const IOContext& _ioc,
-			      const Cursor& begin,
-			      const Cursor& end,
-			      const std::uint32_t max,
-			      const bufferlist& filter,
-			      std::vector<Entry>* ls,
-			      Cursor* cursor,
-			      std::unique_ptr<SimpleOpComp> c) {
-  auto ioc = reinterpret_cast<const IOContextImpl*>(&_ioc.impl);
-
-  impl->objecter->enumerate_objects(
-    ioc->oloc.pool,
-    ioc->oloc.nspace,
-    *reinterpret_cast<const hobject_t*>(&begin.impl),
-    *reinterpret_cast<const hobject_t*>(&end.impl),
-    max,
-    filter,
-    [c = std::move(c), ls, cursor]
-    (bs::error_code ec, std::vector<Entry>&& v,
-     hobject_t&& n) mutable {
-      if (ls)
-	*ls = std::move(v);
-      if (cursor) {
-	Cursor next(static_cast<void*>(&n));
-	*cursor = std::move(next);
-      }
-      ca::dispatch(std::move(c), ec);
-    });
-}
-
-void RADOS::enumerate_objects(std::int64_t pool,
-			      const Cursor& begin,
-			      const Cursor& end,
-			      const std::uint32_t max,
-			      const bufferlist& filter,
-			      std::vector<Entry>* ls,
-			      Cursor* cursor,
-			      std::unique_ptr<SimpleOpComp> c,
-			      std::optional<std::string_view> ns,
-			      std::optional<std::string_view> key) {
-  impl->objecter->enumerate_objects(
-    pool,
-    ns ? *ns : std::string_view{},
-    *reinterpret_cast<const hobject_t*>(&begin.impl),
-    *reinterpret_cast<const hobject_t*>(&end.impl),
-    max,
-    filter,
-    [c = std::move(c), ls, cursor]
-    (bs::error_code ec, std::vector<Entry>&& v,
-     hobject_t&& n) mutable {
-      if (ls)
-	*ls = std::move(v);
-      if (cursor) {
-	Cursor next(static_cast<void*>(&n));
-	*cursor = std::move(next);
-      }
-      ca::dispatch(std::move(c), ec);
-    });
-}
-
-void RADOS::enumerate_objects(const IOContext& _ioc,
-			      const Cursor& begin,
-			      const Cursor& end,
-			      const std::uint32_t max,
-			      const bufferlist& filter,
-			      std::unique_ptr<EnumerateComp> c) {
-  auto ioc = reinterpret_cast<const IOContextImpl*>(&_ioc.impl);
-
-  impl->objecter->enumerate_objects(
-    ioc->oloc.pool,
-    ioc->oloc.nspace,
-    *reinterpret_cast<const hobject_t*>(&begin.impl),
-    *reinterpret_cast<const hobject_t*>(&end.impl),
-    max,
-    filter,
-    [c = std::move(c)]
-    (bs::error_code ec, std::vector<Entry>&& v,
-     hobject_t&& n) mutable {
-      ca::dispatch(std::move(c), ec, std::move(v),
-		   Cursor(static_cast<void*>(&n)));
-    });
-}
-
-void RADOS::enumerate_objects(std::int64_t pool,
-			      const Cursor& begin,
-			      const Cursor& end,
-			      const std::uint32_t max,
-			      const bufferlist& filter,
-			      std::unique_ptr<EnumerateComp> c,
-			      std::optional<std::string_view> ns,
-			      std::optional<std::string_view> key) {
-  impl->objecter->enumerate_objects(
-    pool,
-    ns ? *ns : std::string_view{},
-    *reinterpret_cast<const hobject_t*>(&begin.impl),
-    *reinterpret_cast<const hobject_t*>(&end.impl),
-    max,
-    filter,
-    [c = std::move(c)]
-    (bs::error_code ec, std::vector<Entry>&& v,
-     hobject_t&& n) mutable {
-      ca::dispatch(std::move(c), ec, std::move(v),
-		   Cursor(static_cast<void*>(&n)));
-    });
-}
-
-
-void RADOS::osd_command(int osd, std::vector<std::string>&& cmd,
-			ceph::bufferlist&& in, std::unique_ptr<CommandComp> c) {
-  impl->objecter->osd_command(osd, std::move(cmd), std::move(in), nullptr,
-			      [c = std::move(c)]
-			      (bs::error_code ec,
-			       std::string&& s,
-			       ceph::bufferlist&& b) mutable {
-				ca::dispatch(std::move(c), ec,
-					     std::move(s),
-					     std::move(b));
-			      });
-}
-void RADOS::pg_command(pg_t pg, std::vector<std::string>&& cmd,
-		       ceph::bufferlist&& in, std::unique_ptr<CommandComp> c) {
-  impl->objecter->pg_command(pg, std::move(cmd), std::move(in), nullptr,
-			     [c = std::move(c)]
-			     (bs::error_code ec,
-			      std::string&& s,
-			      ceph::bufferlist&& b) mutable {
-			       ca::dispatch(std::move(c), ec,
-					    std::move(s),
-					    std::move(b));
-			     });
-}
-
-void RADOS::enable_application(std::string_view pool, std::string_view app_name,
-			       bool force, std::unique_ptr<SimpleOpComp> c) {
-  // pre-Luminous clusters will return -EINVAL and application won't be
-  // preserved until Luminous is configured as minimum version.
-  if (!impl->get_required_monitor_features().contains_all(
-	ceph::features::mon::FEATURE_LUMINOUS)) {
-    ca::dispatch(std::move(c), ceph::to_error_code(-EOPNOTSUPP));
-  } else {
-    impl->monclient.start_mon_command(
-      { fmt::format("{{ \"prefix\": \"osd pool application enable\","
-		    "\"pool\": \"{}\", \"app\": \"{}\"{}}}",
-		    pool, app_name,
-		    force ? " ,\"yes_i_really_mean_it\": true" : "")},
-      {}, [c = std::move(c)](bs::error_code e,
-			     std::string, cb::list) mutable {
-	    ca::post(std::move(c), e);
-	  });
-  }
-}
-
-void RADOS::mon_command(std::vector<std::string> command,
-			const cb::list& bl,
-			std::string* outs, cb::list* outbl,
-			std::unique_ptr<SimpleOpComp> c) {
-
-  impl->monclient.start_mon_command(
-    command, bl,
-    [c = std::move(c), outs, outbl](bs::error_code e,
-				    std::string s, cb::list bl) mutable {
-      if (outs)
-	*outs = std::move(s);
-      if (outbl)
-	*outbl = std::move(bl);
-      ca::post(std::move(c), e);
-    });
-}
-
-uint64_t RADOS::instance_id() const {
-  return impl->get_instance_id();
-}
-}
-
-namespace std {
-size_t hash<RADOS::Object>::operator ()(
-  const RADOS::Object& r) const {
-  static constexpr const hash<object_t> H;
-  return H(*reinterpret_cast<const object_t*>(&r.impl));
-}
-}
diff --git a/src/RADOS/RADOSImpl.cc b/src/RADOS/RADOSImpl.cc
deleted file mode 100644
index 4dd816da519..00000000000
--- a/src/RADOS/RADOSImpl.cc
+++ /dev/null
@@ -1,112 +0,0 @@
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2004-2012 Sage Weil <sage@newdream.net>
- *
- * This is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License version 2.1, as published by the Free Software
- * Foundation.  See file COPYING.
- *
- */
-
-
-#include <boost/system/system_error.hpp>
-
-#include "common/common_init.h"
-
-#include "global/global_init.h"
-
-#include "RADOSImpl.h"
-
-namespace RADOS {
-namespace detail {
-
-RADOS::RADOS(boost::asio::io_context& ioctx,
-	     boost::intrusive_ptr<CephContext> _cct)
-  : Dispatcher(_cct.detach()),
-    ioctx(ioctx),
-    monclient(cct, ioctx),
-    moncsd(monclient),
-    mgrclient(cct, nullptr, &monclient.monmap),
-    mgrcsd(mgrclient) {
-  auto err = monclient.build_initial_monmap();
-  if (err < 0)
-    throw std::system_error(ceph::to_error_code(err));
-
-  messenger.reset(Messenger::create_client_messenger(cct, "radosclient"));
-  if (!messenger)
-    throw std::bad_alloc();
-
-  // require OSDREPLYMUX feature.  this means we will fail to talk to
-  // old servers.  this is necessary because otherwise we won't know
-  // how to decompose the reply data into its constituent pieces.
-  messenger->set_default_policy(
-    Messenger::Policy::lossy_client(CEPH_FEATURE_OSDREPLYMUX));
-
-  objecter.reset(new Objecter(cct, messenger.get(), &monclient,
-			      ioctx,
-			      cct->_conf->rados_mon_op_timeout,
-			      cct->_conf->rados_osd_op_timeout));
-
-  objecter->set_balanced_budget();
-  monclient.set_messenger(messenger.get());
-  mgrclient.set_messenger(messenger.get());
-  objecter->init();
-  messenger->add_dispatcher_head(&mgrclient);
-  messenger->add_dispatcher_tail(objecter.get());
-  messenger->start();
-  monclient.set_want_keys(CEPH_ENTITY_TYPE_MON | CEPH_ENTITY_TYPE_OSD | CEPH_ENTITY_TYPE_MGR);
-  err = monclient.init();
-  if (err) {
-    throw boost::system::system_error(ceph::to_error_code(err));
-  }
-  err = monclient.authenticate(cct->_conf->client_mount_timeout);
-  if (err) {
-    throw boost::system::system_error(ceph::to_error_code(err));
-  }
-  messenger->set_myname(entity_name_t::CLIENT(monclient.get_global_id()));
-  // Detect older cluster, put mgrclient into compatible mode
-  mgrclient.set_mgr_optional(
-      !get_required_monitor_features().contains_all(
-        ceph::features::mon::FEATURE_LUMINOUS));
-
-  // MgrClient needs this (it doesn't have MonClient reference itself)
-  monclient.sub_want("mgrmap", 0, 0);
-  monclient.renew_subs();
-
-  mgrclient.init();
-  objecter->set_client_incarnation(0);
-  objecter->start();
-
-  messenger->add_dispatcher_tail(this);
-
-  std::unique_lock l(lock);
-  instance_id = monclient.get_global_id();
-}
-
-bool RADOS::ms_dispatch(Message *m)
-{
-  switch (m->get_type()) {
-  // OSD
-  case CEPH_MSG_OSD_MAP:
-    m->put();
-    return true;
-  }
-  return false;
-}
-
-void RADOS::ms_handle_connect(Connection *con) {}
-bool RADOS::ms_handle_reset(Connection *con) {
-  return false;
-}
-void RADOS::ms_handle_remote_reset(Connection *con) {}
-bool RADOS::ms_handle_refused(Connection *con) {
-  return false;
-}
-
-RADOS::~RADOS() = default;
-}
-}
diff --git a/src/RADOS/RADOSImpl.h b/src/RADOS/RADOSImpl.h
deleted file mode 100644
index c7ce18a9e7d..00000000000
--- a/src/RADOS/RADOSImpl.h
+++ /dev/null
@@ -1,105 +0,0 @@
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2004-2012 Sage Weil <sage@newdream.net>
- *
- * This is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License version 2.1, as published by the Free Software
- * Foundation.  See file COPYING.
- *
- */
-#ifndef CEPH_LIBRADOS_RADOSCLIENT_H
-#define CEPH_LIBRADOS_RADOSCLIENT_H
-
-#include <functional>
-#include <memory>
-#include <string>
-
-#include <boost/asio.hpp>
-#include <boost/intrusive_ptr.hpp>
-
-#include "common/ceph_context.h"
-#include "common/ceph_mutex.h"
-
-#include "mon/MonClient.h"
-
-#include "mgr/MgrClient.h"
-
-#include "osdc/Objecter.h"
-
-
-
-namespace RADOS {
-  class RADOS;
-namespace detail {
-
-class RADOS : public Dispatcher
-{
-  friend ::RADOS::RADOS;
-  struct MsgDeleter {
-    void operator()(Messenger* p) const {
-      if (p) {
-	p->shutdown();
-	p->wait();
-      }
-      delete p;
-    }
-  };
-
-  struct ObjDeleter {
-    void operator()(Objecter* p) const {
-      if (p) {
-	p->shutdown();
-      }
-      delete p;
-    }
-  };
-
-  template<typename T>
-  struct scoped_shutdown {
-    T& m;
-    scoped_shutdown(T& m) : m(m) {}
-
-    ~scoped_shutdown() {
-      m.shutdown();
-    }
-  };
-
-  boost::asio::io_context& ioctx;
-  ceph::mutex lock = ceph::make_mutex("RADOS_unleashed::_::RADOSImpl");
-  int instance_id = -1;
-
-  std::unique_ptr<Messenger, MsgDeleter> messenger;
-
-  MonClient monclient;
-  scoped_shutdown<MonClient> moncsd;
-
-  MgrClient mgrclient;
-  scoped_shutdown<MgrClient> mgrcsd;
-
-  std::unique_ptr<Objecter, ObjDeleter> objecter;
-
-
-public:
-
-  RADOS(boost::asio::io_context& ioctx, boost::intrusive_ptr<CephContext> cct);
-  ~RADOS();
-  bool ms_dispatch(Message *m) override;
-  void ms_handle_connect(Connection *con) override;
-  bool ms_handle_reset(Connection *con) override;
-  void ms_handle_remote_reset(Connection *con) override;
-  bool ms_handle_refused(Connection *con) override;
-  mon_feature_t get_required_monitor_features() const {
-    return monclient.with_monmap(std::mem_fn(&MonMap::get_required_features));
-  }
-  int get_instance_id() const {
-    return instance_id;
-  }
-};
-}
-}
-
-#endif
diff --git a/src/ceph_fuse.cc b/src/ceph_fuse.cc
index 4f889328f95..5ac18116f12 100644
--- a/src/ceph_fuse.cc
+++ b/src/ceph_fuse.cc
@@ -16,9 +16,7 @@
 #include <sys/utsname.h>
 #include <iostream>
 #include <string>
-#include <optional>
 
-#include "common/async/context_pool.h"
 #include "common/config.h"
 #include "common/errno.h"
 
@@ -46,8 +44,6 @@
 
 #define dout_context g_ceph_context
 
-ceph::async::io_context_pool icp;
-
 static void fuse_usage()
 {
   const char* argv[] = {
@@ -227,8 +223,7 @@ int main(int argc, const char **argv, const char *envp[]) {
     int tester_r = 0;
     void *tester_rp = nullptr;
 
-    icp.start(cct->_conf.get_val<std::uint64_t>("client_asio_thread_count"));
-    MonClient *mc = new MonClient(g_ceph_context, icp);
+    MonClient *mc = new MonClient(g_ceph_context);
     int r = mc->build_initial_monmap();
     if (r == -EINVAL) {
       cerr << "failed to generate initial mon list" << std::endl;
@@ -243,7 +238,7 @@ int main(int argc, const char **argv, const char *envp[]) {
     messenger->set_policy(entity_name_t::TYPE_MDS,
 			  Messenger::Policy::lossless_client(0));
 
-    client = new StandaloneClient(messenger, mc, icp);
+    client = new StandaloneClient(messenger, mc);
     if (filer_flags) {
       client->set_filer_flags(filer_flags);
     }
@@ -310,7 +305,6 @@ int main(int argc, const char **argv, const char *envp[]) {
     client->unmount();
     cfuse->finalize();
   out_shutdown:
-    icp.stop();
     client->shutdown();
   out_init_failed:
     unregister_async_signal_handler(SIGHUP, sighup_handler);
diff --git a/src/ceph_mds.cc b/src/ceph_mds.cc
index 4e6714ac44e..38e673755a9 100644
--- a/src/ceph_mds.cc
+++ b/src/ceph_mds.cc
@@ -20,7 +20,6 @@
 #include <iostream>
 #include <string>
 
-#include "common/async/context_pool.h"
 #include "include/ceph_features.h"
 #include "include/compat.h"
 #include "include/random.h"
@@ -177,8 +176,7 @@ int main(int argc, const char **argv)
   register_async_signal_handler(SIGHUP, sighup_handler);
   
   // get monmap
-  ceph::async::io_context_pool ctxpool(2);
-  MonClient mc(g_ceph_context, ctxpool);
+  MonClient mc(g_ceph_context);
   if (mc.build_initial_monmap() < 0)
     forker.exit(1);
   global_init_chdir(g_ceph_context);
@@ -186,7 +184,7 @@ int main(int argc, const char **argv)
   msgr->start();
 
   // start mds
-  mds = new MDSDaemon(g_conf()->name.get_id().c_str(), msgr, &mc, ctxpool);
+  mds = new MDSDaemon(g_conf()->name.get_id().c_str(), msgr, &mc);
 
   // in case we have to respawn...
   mds->orig_argc = argc;
@@ -217,7 +215,6 @@ int main(int argc, const char **argv)
   shutdown_async_signal_handler();
 
  shutdown:
-  ctxpool.stop();
   // yuck: grab the mds lock, so we can be sure that whoever in *mds
   // called shutdown finishes what they were doing.
   mds->mds_lock.lock();
@@ -241,3 +238,4 @@ int main(int argc, const char **argv)
 
   return 0;
 }
+
diff --git a/src/ceph_osd.cc b/src/ceph_osd.cc
index 18101dee122..3e03e863239 100644
--- a/src/ceph_osd.cc
+++ b/src/ceph_osd.cc
@@ -664,10 +664,7 @@ flushjournal_out:
 
   srand(time(NULL) + getpid());
 
-  ceph::async::io_context_pool poolctx(
-    cct->_conf.get_val<std::uint64_t>("osd_asio_thread_count"));
-
-  MonClient mc(g_ceph_context, poolctx);
+  MonClient mc(g_ceph_context);
   if (mc.build_initial_monmap() < 0)
     return -1;
   global_init_chdir(g_ceph_context);
@@ -688,8 +685,7 @@ flushjournal_out:
 		   ms_objecter,
 		   &mc,
 		   data_path,
-		   journal_path,
-		   poolctx);
+		   journal_path);
 
   int err = osdptr->pre_init();
   if (err < 0) {
@@ -744,7 +740,6 @@ flushjournal_out:
   shutdown_async_signal_handler();
 
   // done
-  poolctx.stop();
   delete osdptr;
   delete ms_public;
   delete ms_hb_front_client;
diff --git a/src/ceph_syn.cc b/src/ceph_syn.cc
index 165ea42308a..50e26f2815a 100644
--- a/src/ceph_syn.cc
+++ b/src/ceph_syn.cc
@@ -18,7 +18,6 @@
 
 #include "common/config.h"
 
-#include "common/async/context_pool.h"
 #include "client/SyntheticClient.h"
 #include "client/Client.h"
 
@@ -51,8 +50,7 @@ int main(int argc, const char **argv, char *envp[])
   pick_addresses(g_ceph_context, CEPH_PICK_ADDRESS_PUBLIC);
 
   // get monmap
-  ceph::async::io_context_pool  poolctx(1);
-  MonClient mc(g_ceph_context, poolctx);
+  MonClient mc(g_ceph_context);
   if (mc.build_initial_monmap() < 0)
     return -1;
 
@@ -66,9 +64,9 @@ int main(int argc, const char **argv, char *envp[])
     messengers[i] = Messenger::create_client_messenger(g_ceph_context,
 						       "synclient");
     messengers[i]->bind(g_conf()->public_addr);
-    mclients[i] = new MonClient(g_ceph_context, poolctx);
+    mclients[i] = new MonClient(g_ceph_context);
     mclients[i]->build_initial_monmap();
-    auto client = new StandaloneClient(messengers[i], mclients[i], poolctx);
+    auto client = new StandaloneClient(messengers[i], mclients[i]);
     client->set_filer_flags(syn_filer_flags);
     SyntheticClient *syn = new SyntheticClient(client);
     clients.push_back(client);
@@ -81,8 +79,6 @@ int main(int argc, const char **argv, char *envp[])
        ++p)
     (*p)->start_thread();
 
-  poolctx.stop();
-
   //cout << "waiting for client(s) to finish" << std::endl;
   while (!clients.empty()) {
     Client *client = clients.front();
@@ -103,3 +99,4 @@ int main(int argc, const char **argv, char *envp[])
   }
   return 0;
 }
+
diff --git a/src/client/Client.cc b/src/client/Client.cc
index a9528891fa8..e042539c3b5 100644
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -29,8 +29,6 @@
 #include <boost/lexical_cast.hpp>
 #include <boost/fusion/include/std_pair.hpp>
 
-#include "common/async/waiter.h"
-
 #if defined(__FreeBSD__)
 #define XATTR_CREATE    0x1
 #define XATTR_REPLACE   0x2
@@ -46,7 +44,6 @@
 
 #include "common/config.h"
 #include "common/version.h"
-#include "common/async/waiter.h"
 
 #include "mon/MonClient.h"
 
@@ -127,9 +124,6 @@
 
 #define DEBUG_GETATTR_CAPS (CEPH_CAP_XATTR_SHARED)
 
-namespace bs = boost::system;
-using ceph::async::waiter;
-
 void client_flush_set_callback(void *p, ObjectCacher::ObjectSet *oset)
 {
   Client *client = static_cast<Client*>(p);
@@ -5649,22 +5643,22 @@ int Client::authenticate()
 
 int Client::fetch_fsmap(bool user)
 {
+  int r;
   // Retrieve FSMap to enable looking up daemon addresses.  We need FSMap
   // rather than MDSMap because no one MDSMap contains all the daemons, and
   // a `tell` can address any daemon.
   version_t fsmap_latest;
-  boost::system::error_code ec;
   do {
-    waiter<bs::error_code, version_t, version_t> w;
-    monclient->get_version("fsmap", w);
+    C_SaferCond cond;
+    monclient->get_version("fsmap", &fsmap_latest, NULL, &cond);
     client_lock.unlock();
-    std::tie(ec, fsmap_latest, std::ignore) = w.wait();
+    r = cond.wait();
     client_lock.lock();
-  } while (ec == boost::system::errc::resource_unavailable_try_again);
+  } while (r == -EAGAIN);
 
-  if (ec) {
-    lderr(cct) << "Failed to learn FSMap version: " << ec << dendl;
-    return ceph::from_error_code(ec);
+  if (r < 0) {
+    lderr(cct) << "Failed to learn FSMap version: " << cpp_strerror(r) << dendl;
+    return r;
   }
 
   ldout(cct, 10) << __func__ << " learned FSMap version " << fsmap_latest << dendl;
@@ -11604,9 +11598,9 @@ void Client::_setxattr_maybe_wait_for_osdmap(const char *name, const void *value
     });
 
     if (r == -ENOENT) {
-      waiter<bs::error_code> w;
-      objecter->wait_for_latest_osdmap(w);
-      w.wait();
+      C_SaferCond ctx;
+      objecter->wait_for_latest_osdmap(&ctx);
+      ctx.wait();
     }
   }
 }
@@ -14205,7 +14199,7 @@ int Client::check_pool_perm(Inode *in, int need)
 
     C_SaferCond rd_cond;
     ObjectOperation rd_op;
-    rd_op.stat(nullptr, nullptr, nullptr);
+    rd_op.stat(NULL, (ceph::real_time*)nullptr, NULL);
 
     objecter->mutate(oid, OSDMap::file_to_object_locator(in->layout), rd_op,
 		     nullsnapc, ceph::real_clock::now(), 0, &rd_cond);
@@ -14392,7 +14386,7 @@ void Client::set_session_timeout(unsigned timeout)
 int Client::start_reclaim(const std::string& uuid, unsigned flags,
 			  const std::string& fs_name)
 {
-  std::unique_lock l(client_lock);
+  std::lock_guard l(client_lock);
   if (!initialized)
     return -ENOTCONN;
 
@@ -14468,15 +14462,13 @@ int Client::start_reclaim(const std::string& uuid, unsigned flags,
 
   // use blacklist to check if target session was killed
   // (config option mds_session_blacklist_on_evict needs to be true)
-  ldout(cct, 10) << __func__ << ": waiting for OSD epoch " << reclaim_osd_epoch << dendl;
-  waiter<bs::error_code> w;
-  objecter->wait_for_map(reclaim_osd_epoch, w);
-  l.unlock();
-  auto ec = w.wait();
-  l.lock();
-
-  if (ec)
-    return ceph::from_error_code(ec);
+  C_SaferCond cond;
+  if (!objecter->wait_for_map(reclaim_osd_epoch, &cond)) {
+    ldout(cct, 10) << __func__ << ": waiting for OSD epoch " << reclaim_osd_epoch << dendl;
+    client_lock.unlock();
+    cond.wait();
+    client_lock.lock();
+  }
 
   bool blacklisted = objecter->with_osdmap(
       [this](const OSDMap &osd_map) -> bool {
@@ -14600,9 +14592,8 @@ mds_rank_t Client::_get_random_up_mds() const
 }
 
 
-StandaloneClient::StandaloneClient(Messenger *m, MonClient *mc,
-				   boost::asio::io_context& ictx)
-  : Client(m, mc, new Objecter(m->cct, m, mc, ictx, 0, 0))
+StandaloneClient::StandaloneClient(Messenger *m, MonClient *mc)
+    : Client(m, mc, new Objecter(m->cct, m, mc, NULL, 0, 0))
 {
   monclient->set_messenger(m);
   objecter->set_client_incarnation(0);
diff --git a/src/client/Client.h b/src/client/Client.h
index b75aa47fc1c..a88abb5d86e 100644
--- a/src/client/Client.h
+++ b/src/client/Client.h
@@ -1305,7 +1305,7 @@ private:
 class StandaloneClient : public Client
 {
 public:
-  StandaloneClient(Messenger *m, MonClient *mc, boost::asio::io_context& ictx);
+  StandaloneClient(Messenger *m, MonClient *mc);
 
   ~StandaloneClient() override;
 
diff --git a/src/cls/rgw/cls_rgw_types.h b/src/cls/rgw/cls_rgw_types.h
index 1f519acb751..2b2907ba6fa 100644
--- a/src/cls/rgw/cls_rgw_types.h
+++ b/src/cls/rgw/cls_rgw_types.h
@@ -242,7 +242,7 @@ void decode_packed_val(T& val, bufferlist::const_iterator& bl)
       }
       break;
     default:
-      throw buffer::malformed_input();
+      throw buffer::error();
   }
 }
 
diff --git a/src/common/async/context_pool.h b/src/common/async/context_pool.h
deleted file mode 100644
index 2e90ba95a25..00000000000
--- a/src/common/async/context_pool.h
+++ /dev/null
@@ -1,97 +0,0 @@
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2018 Red Hat <contact@redhat.com>
- * Author: Adam C. Emerson <aemerson@redhat.com>
- *
- * This is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License version 2.1, as published by the Free Software
- * Foundation.  See file COPYING.
- *
- */
-
-#ifndef CEPH_COMMON_ASYNC_CONTEXT_POOL_H
-#define CEPH_COMMON_ASYNC_CONTEXT_POOL_H
-
-#include <cstddef>
-#include <cstdint>
-#include <mutex>
-#include <optional>
-#include <thread>
-#include <vector>
-
-#include <boost/asio/io_context.hpp>
-#include <boost/asio/executor_work_guard.hpp>
-
-#include "common/ceph_mutex.h"
-
-namespace ceph::async {
-class io_context_pool {
-  std::vector<std::thread> threadvec;
-  boost::asio::io_context ioctx;
-  std::optional<boost::asio::executor_work_guard<
-		  boost::asio::io_context::executor_type>> guard;
-  ceph::mutex m = make_mutex("ceph::io_context_pool::m");
-
-  void cleanup() noexcept {
-    guard = std::nullopt;
-    for (auto& th : threadvec) {
-      th.join();
-    }
-    threadvec.clear();
-  }
-public:
-  io_context_pool() noexcept {}
-  io_context_pool(std::int16_t threadcnt) noexcept {
-    start(threadcnt);
-  }
-  ~io_context_pool() {
-    stop();
-  }
-  void start(std::int16_t threadcnt) noexcept {
-    auto l = std::scoped_lock(m);
-    if (threadvec.empty()) {
-      guard.emplace(boost::asio::make_work_guard(ioctx));
-      ioctx.restart();
-      for (std::int16_t i = 0; i < threadcnt; ++i) {
-	// Mark this function as noexcept so any uncaught exceptions
-	// call terminate at point of throw. Otherwise, under
-	// libstdc++, they get caught by the thread cancellation
-	// infrastructure, unwinding the stack and making debugging
-	// much more difficult.
-	threadvec.emplace_back([this]() noexcept {
-				 ioctx.run();
-			       });
-      }
-    }
-  }
-  void finish() noexcept {
-    auto l = std::scoped_lock(m);
-    if (!threadvec.empty()) {
-      cleanup();
-    }
-  }
-  void stop() noexcept {
-    auto l = std::scoped_lock(m);
-    if (!threadvec.empty()) {
-      ioctx.stop();
-      cleanup();
-    }
-  }
-
-  boost::asio::io_context& get_io_context() {
-    return ioctx;
-  }
-  operator boost::asio::io_context&() {
-    return ioctx;
-  }
-  boost::asio::io_context::executor_type get_executor() {
-    return ioctx.get_executor();
-  }
-};
-}
-
-#endif // CEPH_COMMON_ASYNC_CONTEXT_POOL_H
diff --git a/src/common/async/waiter.h b/src/common/async/waiter.h
deleted file mode 100644
index 1e95d007c43..00000000000
--- a/src/common/async/waiter.h
+++ /dev/null
@@ -1,257 +0,0 @@
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
- *
- * This is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License version 2.1, as published by the Free Software
- * Foundation.  See file COPYING.
- *
- */
-
-#ifndef CEPH_COMMON_WAITER_H
-#define CEPH_COMMON_WAITER_H
-
-#include <condition_variable>
-#include <tuple>
-
-#include <boost/asio/async_result.hpp>
-
-#include "include/ceph_assert.h"
-#include "include/function2.hpp"
-
-#include "common/ceph_mutex.h"
-
-namespace ceph::async {
-namespace detail {
-// For safety reasons (avoiding undefined behavior around sequence
-// points) std::reference_wrapper disallows move construction. This
-// harms us in cases where we want to pass a reference in to something
-// that unavoidably moves.
-//
-// It should not be used generally.
-template<typename T>
-class rvalue_reference_wrapper {
-public:
-  // types
-  using type = T;
-
-  rvalue_reference_wrapper(T& r) noexcept
-    : p(std::addressof(r)) {}
-
-  // We write our semantics to match those of reference collapsing. If
-  // we're treated as an lvalue, collapse to one.
-
-  rvalue_reference_wrapper(const rvalue_reference_wrapper&) noexcept = default;
-  rvalue_reference_wrapper(rvalue_reference_wrapper&&) noexcept = default;
-
-  // assignment
-  rvalue_reference_wrapper& operator=(
-    const rvalue_reference_wrapper& x) noexcept = default;
-  rvalue_reference_wrapper& operator=(
-    rvalue_reference_wrapper&& x) noexcept = default;
-
-  operator T& () const noexcept {
-    return *p;
-  }
-  T& get() const noexcept {
-    return *p;
-  }
-
-  operator T&& () noexcept {
-    return std::move(*p);
-  }
-  T&& get() noexcept {
-    return std::move(*p);
-  }
-
-  template<typename... Args>
-  std::result_of_t<T&(Args&&...)> operator ()(Args&&... args ) const {
-    return (*p)(std::forward<Args>(args)...);
-  }
-
-  template<typename... Args>
-  std::result_of_t<T&&(Args&&...)> operator ()(Args&&... args ) {
-    return std::move(*p)(std::forward<Args>(args)...);
-  }
-
-private:
-  T* p;
-};
-
-class base {
-protected:
-  ceph::mutex lock = ceph::make_mutex("ceph::async::detail::base::lock");
-  ceph::condition_variable cond;
-  bool has_value = false;
-
-  ~base() = default;
-
-  auto wait_base() {
-    std::unique_lock l(lock);
-    cond.wait(l, [this](){ return has_value; });
-    return l;
-  }
-
-  auto exec_base() {
-    std::unique_lock l(lock);
-    // There's no really good way to handle being called twice
-    // without being reset.
-    ceph_assert(!has_value);
-    has_value = true;
-    cond.notify_one();
-    return l;
-  }
-};
-}
-
-// waiter is a replacement for C_SafeCond and friends. It is the
-// moral equivalent of a future but plays well with a world of
-// callbacks.
-template<typename ...S>
-class waiter;
-
-template<>
-class waiter<> final : public detail::base {
-public:
-  void wait() {
-    wait_base();
-    has_value = false;
-  }
-
-  void operator()() {
-    exec_base();
-  }
-
-  auto ref() {
-    return detail::rvalue_reference_wrapper(*this);
-  }
-
-
-  operator fu2::unique_function<void() &&>() {
-    return fu2::unique_function<void() &&>(ref());
-  }
-};
-
-template<typename Ret>
-class waiter<Ret> final : public detail::base {
-  std::aligned_storage_t<sizeof(Ret)> ret;
-
-public:
-  Ret wait() {
-    auto l = wait_base();
-    auto r = reinterpret_cast<Ret*>(&ret);
-    auto t = std::move(*r);
-    r->~Ret();
-    has_value = false;
-    return t;
-  }
-
-  void operator()(Ret&& _ret) {
-    auto l = exec_base();
-    auto r = reinterpret_cast<Ret*>(&ret);
-    *r = std::move(_ret);
-  }
-
-  void operator()(const Ret& _ret) {
-    auto l = exec_base();
-    auto r = reinterpret_cast<Ret*>(&ret);
-    *r = std::move(_ret);
-  }
-
-  auto ref() {
-    return detail::rvalue_reference_wrapper(*this);
-  }
-
-  operator fu2::unique_function<void(Ret) &&>() {
-    return fu2::unique_function<void(Ret) &&>(ref());
-  }
-
-  ~waiter() {
-    if (has_value)
-      reinterpret_cast<Ret*>(&ret)->~Ret();
-  }
-};
-
-template<typename ...Ret>
-class waiter final : public detail::base {
-  std::tuple<Ret...> ret;
-
-public:
-  std::tuple<Ret...> wait() {
-    using std::tuple;
-    auto l = wait_base();
-    return std::move(ret);
-    auto r = reinterpret_cast<std::tuple<Ret...>*>(&ret);
-    auto t = std::move(*r);
-    r->~tuple<Ret...>();
-    has_value = false;
-    return t;
-  }
-
-  void operator()(Ret&&... _ret) {
-    auto l = exec_base();
-    auto r = reinterpret_cast<std::tuple<Ret...>*>(&ret);
-    *r = std::forward_as_tuple(_ret...);
-  }
-
-  void operator()(const Ret&... _ret) {
-    auto l = exec_base();
-    auto r = reinterpret_cast<std::tuple<Ret...>*>(&ret);
-    *r = std::forward_as_tuple(_ret...);
-  }
-
-  auto ref() {
-    return detail::rvalue_reference_wrapper(*this);
-  }
-
-  operator fu2::unique_function<void(Ret...) &&>() {
-    return fu2::unique_function<void(Ret...) &&>(ref());
-  }
-
-  ~waiter() {
-    using std::tuple;
-    if (has_value)
-      reinterpret_cast<tuple<Ret...>*>(&ret)->~tuple<Ret...>();
-  }
-};
-}
-
-namespace boost::asio {
-template<typename ...S>
-class async_result<ceph::async::waiter<S...>, void(S...)> {
-public:
-  using completion_handler_type =
-    ceph::async::detail::rvalue_reference_wrapper<ceph::async::waiter<S...>>;
-
-  using return_type = void;
-
-  explicit async_result(completion_handler_type& h) {}
-
-  return_type get() {}
-
-  async_result(const async_result&) = delete;
-  async_result& operator=(const async_result&) = delete;
-};
-
-template<typename ...S>
-struct async_completion<ceph::async::waiter<S...>, void(S...)> {
-  using completion_handler_type =
-    typename boost::asio::async_result<ceph::async::waiter<S...>,
-				       void(S...)>::completion_handler_type;
-
-  explicit async_completion(ceph::async::waiter<S...>& w)
-    : completion_handler(ceph::async::detail::rvalue_reference_wrapper(w)),
-      result(completion_handler){}
-
-  completion_handler_type completion_handler;
-
-  /// The result of the asynchronous operation's initiating function.
-  async_result<ceph::async::waiter<S...>, void(S...)> result;
-};
-}
-
-#endif // CEPH_COMMON_WAITER_H
diff --git a/src/common/buffer.cc b/src/common/buffer.cc
index 9ea83101826..b7d3591600a 100644
--- a/src/common/buffer.cc
+++ b/src/common/buffer.cc
@@ -33,7 +33,6 @@
 #include "common/valgrind.h"
 #include "common/deleter.h"
 #include "common/RWLock.h"
-#include "common/error_code.h"
 #include "include/spinlock.h"
 #include "include/scope_guard.h"
 
@@ -71,6 +70,21 @@ static ceph::spinlock debug_lock;
     return buffer_missed_crc;
   }
 
+  const char * buffer::error::what() const throw () {
+    return "buffer::exception";
+  }
+  const char * buffer::bad_alloc::what() const throw () {
+    return "buffer::bad_alloc";
+  }
+  const char * buffer::end_of_buffer::what() const throw () {
+    return "buffer::end_of_buffer";
+  }
+  const char * buffer::malformed_input::what() const throw () {
+    return buf;
+  }
+  buffer::error_code::error_code(int error) :
+    buffer::malformed_input(cpp_strerror(error).c_str()), code(error) {}
+
   /*
    * raw_combined is always placed within a single allocation along
    * with the data buffer.  the data goes at the beginning, and
@@ -2252,6 +2266,11 @@ std::ostream& buffer::operator<<(std::ostream& out, const buffer::list& bl) {
   return out;
 }
 
+std::ostream& buffer::operator<<(std::ostream& out, const buffer::error& e)
+{
+  return out << e.what();
+}
+
 MEMPOOL_DEFINE_OBJECT_FACTORY(buffer::raw_malloc, buffer_raw_malloc,
 			      buffer_meta);
 MEMPOOL_DEFINE_OBJECT_FACTORY(buffer::raw_posix_aligned,
@@ -2264,81 +2283,3 @@ MEMPOOL_DEFINE_OBJECT_FACTORY(buffer::raw_unshareable, buffer_raw_unshareable,
 MEMPOOL_DEFINE_OBJECT_FACTORY(buffer::raw_static, buffer_raw_static,
 			      buffer_meta);
 
-
-namespace ceph::buffer {
-inline namespace v14_2_0 {
-
-class buffer_error_category : public ceph::converting_category {
-public:
-  buffer_error_category(){}
-  const char* name() const noexcept override;
-  std::string message(int ev) const override;
-  boost::system::error_condition default_error_condition(int ev) const noexcept
-    override;
-  using ceph::converting_category::equivalent;
-  bool equivalent(int ev, const boost::system::error_condition& c) const
-    noexcept override;
-  int from_code(int ev) const noexcept override;
-};
-
-const char* buffer_error_category::name() const noexcept {
-  return "buffer";
-}
-
-std::string buffer_error_category::message(int ev) const {
-  using ceph::buffer::errc;
-  if (ev == 0)
-    return "No error";
-
-  switch (static_cast<errc>(ev)) {
-  case errc::bad_alloc:
-    return "Bad allocation";
-
-  case errc::end_of_buffer:
-    return "End of buffer";
-
-  case errc::malformed_input:
-    return "Malformed input";
-  }
-
-  return "Unknown error";
-}
-
-boost::system::error_condition
-buffer_error_category::default_error_condition(int ev)const noexcept {
-  using ceph::buffer::errc;
-  switch (static_cast<errc>(ev)) {
-  case errc::bad_alloc:
-    return boost::system::errc::not_enough_memory;
-  case errc::end_of_buffer:
-  case errc::malformed_input:
-    return boost::system::errc::io_error;
-  }
-  return { ev, *this };
-}
-
-bool buffer_error_category::equivalent(int ev, const boost::system::error_condition& c) const noexcept {
-  return default_error_condition(ev) == c;
-}
-
-int buffer_error_category::from_code(int ev) const noexcept {
-  using ceph::buffer::errc;
-  switch (static_cast<errc>(ev)) {
-  case errc::bad_alloc:
-    return -ENOMEM;
-
-  case errc::end_of_buffer:
-    return -EIO;
-
-  case errc::malformed_input:
-    return -EIO;
-  }
-  return -EDOM;
-}
-
-const boost::system::error_category& buffer_category() noexcept {
-  static const buffer_error_category c;
-  return c;
-}
-}
-}
diff --git a/src/common/ceph_context.h b/src/common/ceph_context.h
index ae590865945..6b6e9482b99 100644
--- a/src/common/ceph_context.h
+++ b/src/common/ceph_context.h
@@ -25,8 +25,6 @@
 #include <typeinfo>
 #include <typeindex>
 
-#include <boost/intrusive_ptr.hpp>
-
 #include "include/any.h"
 
 #include "common/cmdparse.h"
@@ -365,15 +363,4 @@ private:
 };
 #endif	// WITH_SEASTAR
 
-inline void intrusive_ptr_add_ref(CephContext* cct)
-{
-  cct->get();
-}
-
-inline void intrusive_ptr_release(CephContext* cct)
-{
-  cct->put();
-}
-
-
 #endif
diff --git a/src/common/ceph_time.h b/src/common/ceph_time.h
index a2018703a27..ae4a17c7812 100644
--- a/src/common/ceph_time.h
+++ b/src/common/ceph_time.h
@@ -18,7 +18,6 @@
 #include <chrono>
 #include <iostream>
 #include <string>
-#include <optional>
 #include <sys/time.h>
 
 #include "include/ceph_assert.h"
@@ -435,9 +434,6 @@ namespace ceph {
       return std::chrono::duration_cast<timespan>(
 	std::chrono::duration<double>(d));
     }
-    inline std::optional<timespan> maybe_timespan(const double d) {
-      return d ? std::make_optional(make_timespan(d)) : std::nullopt;
-    }
   }
 
   std::ostream& operator<<(std::ostream& m, const timespan& t);
diff --git a/src/common/ceph_timer.h b/src/common/ceph_timer.h
index 0090f7ccdfe..b29fac9c3fc 100644
--- a/src/common/ceph_timer.h
+++ b/src/common/ceph_timer.h
@@ -19,9 +19,14 @@
 #include <thread>
 #include <boost/intrusive/set.hpp>
 
-#include "common/detail/construct_suspended.h"
-
 namespace ceph {
+
+  /// Newly constructed timer should be suspended at point of
+  /// construction.
+
+  struct construct_suspended_t { };
+  constexpr construct_suspended_t construct_suspended { };
+
   namespace timer_detail {
     using boost::intrusive::member_hook;
     using boost::intrusive::set_member_hook;
diff --git a/src/common/config.cc b/src/common/config.cc
index 4c2af4ce604..0259ef2f8ee 100644
--- a/src/common/config.cc
+++ b/src/common/config.cc
@@ -1306,7 +1306,7 @@ void md_config_t::_get_my_sections(const ConfigValues& values,
 {
   sections.push_back(values.name.to_str());
 
-  sections.push_back(values.name.get_type_name().data());
+  sections.push_back(values.name.get_type_name());
 
   sections.push_back("global");
 }
diff --git a/src/common/detail/construct_suspended.h b/src/common/detail/construct_suspended.h
deleted file mode 100644
index 521bda0f833..00000000000
--- a/src/common/detail/construct_suspended.h
+++ /dev/null
@@ -1,24 +0,0 @@
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2018 Red Hat <contact@redhat.com>
- * Author: Adam C. Emerson <aemerson@redhat.com>
- *
- * This is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License version 2.1, as published by the Free Software
- * Foundation.  See file COPYING.
- *
- */
-
-#ifndef CEPH_COMMON_DETAIL_CONSTRUCT_SUSPENDED_H
-#define CEPH_COMMON_DETAIL_CONSTRUCT_SUSPENDED_H
-
-namespace ceph {
-  struct construct_suspended_t { };
-  inline constexpr construct_suspended_t construct_suspended { };
-}
-
-#endif // CEPH_COMMON_DETAIL_CONSTRUCT_SUSPENDED_H
diff --git a/src/common/entity_name.cc b/src/common/entity_name.cc
index 2eb24829a1c..37d02bd94df 100644
--- a/src/common/entity_name.cc
+++ b/src/common/entity_name.cc
@@ -42,22 +42,22 @@ to_cstr() const
 }
 
 bool EntityName::
-from_str(std::string_view s)
+from_str(const string& s)
 {
   size_t pos = s.find('.');
 
   if (pos == string::npos)
     return false;
-
-  auto type_ = s.substr(0, pos);
-  auto id_ = s.substr(pos + 1);
+ 
+  string type_ = s.substr(0, pos);
+  string id_ = s.substr(pos + 1);
   if (set(type_, id_))
     return false;
   return true;
 }
 
 void EntityName::
-set(uint32_t type_, std::string_view id_)
+set(uint32_t type_, const std::string &id_)
 {
   type = type_;
   id = id_;
@@ -72,9 +72,9 @@ set(uint32_t type_, std::string_view id_)
 }
 
 int EntityName::
-set(std::string_view type_, std::string_view id_)
+set(const std::string &type_, const std::string &id_)
 {
-  uint32_t t = str_to_ceph_entity_type(type_);
+  uint32_t t = str_to_ceph_entity_type(type_.c_str());
   if (t == CEPH_ENTITY_TYPE_ANY)
     return -EINVAL;
   set(t, id_);
@@ -88,13 +88,13 @@ set_type(uint32_t type_)
 }
 
 int EntityName::
-set_type(std::string_view type_)
+set_type(const char *type_)
 {
   return set(type_, id);
 }
 
 void EntityName::
-set_id(std::string_view id_)
+set_id(const std::string &id_)
 {
   set(type, id_);
 }
@@ -106,13 +106,13 @@ void EntityName::set_name(entity_name_t n)
   set(n.type(), s);
 }
 
-std::string_view EntityName::
+const char* EntityName::
 get_type_str() const
 {
   return ceph_entity_type_name(type);
 }
 
-std::string_view EntityName::
+const char *EntityName::
 get_type_name() const
 {
   return ceph_entity_type_name(type);
diff --git a/src/common/entity_name.h b/src/common/entity_name.h
index 886c4b4946f..1dd56f66d74 100644
--- a/src/common/entity_name.h
+++ b/src/common/entity_name.h
@@ -15,8 +15,6 @@
 #ifndef CEPH_COMMON_ENTITY_NAME_H
 #define CEPH_COMMON_ENTITY_NAME_H
 
-#include <string_view>
-
 #include <ifaddrs.h>
 
 #include "msg/msg_types.h"
@@ -44,15 +42,15 @@ struct EntityName
 
   const std::string& to_str() const;
   const char *to_cstr() const;
-  bool from_str(std::string_view s);
-  void set(uint32_t type_, std::string_view id_);
-  int set(std::string_view type_, std::string_view id_);
+  bool from_str(const std::string& s);
+  void set(uint32_t type_, const std::string &id_);
+  int set(const std::string &type_, const std::string &id_);
   void set_type(uint32_t type_);
-  int set_type(std::string_view type);
-  void set_id(std::string_view id_);
+  int set_type(const char *type);
+  void set_id(const std::string &id_);
   void set_name(entity_name_t n);
 
-  std::string_view get_type_str() const;
+  const char* get_type_str() const;
 
   uint32_t get_type() const { return type; }
   bool is_osd() const { return get_type() == CEPH_ENTITY_TYPE_OSD; }
@@ -61,7 +59,7 @@ struct EntityName
   bool is_client() const { return get_type() == CEPH_ENTITY_TYPE_CLIENT; }
   bool is_mon() const { return get_type() == CEPH_ENTITY_TYPE_MON; }
 
-  std::string_view get_type_name() const;
+  const char * get_type_name() const;
   const std::string &get_id() const;
   bool has_default_id() const;
 
diff --git a/src/common/error_code.cc b/src/common/error_code.cc
deleted file mode 100644
index 52eee7f1b5e..00000000000
--- a/src/common/error_code.cc
+++ /dev/null
@@ -1,182 +0,0 @@
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2017 Red Hat, Inc. <contact@redhat.com>
- *
- * Author: Adam C. Emerson <aemerson@redhat.com>
- *
- * This is free software; you can redistribute it and/or modify it
- * under the terms of the GNU Lesser General Public License version
- * 2.1, as published by the Free Software Foundation.  See file
- * COPYING.
- */
-
-#include <exception>
-
-#include "common/error_code.h"
-
-using boost::system::error_category;
-using boost::system::error_condition;
-using boost::system::generic_category;
-using boost::system::system_category;
-
-namespace ceph {
-
-// A category for error conditions particular to Ceph
-
-class ceph_error_category : public converting_category {
-public:
-  ceph_error_category(){}
-  const char* name() const noexcept override;
-  std::string message(int ev) const override;
-  using converting_category::equivalent;
-  bool equivalent(const boost::system::error_code& c,
-		  int ev) const noexcept override;
-  int from_code(int ev) const noexcept override;
-};
-
-const char* ceph_error_category::name() const noexcept {
-  return "ceph";
-}
-
-std::string ceph_error_category::message(int ev) const {
-  if (ev == 0)
-    return "No error";
-
-  switch (static_cast<errc>(ev)) {
-
-  case errc::not_in_map:
-    return "Map does not contain requested entry.";
-  case errc::does_not_exist:
-    return "Item does not exist";
-  case errc::failure:
-    return "An internal fault or inconsistency occurred";
-  case errc::exists:
-    return "Already exists";
-  case errc::limit_exceeded:
-    return "Attempt to use too much";
-  case errc::auth:
-    return "Authentication error";
-  case errc::conflict:
-    return "Conflict detected or precondition failed";
-  }
-
-  return "Unknown error.";
-}
-
-bool ceph_error_category::equivalent(const boost::system::error_code& c,
-				     int ev) const noexcept {
-  if (c.category() == system_category()) {
-    if (c.value() == boost::system::errc::no_such_file_or_directory) {
-      if (ev == static_cast<int>(errc::not_in_map) ||
-	  ev == static_cast<int>(errc::does_not_exist)) {
-	// Blargh. A bunch of stuff returns ENOENT now, so just to be safe.
-	return true;
-      }
-    }
-    if (c.value() == boost::system::errc::io_error) {
-      if (ev == static_cast<int>(errc::failure)) {
-	return true;
-      }
-    }
-    if (c.value() == boost::system::errc::file_exists) {
-      if (ev == static_cast<int>(errc::exists)) {
-	return true;
-      }
-    }
-    if (c.value() == boost::system::errc::no_space_on_device ||
-	c.value() == boost::system::errc::invalid_argument) {
-      if (ev == static_cast<int>(errc::limit_exceeded)) {
-	return true;
-      }
-    }
-    if (c.value() == boost::system::errc::operation_not_permitted) {
-      if (ev == static_cast<int>(ceph::errc::conflict)) {
-	return true;
-      }
-    }
-  }
-  return false;
-}
-
-int ceph_error_category::from_code(int ev) const noexcept {
-  if (ev == 0)
-    return 0;
-
-  switch (static_cast<errc>(ev)) {
-  case errc::not_in_map:
-  case errc::does_not_exist:
-    // What we use now.
-    return -ENOENT;
-  case errc::failure:
-    return -EIO;
-  case errc::exists:
-    return -EEXIST;
-  case errc::limit_exceeded:
-    return -EIO;
-  case errc::auth:
-    return -EACCES;
-  case errc::conflict:
-    return -EINVAL;
-  }
-  return -EDOM;
-}
-
-const error_category& ceph_category() noexcept {
-  static const ceph_error_category c;
-  return c;
-}
-
-
-// This is part of the glue for hooking new code to old. Since
-// Context* and other things give us integer codes from errno, wrap
-// them in an error_code.
-boost::system::error_code to_error_code(int ret) noexcept
-{
-  if (ret == 0)
-    return {};
-  return { std::abs(ret), boost::system::system_category() };
-}
-
-// This is more complicated. For the case of categories defined
-// elsewhere, we have to convert everything here.
-int from_error_code(boost::system::error_code e) noexcept
-{
-  if (!e)
-    return 0;
-
-  auto c = dynamic_cast<const converting_category*>(&e.category());
-  // For categories we define
-  if (c)
-    return c->from_code(e.value());
-
-  // For categories matching values of errno
-  if (e.category() == boost::system::system_category() ||
-      e.category() == boost::system::generic_category() ||
-      // ASIO uses the system category for these and matches system
-      // error values.
-      e.category() == boost::asio::error::get_netdb_category() ||
-      e.category() == boost::asio::error::get_addrinfo_category())
-    return -e.value();
-
-  if (e.category() == boost::asio::error::get_misc_category()) {
-    // These values are specific to asio
-    switch (e.value()) {
-    case boost::asio::error::already_open:
-      return -EIO;
-    case boost::asio::error::eof:
-      return -EIO;
-    case boost::asio::error::not_found:
-      return -ENOENT;
-    case boost::asio::error::fd_set_failure:
-      return -EINVAL;
-    }
-  }
-  // Add any other categories we use here.
-
-  // Marcus likes this as a sentinel for 'Error code? What error code?'
-  return -EDOM;
-}
-}
diff --git a/src/common/error_code.h b/src/common/error_code.h
deleted file mode 100644
index 2fc7cdd377c..00000000000
--- a/src/common/error_code.h
+++ /dev/null
@@ -1,75 +0,0 @@
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2017 Red Hat, Inc. <contact@redhat.com>
- *
- * Author: Adam C. Emerson <aemerson@redhat.com>
- *
- * This is free software; you can redistribute it and/or modify it
- * under the terms of the GNU Lesser General Public License version
- * 2.1, as published by the Free Software Foundation.  See file
- * COPYING.
- */
-
-#ifndef COMMON_CEPH_ERROR_CODE
-#define COMMON_CEPH_ERROR_CODE
-
-#include <netdb.h>
-
-#include <boost/system/error_code.hpp>
-#include <boost/asio.hpp>
-
-namespace ceph {
-
-// This is for error categories we define, so we can specify the
-// equivalent integral value at the point of definition.
-class converting_category : public boost::system::error_category {
-public:
-  virtual int from_code(int code) const noexcept = 0;
-};
-
-const boost::system::error_category& ceph_category() noexcept;
-
-enum class errc {
-  not_in_map = 1, // The requested item was not found in the map
-  does_not_exist, // Item does not exist
-  failure, // An internal fault or inconsistency
-  exists, // Already exists
-  limit_exceeded, // Attempting to use too much of something
-  auth, // May not be an auth failure. It could be that the
-	// preconditions to attempt auth failed.
-  conflict, // Conflict or precondition failure
-};
-}
-
-namespace boost {
-namespace system {
-template<>
-struct is_error_condition_enum<::ceph::errc> {
-  static const bool value = true;
-};
-template<>
-struct is_error_code_enum<::ceph::errc> {
-  static const bool value = false;
-};
-}
-}
-
-namespace ceph {
-//  explicit conversion:
-inline boost::system::error_code make_error_code(errc e) noexcept {
-  return { static_cast<int>(e), ceph_category() };
-}
-
-// implicit conversion:
-inline boost::system::error_condition make_error_condition(errc e) noexcept {
-  return { static_cast<int>(e), ceph_category() };
-}
-
-boost::system::error_code to_error_code(int ret) noexcept;
-int from_error_code(boost::system::error_code e) noexcept;
-}
-
-#endif // COMMON_CEPH_ERROR_CODE
diff --git a/src/common/options.cc b/src/common/options.cc
index 7f65b8bd048..ddfef20fce2 100644
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -5246,18 +5246,6 @@ std::vector<Option> get_global_options() {
     Option("debug_heartbeat_testing_span", Option::TYPE_INT, Option::LEVEL_DEV)
     .set_default(0)
     .set_description("Override 60 second periods for testing only"),
-
-    Option("librados_thread_count", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
-    .set_default(2)
-    .set_min(1)
-    .set_description("Size of thread pool for Objecter")
-    .add_tag("client"),
-
-    Option("osd_asio_thread_count", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
-    .set_default(2)
-    .set_min(1)
-    .set_description("Size of thread pool for ASIO completions")
-    .add_tag("osd")
   });
 }
 
@@ -7982,12 +7970,6 @@ std::vector<Option> get_mds_options() {
      .set_flag(Option::FLAG_RUNTIME)
      .set_description("max snapshots per directory")
      .set_long_description("maximum number of snapshots that can be created per directory"),
-
-    Option("mds_asio_thread_count", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
-    .set_default(2)
-    .set_min(1)
-    .set_description("Size of thread pool for ASIO completions")
-    .add_tag("mds")
   });
 }
 
@@ -8224,12 +8206,6 @@ std::vector<Option> get_mds_client_options() {
     Option("debug_allow_any_pool_priority", Option::TYPE_BOOL, Option::LEVEL_DEV)
     .set_default(false)
     .set_description("Allow any pool priority to be set to test conversion to new range"),
-
-    Option("client_asio_thread_count", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
-    .set_default(2)
-    .set_min(1)
-    .set_description("Size of thread pool for ASIO completions")
-    .add_tag("client")
   });
 }
 
diff --git a/src/common/snap_types.h b/src/common/snap_types.h
index 5fd1e08bc21..9802bdea126 100644
--- a/src/common/snap_types.h
+++ b/src/common/snap_types.h
@@ -51,7 +51,7 @@ struct SnapContext {
     seq = 0;
     snaps.clear();
   }
-  bool empty() const { return seq == 0; }
+  bool empty() { return seq == 0; }
 
   void encode(ceph::buffer::list& bl) const {
     using ceph::encode;
diff --git a/src/crimson/osd/osd.h b/src/crimson/osd/osd.h
index d0beee2fad8..843c57e6f73 100644
--- a/src/crimson/osd/osd.h
+++ b/src/crimson/osd/osd.h
@@ -27,13 +27,13 @@
 #include "crimson/osd/pg_map.h"
 #include "crimson/osd/osd_operations/peering_event.h"
 
-#include "messages/MOSDOp.h"
 #include "osd/PeeringState.h"
 #include "osd/osd_types.h"
 #include "osd/osd_perf_counters.h"
 #include "osd/PGPeeringEvent.h"
 
 class MOSDMap;
+class MOSDOp;
 class MOSDRepOpReply;
 class MOSDRepOp;
 class OSDMap;
diff --git a/src/crimson/osd/osd_operations/client_request.h b/src/crimson/osd/osd_operations/client_request.h
index 09658e2bf9d..8940194f62f 100644
--- a/src/crimson/osd/osd_operations/client_request.h
+++ b/src/crimson/osd/osd_operations/client_request.h
@@ -6,7 +6,8 @@
 #include "crimson/net/Connection.h"
 #include "crimson/osd/osd_operation.h"
 #include "crimson/common/type_helpers.h"
-#include "messages/MOSDOp.h"
+
+class MOSDOp;
 
 namespace crimson::osd {
 class PG;
diff --git a/src/crimson/osd/pg.h b/src/crimson/osd/pg.h
index ecc2dfd5974..4c5cd11b542 100644
--- a/src/crimson/osd/pg.h
+++ b/src/crimson/osd/pg.h
@@ -13,8 +13,6 @@
 
 #include "common/dout.h"
 #include "crimson/net/Fwd.h"
-#include "messages/MOSDRepOpReply.h"
-#include "messages/MOSDOpReply.h"
 #include "os/Transaction.h"
 #include "osd/osd_types.h"
 #include "osd/osd_internal_types.h"
diff --git a/src/crimson/osd/pg_backend.h b/src/crimson/osd/pg_backend.h
index 60f14b443d9..6530ef11602 100644
--- a/src/crimson/osd/pg_backend.h
+++ b/src/crimson/osd/pg_backend.h
@@ -12,13 +12,11 @@
 #include "crimson/os/futurized_collection.h"
 #include "crimson/osd/acked_peers.h"
 #include "crimson/common/shared_lru.h"
-#include "messages/MOSDOp.h"
-#include "messages/MOSDOpReply.h"
-#include "os/Transaction.h"
 #include "osd/osd_types.h"
 #include "osd/osd_internal_types.h"
 
 struct hobject_t;
+class MOSDRepOpReply;
 
 namespace ceph::os {
   class Transaction;
diff --git a/src/global/global_init.cc b/src/global/global_init.cc
index 314cf3def03..dff97e5c7ee 100644
--- a/src/global/global_init.cc
+++ b/src/global/global_init.cc
@@ -12,7 +12,6 @@
  *
  */
 
-#include "common/async/context_pool.h"
 #include "common/ceph_argparse.h"
 #include "common/code_environment.h"
 #include "common/config.h"
@@ -333,16 +332,13 @@ global_init(const std::map<std::string,std::string> *defaults,
     // make sure our mini-session gets legacy values
     g_conf().apply_changes(nullptr);
 
-    ceph::async::io_context_pool cp(1);
-    MonClient mc_bootstrap(g_ceph_context, cp);
+    MonClient mc_bootstrap(g_ceph_context);
     if (mc_bootstrap.get_monmap_and_config() < 0) {
-      cp.stop();
       g_ceph_context->_log->flush();
       cerr << "failed to fetch mon config (--no-mon-config to skip)"
 	   << std::endl;
       _exit(1);
     }
-    cp.stop();
   }
 
   // Expand metavariables. Invoke configuration observers. Open log file.
@@ -402,6 +398,16 @@ global_init(const std::map<std::string,std::string> *defaults,
   return boost::intrusive_ptr<CephContext>{g_ceph_context, false};
 }
 
+void intrusive_ptr_add_ref(CephContext* cct)
+{
+  cct->get();
+}
+
+void intrusive_ptr_release(CephContext* cct)
+{
+  cct->put();
+}
+
 void global_print_banner(void)
 {
   output_ceph_version();
diff --git a/src/global/global_init.h b/src/global/global_init.h
index f4c9a4a8c82..fe6f557bb25 100644
--- a/src/global/global_init.h
+++ b/src/global/global_init.h
@@ -20,10 +20,11 @@
 #include <map>
 #include <boost/intrusive_ptr.hpp>
 #include "include/ceph_assert.h"
-#include "common/ceph_context.h"
 #include "common/code_environment.h"
 #include "common/common_init.h"
 
+class CephContext;
+
 /*
  * global_init is the first initialization function that
  * daemons and utility programs need to call. It takes care of a lot of
@@ -39,6 +40,9 @@ global_init(
   const char *data_dir_option = 0,
   bool run_pre_init = true);
 
+void intrusive_ptr_add_ref(CephContext* cct);
+void intrusive_ptr_release(CephContext* cct);
+
 // just the first half; enough to get config parsed but doesn't start up the
 // cct or log.
 void global_pre_init(const std::map<std::string,std::string> *defaults,
diff --git a/src/include/Context.h b/src/include/Context.h
index bef85ca5b52..6d39be55ba1 100644
--- a/src/include/Context.h
+++ b/src/include/Context.h
@@ -18,15 +18,10 @@
 
 #include "common/dout.h"
 
-#include <functional>
+#include <boost/function.hpp>
 #include <list>
-#include <memory>
 #include <set>
-
-#include <boost/function.hpp>
-#include <boost/system/error_code.hpp>
-
-#include "common/error_code.h"
+#include <memory>
 
 #include "include/ceph_assert.h"
 #include "common/ceph_mutex.h"
@@ -53,23 +48,6 @@ class GenContext {
     finish(std::forward<C>(t));
     delete this;
   }
-
-  template <typename C>
-  void operator()(C &&t) noexcept {
-    complete(std::forward<C>(t));
-  }
-
-  template<typename U = T>
-  auto operator()() noexcept
-    -> typename std::enable_if<std::is_default_constructible<U>::value,
-			       void>::type {
-    complete(T{});
-  }
-
-
-  std::reference_wrapper<GenContext> func() {
-    return std::ref(*this);
-  }
 };
 
 template <typename T>
@@ -106,20 +84,6 @@ class Context {
     }
     return false;
   }
-  void complete(boost::system::error_code ec) {
-    complete(ceph::from_error_code(ec));
-  }
-  void operator()(boost::system::error_code ec) noexcept {
-    complete(ec);
-  }
-
-  void operator()() noexcept {
-    complete({});
-  }
-
-  std::reference_wrapper<Context> func() {
-    return std::ref(*this);
-  }
 };
 
 /**
@@ -162,10 +126,7 @@ class LambdaContext : public Context {
 public:
   LambdaContext(T &&t) : t(std::forward<T>(t)) {}
   void finish(int r) override {
-    if constexpr (std::is_invocable_v<T, int>)
-      t(r);
-    else
-      t();
+    t(r);
   }
 private:
   T t;
@@ -522,14 +483,6 @@ public:
   virtual ContextType *build() = 0;
 };
 
-inline auto lambdafy(Context *c) {
-  return [fin = std::unique_ptr<Context>(c)]
-    (boost::system::error_code ec) mutable {
-	   fin.release()->complete(ceph::from_error_code(ec));
-	 };
-}
-
-
 #undef mydout
 
 #endif
diff --git a/src/include/RADOS/RADOS.hpp b/src/include/RADOS/RADOS.hpp
deleted file mode 100644
index a2242c98ad7..00000000000
--- a/src/include/RADOS/RADOS.hpp
+++ /dev/null
@@ -1,1040 +0,0 @@
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2018 Red Hat <contact@redhat.com>
- * Author: Adam C. Emerson
- *
- * This is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License version 2.1, as published by the Free Software
- * Foundation.  See file COPYING.
- *
- */
-
-#ifndef RADOS_UNLEASHED_HPP
-#define RADOS_UNLEASHED_HPP
-
-#include <cstddef>
-#include <memory>
-#include <tuple>
-#include <string>
-#include <string_view>
-#include <type_traits>
-#include <variant>
-
-#include <boost/asio.hpp>
-
-#include <boost/container/flat_map.hpp>
-#include <boost/container/flat_set.hpp>
-#include <boost/uuid/uuid.hpp>
-
-
-// HATE. LET ME TELL YOU HOW MUCH I'VE COME TO HATE BOOST.SYSTEM SINCE
-// I BEGAN TO LIVE.
-
-#include <boost/system/error_code.hpp>
-
-#include "include/function2.hpp"
-#include "include/expected.hpp"
-
-// Needed for type erasure and template support. We can't really avoid
-// it.
-
-#include "common/async/completion.h"
-
-// These are needed for RGW, but in general as a 'shiny new interface'
-// we should try to use forward declarations and provide standard alternatives.
-
-#include "include/ceph_fs.h"
-#include "include/rados/rados_types.hpp"
-#include "include/rados.h"
-#include "include/buffer.h"
-#include "include/object.h"
-
-#include "common/ceph_time.h"
-
-// Figure out exactly what pool stat info we want to expose to clients
-// and get rid of this.
-
-#include "osd/osd_types.h"
-
-#include "librados/ListObjectImpl.h"
-
-class CephContext;
-
-namespace RADOS {
-class Object;
-}
-namespace std {
-template<>
-struct hash<RADOS::Object>;
-}
-
-namespace RADOS {
-namespace detail {
-class RADOS;
-}
-
-class RADOS;
-
-// Exists mostly so that repeated operations on the same object don't
-// have to pay for the string copy to construct an object_t.
-
-class Object final {
-  friend RADOS;
-  friend std::hash<Object>;
-
-public:
-  Object(std::string_view s);
-  Object(std::string&& s);
-  Object(const std::string& s);
-  ~Object();
-
-  Object(const Object& o);
-  Object& operator =(const Object& o);
-
-  Object(Object&& o);
-  Object& operator =(Object&& o);
-
-  operator std::string_view() const;
-
-  friend std::ostream& operator <<(std::ostream& m, const Object& o);
-  friend bool operator <(const Object& lhs, const Object& rhs);
-  friend bool operator <=(const Object& lhs, const Object& rhs);
-  friend bool operator >=(const Object& lhs, const Object& rhs);
-  friend bool operator >(const Object& lhs, const Object& rhs);
-
-  friend bool operator ==(const Object& lhs, const Object& rhs);
-  friend bool operator !=(const Object& lhs, const Object& rhs);
-
-private:
-
-  static constexpr std::size_t impl_size = 4 * 8;
-  std::aligned_storage_t<impl_size> impl;
-};
-
-// Not the same as the librados::IoCtx, but it does gather together
-// some of the same metadata. Since we're likely to do multiple
-// operations in the same pool or namespace, it doesn't make sense to
-// redo a bunch of lookups and string copies.
-
-struct IOContext final {
-  friend RADOS;
-
-  IOContext();
-  explicit IOContext(std::int64_t pool);
-  IOContext(std::int64_t _pool, std::string_view _ns);
-  IOContext(std::int64_t _pool, std::string&& _ns);
-  ~IOContext();
-
-  IOContext(const IOContext& rhs);
-  IOContext& operator =(const IOContext& rhs);
-
-  IOContext(IOContext&& rhs);
-  IOContext& operator =(IOContext&& rhs);
-
-  std::int64_t pool() const;
-  void pool(std::int64_t _pool);
-
-  std::string_view ns() const;
-  void ns(std::string_view _ns);
-  void ns(std::string&& _ns);
-
-  // Because /some fool/ decided to disallow optional references,
-  // you'd have to construct a string in an optional which I would then
-  // take an optional reference to. Thus a separate 'clear' method.
-  std::optional<std::string_view> key() const;
-  void key(std::string_view _key);
-  void key(std::string&& _key);
-  void clear_key();
-
-  std::optional<std::int64_t> hash() const;
-  void hash(std::int64_t _hash);
-  void clear_hash();
-
-  std::optional<std::uint64_t> read_snap() const;
-  void read_snap(std::optional<std::uint64_t> _snapid);
-
-  // I can't actually move-construct here since snapid_t is its own
-  // separate class type, not an alias.
-  std::optional<
-    std::pair<std::uint64_t,
-	      std::vector<std::uint64_t>>> write_snap_context() const;
-  void write_snap_context(std::optional<
-			  std::pair<std::uint64_t,
-			              std::vector<std::uint64_t>>> snapc);
-private:
-
-  static constexpr std::size_t impl_size = 16 * 8;
-  std::aligned_storage_t<impl_size> impl;
-};
-
-inline constexpr std::string_view all_nspaces("\001"sv);
-
-enum class cmpxattr_op : std::uint8_t {
-  eq  = 1,
-  ne  = 2,
-  gt  = 3,
-  gte = 4,
-  lt  = 5,
-  lte = 6
-};
-
-namespace alloc_hint {
-enum alloc_hint_t {
-  sequential_write = CEPH_OSD_ALLOC_HINT_FLAG_SEQUENTIAL_WRITE,
-  random_write = CEPH_OSD_ALLOC_HINT_FLAG_RANDOM_WRITE,
-  sequential_read = CEPH_OSD_ALLOC_HINT_FLAG_SEQUENTIAL_READ,
-  random_read = CEPH_OSD_ALLOC_HINT_FLAG_RANDOM_READ,
-  append_only = CEPH_OSD_ALLOC_HINT_FLAG_APPEND_ONLY,
-  immutable = CEPH_OSD_ALLOC_HINT_FLAG_IMMUTABLE,
-  shortlived = CEPH_OSD_ALLOC_HINT_FLAG_SHORTLIVED,
-  longlived = CEPH_OSD_ALLOC_HINT_FLAG_LONGLIVED,
-  compressible = CEPH_OSD_ALLOC_HINT_FLAG_COMPRESSIBLE,
-  incompressible = CEPH_OSD_ALLOC_HINT_FLAG_INCOMPRESSIBLE
-};
-}
-
-class Op {
-  friend RADOS;
-
-public:
-
-  Op(const Op&) = delete;
-  Op& operator =(const Op&) = delete;
-  Op(Op&&);
-  Op& operator =(Op&&);
-  ~Op();
-
-  void set_excl();
-  void set_failok();
-  void set_fadvise_random();
-  void set_fadvise_sequential();
-  void set_fadvise_willneed();
-  void set_fadvise_dontneed();
-  void set_fadvise_nocache();
-
-  void cmpext(uint64_t off, ceph::buffer::list&& cmp_bl, std::size_t* s);
-  void cmpxattr(std::string_view name, cmpxattr_op op, const ceph::buffer::list& val);
-  void cmpxattr(std::string_view name, cmpxattr_op op, std::uint64_t val);
-  void assert_version(uint64_t ver);
-  void assert_exists();
-  void cmp_omap(const boost::container::flat_map<
-		  std::string,
-		  std::pair<ceph::buffer::list, int>>& assertions);
-
-  std::size_t size() const;
-  using Signature = void(boost::system::error_code);
-  using Completion = ceph::async::Completion<Signature>;
-
-  friend std::ostream& operator <<(std::ostream& m, const Op& o);
-protected:
-  Op();
-  static constexpr std::size_t impl_size = 170 * 8;
-  std::aligned_storage_t<impl_size> impl;
-};
-
-// This class is /not/ thread-safe. If you want you can wrap it in
-// something that locks it.
-
-class ReadOp final : public Op {
-  friend RADOS;
-
-public:
-
-  ReadOp() = default;
-  ReadOp(const ReadOp&) = delete;
-  ReadOp(ReadOp&&) = default;
-
-  ReadOp& operator =(const ReadOp&) = delete;
-  ReadOp& operator =(ReadOp&&) = default;
-
-  void read(size_t off, uint64_t len, ceph::buffer::list* out,
-	    boost::system::error_code* ec = nullptr);
-  void get_xattr(std::string_view name, ceph::buffer::list* out,
-		 boost::system::error_code* ec = nullptr);
-  void get_omap_header(ceph::buffer::list*,
-		       boost::system::error_code* ec = nullptr);
-
-  void sparse_read(uint64_t off, uint64_t len,
-		   ceph::buffer::list* out,
-		   std::vector<std::pair<std::uint64_t, std::uint64_t>>* extents,
-		   boost::system::error_code* ec = nullptr);
-
-  void stat(std::uint64_t* size, ceph::real_time* mtime,
-	    boost::system::error_code* ec = nullptr);
-
-  void get_omap_keys(std::optional<std::string_view> start_after,
-		     std::uint64_t max_return,
-		     boost::container::flat_set<std::string>* keys,
-		     bool* truncated,
-		     boost::system::error_code* ec = nullptr);
-
-
-  void get_xattrs(boost::container::flat_map<std::string,
-		                             ceph::buffer::list>* kv,
-		     boost::system::error_code* ec = nullptr);
-
-  void get_omap_vals(std::optional<std::string_view> start_after,
-		     std::optional<std::string_view> filter_prefix,
-		     uint64_t max_return,
-		     boost::container::flat_map<std::string,
-		                                ceph::buffer::list>* kv,
-		     bool* truncated,
-		     boost::system::error_code* ec = nullptr);
-
-
-  void get_omap_vals_by_keys(const boost::container::flat_set<std::string>& keys,
-			     boost::container::flat_map<std::string,
-			                                ceph::buffer::list>* kv,
-			     boost::system::error_code* ec = nullptr);
-
-  void list_watchers(std::vector<obj_watch_t>* watchers,
-		     boost::system::error_code* ec = nullptr);
-
-  void list_snaps(librados::snap_set_t* snaps,
-		  boost::system::error_code* ec = nullptr);
-
-  void exec(std::string_view cls, std::string_view method,
-	    const ceph::buffer::list& inbl,
-	    ceph::buffer::list* out,
-	    boost::system::error_code* ec = nullptr);
-  void exec(std::string_view cls, std::string_view method,
-	    const ceph::buffer::list& inbl,
-	    fu2::unique_function<void (boost::system::error_code,
-				       const ceph::buffer::list&) &&> f);
-};
-
-class WriteOp final : public Op {
-  friend RADOS;
-public:
-
-  WriteOp() = default;
-  WriteOp(const WriteOp&) = delete;
-  WriteOp(WriteOp&&) = default;
-
-  WriteOp& operator =(const WriteOp&) = delete;
-  WriteOp& operator =(WriteOp&&) = default;
-
-  void set_mtime(ceph::real_time t);
-  void create(bool exclusive);
-  void write(uint64_t off, ceph::buffer::list&& bl);
-  void write_full(ceph::buffer::list&& bl);
-  void writesame(std::uint64_t off, std::uint64_t write_len,
-		 ceph::buffer::list&& bl);
-  void append(ceph::buffer::list&& bl);
-  void remove();
-  void truncate(uint64_t off);
-  void zero(uint64_t off, uint64_t len);
-  void rmxattr(std::string_view name);
-  void setxattr(std::string_view name,
-		ceph::buffer::list&& bl);
-  void rollback(uint64_t snapid);
-  void set_omap(const boost::container::flat_map<std::string,
-		                                 ceph::buffer::list>& map);
-  void set_omap_header(ceph::buffer::list&& bl);
-  void clear_omap();
-  void rm_omap_keys(const boost::container::flat_set<std::string>& to_rm);
-  void set_alloc_hint(uint64_t expected_object_size,
-		      uint64_t expected_write_size,
-		      alloc_hint::alloc_hint_t flags);
-  void exec(std::string_view cls, std::string_view method,
-	    const ceph::buffer::list& inbl, boost::system::error_code* ec = nullptr);
-};
-
-// Come back and refactor this layer properly at some point.
-using Entry = librados::ListObjectImpl;
-class Cursor final {
-public:
-  static Cursor begin();
-  static Cursor end();
-
-  Cursor();
-  Cursor(const Cursor&);
-  Cursor& operator =(const Cursor&);
-  Cursor(Cursor&&);
-  Cursor& operator =(Cursor&&);
-  ~Cursor();
-
-  friend bool operator ==(const Cursor& lhs,
-			  const Cursor& rhs);
-  friend bool operator !=(const Cursor& lhs,
-			  const Cursor& rhs);
-  friend bool operator <(const Cursor& lhs,
-			 const Cursor& rhs);
-  friend bool operator <=(const Cursor& lhs,
-			  const Cursor& rhs);
-  friend bool operator >=(const Cursor& lhs,
-			  const Cursor& rhs);
-  friend bool operator >(const Cursor& lhs,
-			 const Cursor& rhs);
-
-  std::string to_str() const;
-  static std::optional<Cursor> from_str(const std::string& s);
-
-private:
-  struct end_magic_t {};
-  Cursor(end_magic_t);
-  Cursor(void*);
-  friend RADOS;
-  static constexpr std::size_t impl_size = 16 * 8;
-  std::aligned_storage_t<impl_size> impl;
-};
-
-class RADOS final
-{
-public:
-  static constexpr std::tuple<uint32_t, uint32_t, uint32_t> version() {
-    return {0, 0, 1};
-  }
-
-  using BuildSig = void(boost::system::error_code, RADOS);
-  using BuildComp = ceph::async::Completion<BuildSig>;
-  class Builder {
-    std::optional<std::string> conf_files;
-    std::optional<std::string> cluster;
-    std::optional<std::string> name;
-    std::vector<std::pair<std::string, std::string>> configs;
-    bool no_default_conf = false;
-    bool no_mon_conf = false;
-
-  public:
-    Builder() = default;
-    Builder& add_conf_file(std::string_view v);
-    Builder& set_cluster(std::string_view c) {
-      cluster = std::string(c);
-      return *this;
-    }
-    Builder& set_name(std::string_view n) {
-      name = std::string(n);
-      return *this;
-    }
-    Builder& set_no_default_conf() {
-      no_default_conf = true;
-      return *this;
-    }
-    Builder& set_no_mon_conf() {
-      no_mon_conf = true;
-      return *this;
-    }
-    Builder& set_conf_option(std::string_view opt, std::string_view val) {
-      configs.emplace_back(std::string(opt), std::string(val));
-      return *this;
-    }
-
-    template<typename CompletionToken>
-    auto build(boost::asio::io_context& ioctx, CompletionToken&& token) {
-      boost::asio::async_completion<CompletionToken, BuildSig> init(token);
-      build(ioctx,
-	    BuildComp::create(ioctx.get_executor(),
-			      std::move(init.completion_handler)));
-      return init.result.get();
-    }
-
-  private:
-    void build(boost::asio::io_context& ioctx,
-	       std::unique_ptr<BuildComp> c);
-  };
-
-
-  template<typename CompletionToken>
-  static auto make_with_cct(CephContext* cct,
-			    boost::asio::io_context& ioctx,
-			    CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, BuildSig> init(token);
-    make_with_cct(cct, ioctx,
-		  BuildComp::create(ioctx.get_executor(),
-				    std::move(init.completion_handler)));
-    return init.result.get();
-  }
-
-  // Because the coroutine library is stupidly written and doesn't
-  // support types that aren't default constructible.
-  //
-  // Under protest. I don't want a default constructor and I don't
-  // want people using it.
-  RADOS();
-
-  RADOS(const RADOS&) = delete;
-  RADOS& operator =(const RADOS&) = delete;
-
-  RADOS(RADOS&&);
-  RADOS& operator =(RADOS&&);
-
-  ~RADOS();
-
-  CephContext* cct();
-
-  using executor_type = boost::asio::io_context::executor_type;
-  executor_type get_executor();
-
-  template<typename CompletionToken>
-  auto execute(const Object& o, const IOContext& ioc, ReadOp&& op,
-	       ceph::buffer::list* bl,
-	       CompletionToken&& token, version_t* objver = nullptr) {
-    boost::asio::async_completion<CompletionToken, Op::Signature> init(token);
-    execute(o, ioc, std::move(op), bl,
-	    ReadOp::Completion::create(get_executor(),
-				       std::move(init.completion_handler)),
-	    objver);
-    return init.result.get();
-  }
-
-  template<typename CompletionToken>
-  auto execute(const Object& o, const IOContext& ioc, WriteOp&& op,
-	       CompletionToken&& token, version_t* objver = nullptr) {
-    boost::asio::async_completion<CompletionToken, Op::Signature> init(token);
-    execute(o, ioc, std::move(op),
-	    Op::Completion::create(get_executor(),
-				   std::move(init.completion_handler)),
-	    objver);
-    return init.result.get();
-  }
-
-  template<typename CompletionToken>
-  auto execute(const Object& o, std::int64_t pool,
-	       ReadOp&& op,
-	       ceph::buffer::list* bl,
-	       CompletionToken&& token,
-	       std::optional<std::string_view> ns = {},
-	       std::optional<std::string_view> key = {},
-	       version_t* objver = nullptr) {
-    boost::asio::async_completion<CompletionToken, Op::Signature> init(token);
-    execute(o, pool, std::move(op), bl,
-	    ReadOp::Completion::create(get_executor(),
-				       std::move(init.completion_handler)),
-	    ns, key, objver);
-    return init.result.get();
-  }
-
-  template<typename CompletionToken>
-  auto execute(const Object& o, std::int64_t pool, WriteOp&& op,
-	       CompletionToken&& token,
-	       std::optional<std::string_view> ns = {},
-	       std::optional<std::string_view> key = {},
-	       version_t* objver = nullptr) {
-    boost::asio::async_completion<CompletionToken, Op::Signature> init(token);
-    execute(o, pool, std::move(op),
-	    Op::Completion::create(get_executor(),
-				   std::move(init.completion_handler)),
-	    ns, key, objver);
-    return init.result.get();
-  }
-
-  boost::uuids::uuid get_fsid() const noexcept;
-
-  using LookupPoolSig = void(boost::system::error_code,
-			     std::int64_t);
-  using LookupPoolComp = ceph::async::Completion<LookupPoolSig>;
-  template<typename CompletionToken>
-  auto lookup_pool(std::string_view name,
-		   CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, LookupPoolSig> init(token);
-    lookup_pool(name,
-		LookupPoolComp::create(get_executor(),
-				       std::move(init.completion_handler)));
-    return init.result.get();
-  }
-
-  std::optional<uint64_t> get_pool_alignment(int64_t pool_id);
-
-  using LSPoolsSig = void(std::vector<std::pair<std::int64_t, std::string>>);
-  using LSPoolsComp = ceph::async::Completion<LSPoolsSig>;
-  template<typename CompletionToken>
-  auto list_pools(CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, LSPoolsSig> init(token);
-    list_pools(LSPoolsComp::create(get_executor(),
-				   std::move(init.completion_handler)));
-    return init.result.get();
-  }
-
-
-
-  using SimpleOpSig = void(boost::system::error_code);
-  using SimpleOpComp = ceph::async::Completion<SimpleOpSig>;
-  template<typename CompletionToken>
-  auto create_pool_snap(int64_t pool, std::string_view snapName,
-			CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, SimpleOpSig> init(token);
-    create_pool_snap(pool, snapName,
-		     SimpleOpComp::create(get_executor(),
-					  std::move(init.completion_handler)));
-    return init.result.get();
-  }
-
-  using SMSnapSig = void(boost::system::error_code, snapid_t);
-  using SMSnapComp = ceph::async::Completion<SMSnapSig>;
-  template<typename CompletionToken>
-  auto allocate_selfmanaged_snap(int64_t pool,
-				 CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, SMSnapSig> init(token);
-    allocate_selfmanaged_snap(pool,
-			      SMSnapComp::create(
-				get_executor(),
-				std::move(init.completion_handler)));
-    return init.result.get();
-  }
-
-  template<typename CompletionToken>
-  auto delete_pool_snap(int64_t pool, std::string_view snapName,
-			CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, SimpleOpSig> init(token);
-    delete_pool_snap(pool, snapName,
-		     SimpleOpComp::create(get_executor(),
-					  std::move(init.completion_handler)));
-    return init.result.get();
-  }
-
-  template<typename CompletionToken>
-  auto delete_selfmanaged_snap(int64_t pool, std::string_view snapName,
-			       CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, SimpleOpSig> init(token);
-    delete_selfmanaged_snap(pool, snapName,
-			    SimpleOpComp::create(
-			      get_executor(),
-			      std::move(init.completion_handler)));
-    return init.result.get();
-  }
-
-  template<typename CompletionToken>
-  auto create_pool(std::string_view name, std::optional<int> crush_rule,
-		   CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, SimpleOpSig> init(token);
-    create_pool(name, crush_rule,
-		SimpleOpComp::create(get_executor(),
-				     std::move(init.completion_handler)));
-    return init.result.get();
-  }
-
-  template<typename CompletionToken>
-  auto delete_pool(std::string_view name,
-		   CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, SimpleOpSig> init(token);
-    delete_pool(name,
-		SimpleOpComp::create(get_executor(),
-				     std::move(init.completion_handler)));
-    return init.result.get();
-  }
-
-  template<typename CompletionToken>
-  auto delete_pool(int64_t pool,
-		   CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, SimpleOpSig> init(token);
-    delete_pool(pool,
-		SimpleOpComp::create(get_executor(),
-				     std::move(init.completion_handler)));
-    return init.result.get();
-  }
-
-  using PoolStatSig = void(boost::system::error_code,
-			   boost::container::flat_map<std::string,
-			                              pool_stat_t>, bool);
-  using PoolStatComp = ceph::async::Completion<PoolStatSig>;
-  template<typename CompletionToken>
-  auto stat_pools(const std::vector<std::string>& pools,
-		  CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, PoolStatSig> init(token);
-    stat_pools(pools,
-	       PoolStatComp::create(get_executor(),
-				    std::move(init.completion_handler)));
-    return init.result.get();
-  }
-
-  using StatFSSig = void(boost::system::error_code,
-			 ceph_statfs);
-  using StatFSComp = ceph::async::Completion<StatFSSig>;
-  template<typename CompletionToken>
-  auto statfs(std::optional<int64_t> pool,
-	      CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, StatFSSig> init(token);
-    ceph_statfs(pool, StatFSComp::create(get_executor(),
-					 std::move(init.completion_handler)));
-    return init.result.get();
-  }
-
-  using WatchCB = fu2::unique_function<void(boost::system::error_code,
-					    uint64_t notify_id,
-					    uint64_t cookie,
-					    uint64_t notifier_id,
-					    ceph::buffer::list&& bl)>;
-
-  using WatchSig = void(boost::system::error_code ec,
-			uint64_t cookie);
-  using WatchComp = ceph::async::Completion<WatchSig>;
-  template<typename CompletionToken>
-  auto watch(const Object& o, const IOContext& ioc,
-	     std::optional<std::chrono::seconds> timeout,
-	     WatchCB&& cb, CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, WatchSig> init(token);
-    watch(o, ioc, timeout, std::move(cb),
-	  WatchComp::create(get_executor(),
-			    std::move(init.completion_handler)));
-    return init.result.get();
-  }
-
-  template<typename CompletionToken>
-  auto watch(const Object& o, std::int64_t pool,
-	     std::optional<std::chrono::seconds> timeout,
-	     WatchCB&& cb, CompletionToken&& token,
-	     std::optional<std::string_view> ns = {},
-	     std::optional<std::string_view> key = {}) {
-    boost::asio::async_completion<CompletionToken, WatchSig> init(token);
-    watch(o, pool, timeout, std::move(cb),
-	  WatchComp::create(get_executor(),
-			    std::move(init.completion_handler)),
-	  ns, key);
-    return init.result.get();
-  }
-
-  template<typename CompletionToken>
-  auto notify_ack(const Object& o,
-		  const IOContext& ioc,
-		  uint64_t notify_id,
-		  uint64_t cookie,
-		  ceph::buffer::list&& bl,
-		  CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, SimpleOpSig> init(token);
-    notify_ack(o, ioc, notify_id, cookie, std::move(bl),
-	       SimpleOpComp::create(get_executor(),
-				    std::move(init.completion_handler)));
-    return init.result.get();
-  }
-
-  template<typename CompletionToken>
-  auto notify_ack(const Object& o,
-		  std::int64_t pool,
-		  uint64_t notify_id,
-		  uint64_t cookie,
-		  ceph::buffer::list&& bl,
-		  CompletionToken&& token,
-		  std::optional<std::string_view> ns = {},
-		  std::optional<std::string_view> key = {}) {
-    boost::asio::async_completion<CompletionToken, WatchSig> init(token);
-    notify_ack(o, pool, notify_id, cookie, std::move(bl),
-	       SimpleOpComp::create(get_executor(),
-				    std::move(init.completion_handler)),
-	       ns, key);
-    return init.result.get();
-  }
-
-  template<typename CompletionToken>
-  auto unwatch(uint64_t cookie, const IOContext& ioc,
-	       CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, SimpleOpSig> init(token);
-    unwatch(cookie, ioc,
-	    SimpleOpComp::create(get_executor(),
-				 std::move(init.completion_handler)));
-    return init.result.get();
-  }
-
-  template<typename CompletionToken>
-  auto unwatch(uint64_t cookie, std::int64_t pool,
-	       CompletionToken&& token,
-	       std::optional<std::string_view> ns = {},
-	       std::optional<std::string_view> key = {}) {
-    boost::asio::async_completion<CompletionToken, SimpleOpSig> init(token);
-    unwatch(cookie, pool,
-	    SimpleOpComp::create(get_executor(),
-				 std::move(init.completion_handler)),
-	    ns, key);
-    return init.result.get();
-  }
-
-  // This is one of those places where having to force everything into
-  // a .cc file is really infuriating. If we had modules, that would
-  // let us separate out the implementation details without
-  // sacrificing all the benefits of templates.
-  using VoidOpSig = void();
-  using VoidOpComp = ceph::async::Completion<VoidOpSig>;
-  template<typename CompletionToken>
-  auto flush_watch(CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, VoidOpSig> init(token);
-    flush_watch(VoidOpComp::create(get_executor(),
-				   std::move(init.completion_handler)));
-    return init.result.get();
-  }
-
-  using NotifySig = void(boost::system::error_code, ceph::buffer::list);
-  using NotifyComp = ceph::async::Completion<NotifySig>;
-  template<typename CompletionToken>
-  auto notify(const Object& oid, const IOContext& ioc, ceph::buffer::list&& bl,
-	      std::optional<std::chrono::milliseconds> timeout,
-	      CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, NotifySig> init(token);
-    notify(oid, ioc, std::move(bl), timeout,
-	   NotifyComp::create(get_executor(),
-			      std::move(init.completion_handler)));
-
-    return init.result.get();
-  }
-
-  template<typename CompletionToken>
-  auto notify(const Object& oid, std::int64_t pool, ceph::buffer::list&& bl,
-	      std::optional<std::chrono::milliseconds> timeout,
-	      CompletionToken&& token,
-	      std::optional<std::string_view> ns = {},
-	      std::optional<std::string_view> key = {}) {
-    boost::asio::async_completion<CompletionToken, NotifySig> init(token);
-    notify(oid, pool, bl, timeout,
-	   NotifyComp::create(get_executor(),
-			      std::move(init.completion_handler)),
-	   ns, key);
-
-    return init.result.get();
-  }
-
-  template<typename CompletionToken>
-  auto enumerate_objects(const IOContext& ioc, const Cursor& begin,
-			 const Cursor& end, const std::uint32_t max,
-			 const ceph::buffer::list& filter,
-			 std::vector<Entry>* ls,
-			 Cursor* cursor,
-			 CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, SimpleOpSig> init(token);
-    enumerate_objects(ioc, begin, end, max, filter,
-		      ls, cursor,
-		      SimpleOpComp::create(get_executor(),
-					   std::move(init.completion_handler)));
-    return init.result.get();
-  }
-
-  template<typename CompletionToken>
-  auto enumerate_objects(std::int64_t pool, const Cursor& begin,
-			 const Cursor& end, const std::uint32_t max,
-			 const ceph::buffer::list& filter,
-			 std::vector<Entry>* ls,
-			 Cursor* cursor,
-			 CompletionToken&& token,
-			 std::optional<std::string_view> ns = {},
-			 std::optional<std::string_view> key = {}) {
-    boost::asio::async_completion<CompletionToken, SimpleOpSig> init(token);
-    enumerate_objects(pool, begin, end, max, filter,
-		      ls, cursor,
-		      SimpleOpComp::create(get_executor(),
-					   std::move(init.completion_handler)),
-		      ns, key);
-    return init.result.get();
-  }
-
-  // The versions with pointers are fine for coroutines, but
-  // extraordinarily unappealing for callback-oriented programming.
-  using EnumerateSig = void(boost::system::error_code,
-			    std::vector<Entry>,
-			    Cursor);
-  using EnumerateComp = ceph::async::Completion<EnumerateSig>;
-  template<typename CompletionToken>
-  auto enumerate_objects(const IOContext& ioc, const Cursor& begin,
-			 const Cursor& end, const std::uint32_t max,
-			 const ceph::buffer::list& filter,
-			 CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, EnumerateSig> init(token);
-    enumerate_objects(ioc, begin, end, max, filter,
-		      EnumerateComp::create(get_executor(),
-					    std::move(init.completion_handler)));
-    return init.result.get();
-  }
-
-  template<typename CompletionToken>
-  auto enumerate_objects(std::int64_t pool, const Cursor& begin,
-			 const Cursor& end, const std::uint32_t max,
-			 const ceph::buffer::list& filter,
-			 CompletionToken&& token,
-			 std::optional<std::string_view> ns = {},
-			 std::optional<std::string_view> key = {}) {
-    boost::asio::async_completion<CompletionToken, EnumerateSig> init(token);
-    enumerate_objects(pool, begin, end, max, filter,
-		      EnumerateComp::create(get_executor(),
-					    std::move(init.completion_handler)),
-		      ns, key);
-    return init.result.get();
-  }
-
-  using CommandSig = void(boost::system::error_code,
-			  std::string, ceph::buffer::list);
-  using CommandComp = ceph::async::Completion<CommandSig>;
-  template<typename CompletionToken>
-  auto osd_command(int osd, std::vector<std::string>&& cmd,
-		   ceph::buffer::list&& in, CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, CommandSig> init(token);
-    osd_command(osd, std::move(cmd), std::move(in),
-		CommandComp::create(get_executor(),
-				      std::move(init.completion_handler)));
-    return init.result.get();
-  }
-  template<typename CompletionToken>
-  auto pg_command(pg_t pg, std::vector<std::string>&& cmd,
-		  ceph::buffer::list&& in, CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, CommandSig> init(token);
-    pg_command(pg, std::move(cmd), std::move(in),
-	       CommandComp::create(get_executor(),
-				      std::move(init.completion_handler)));
-    return init.result.get();
-  }
-
-  template<typename CompletionToken>
-  auto mon_command(std::vector<std::string> command,
-		   const ceph::buffer::list& bl,
-		   std::string* outs, ceph::buffer::list* outbl,
-		   CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, SimpleOpSig> init(token);
-    mon_command(command, bl, outs, outbl,
-		SimpleOpComp::create(get_executor(),
-				     std::move(init.completion_handler)));
-    return init.result.get();
-  }
-
-  template<typename CompletionToken>
-  auto enable_application(std::string_view pool, std::string_view app_name,
-			  bool force, CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, SimpleOpSig> init(token);
-    enable_application(pool, app_name, force,
-		       SimpleOpComp::create(get_executor(),
-					    std::move(init.completion_handler)));
-    return init.result.get();
-  }
-  uint64_t instance_id() const;
-
-private:
-
-  friend Builder;
-
-  RADOS(std::unique_ptr<detail::RADOS> impl);
-  static void make_with_cct(CephContext* cct,
-			    boost::asio::io_context& ioctx,
-		    std::unique_ptr<BuildComp> c);
-
-  void execute(const Object& o, const IOContext& ioc, ReadOp&& op,
-	       ceph::buffer::list* bl, std::unique_ptr<Op::Completion> c,
-	       version_t* objver);
-
-  void execute(const Object& o, const IOContext& ioc, WriteOp&& op,
-	       std::unique_ptr<Op::Completion> c, version_t* objver);
-
-  void execute(const Object& o, std::int64_t pool, ReadOp&& op,
-	       ceph::buffer::list* bl, std::unique_ptr<Op::Completion> c,
-	       std::optional<std::string_view> ns,
-	       std::optional<std::string_view> key,
-	       version_t* objver);
-
-  void execute(const Object& o, std::int64_t pool, WriteOp&& op,
-	       std::unique_ptr<Op::Completion> c,
-	       std::optional<std::string_view> ns,
-	       std::optional<std::string_view> key,
-	       version_t* objver);
-
-  void lookup_pool(std::string_view name, std::unique_ptr<LookupPoolComp> c);
-  void list_pools(std::unique_ptr<LSPoolsComp> c);
-  void create_pool_snap(int64_t pool, std::string_view snapName,
-			std::unique_ptr<SimpleOpComp> c);
-  void allocate_selfmanaged_snap(int64_t pool, std::unique_ptr<SMSnapComp> c);
-  void delete_pool_snap(int64_t pool, std::string_view snapName,
-			std::unique_ptr<SimpleOpComp> c);
-  void delete_selfmanaged_snap(int64_t pool, snapid_t snap,
-			       std::unique_ptr<SimpleOpComp> c);
-  void create_pool(std::string_view name, std::optional<int> crush_rule,
-		   std::unique_ptr<SimpleOpComp> c);
-  void delete_pool(std::string_view name,
-		   std::unique_ptr<SimpleOpComp> c);
-  void delete_pool(int64_t pool,
-		   std::unique_ptr<SimpleOpComp> c);
-  void stat_pools(const std::vector<std::string>& pools,
-		  std::unique_ptr<PoolStatComp> c);
-  void stat_fs(std::optional<std::int64_t> pool,
-	       std::unique_ptr<StatFSComp> c);
-
-  void watch(const Object& o, const IOContext& ioc,
-	     std::optional<std::chrono::seconds> timeout,
-	     WatchCB&& cb, std::unique_ptr<WatchComp> c);
-  void watch(const Object& o, std::int64_t pool,
-	     std::optional<std::chrono::seconds> timeout,
-	     WatchCB&& cb, std::unique_ptr<WatchComp> c,
-	     std::optional<std::string_view> ns,
-	     std::optional<std::string_view> key);
-  tl::expected<ceph::timespan, boost::system::error_code>
-  watch_check(uint64_t cookie);
-  void notify_ack(const Object& o,
-		  const IOContext& _ioc,
-		  uint64_t notify_id,
-		  uint64_t cookie,
-		  ceph::buffer::list&& bl,
-		  std::unique_ptr<SimpleOpComp>);
-  void notify_ack(const Object& o,
-		  std::int64_t pool,
-		  uint64_t notify_id,
-		  uint64_t cookie,
-		  ceph::buffer::list&& bl,
-		  std::unique_ptr<SimpleOpComp>,
-		  std::optional<std::string_view> ns,
-		  std::optional<std::string_view> key);
-  void unwatch(uint64_t cookie, const IOContext& ioc,
-	       std::unique_ptr<SimpleOpComp>);
-  void unwatch(uint64_t cookie, std::int64_t pool,
-	       std::unique_ptr<SimpleOpComp>,
-	       std::optional<std::string_view> ns,
-	       std::optional<std::string_view> key);
-  void notify(const Object& oid, const IOContext& ioctx,
-	      ceph::buffer::list&& bl,
-	      std::optional<std::chrono::milliseconds> timeout,
-	      std::unique_ptr<NotifyComp> c);
-  void notify(const Object& oid, std::int64_t pool,
-	      ceph::buffer::list&& bl,
-	      std::optional<std::chrono::milliseconds> timeout,
-	      std::unique_ptr<NotifyComp> c,
-	      std::optional<std::string_view> ns,
-	      std::optional<std::string_view> key);
-  void flush_watch(std::unique_ptr<VoidOpComp>);
-
-  void enumerate_objects(const IOContext& ioc, const Cursor& begin,
-			 const Cursor& end, const std::uint32_t max,
-			 const ceph::buffer::list& filter,
-			 std::vector<Entry>* ls,
-			 Cursor* cursor,
-			 std::unique_ptr<SimpleOpComp> c);
-  void enumerate_objects(std::int64_t pool, const Cursor& begin,
-			 const Cursor& end, const std::uint32_t max,
-			 const ceph::buffer::list& filter,
-			 std::vector<Entry>* ls,
-			 Cursor* cursor,
-			 std::unique_ptr<SimpleOpComp> c,
-			 std::optional<std::string_view> ns,
-			 std::optional<std::string_view> key);
-  void enumerate_objects(const IOContext& ioc, const Cursor& begin,
-			 const Cursor& end, const std::uint32_t max,
-			 const ceph::buffer::list& filter,
-			 std::unique_ptr<EnumerateComp> c);
-  void enumerate_objects(std::int64_t pool, const Cursor& begin,
-			 const Cursor& end, const std::uint32_t max,
-			 const ceph::buffer::list& filter,
-			 std::unique_ptr<EnumerateComp> c,
-			 std::optional<std::string_view> ns,
-			 std::optional<std::string_view> key);
-  void osd_command(int osd, std::vector<std::string>&& cmd,
-		   ceph::buffer::list&& in, std::unique_ptr<CommandComp> c);
-  void pg_command(pg_t pg, std::vector<std::string>&& cmd,
-		  ceph::buffer::list&& in, std::unique_ptr<CommandComp> c);
-
-  void mon_command(std::vector<std::string> command,
-		   const ceph::buffer::list& bl,
-		   std::string* outs, ceph::buffer::list* outbl,
-		   std::unique_ptr<SimpleOpComp> c);
-
-  void enable_application(std::string_view pool, std::string_view app_name,
-			  bool force, std::unique_ptr<SimpleOpComp> c);
-
-
-  // Since detail::RADOS has immovable things inside it, hold a
-  // unique_ptr to it so we can be moved.
-  std::unique_ptr<detail::RADOS> impl;
-};
-}
-
-namespace std {
-template<>
-struct hash<RADOS::Object> {
-  size_t operator ()(const RADOS::Object& r) const;
-};
-} // namespace std
-
-#endif
diff --git a/src/include/RADOS/buffer_fwd.h b/src/include/RADOS/buffer_fwd.h
deleted file mode 120000
index bd1f6f1b064..00000000000
--- a/src/include/RADOS/buffer_fwd.h
+++ /dev/null
@@ -1 +0,0 @@
-../buffer_fwd.h
\ No newline at end of file
diff --git a/src/include/RADOS/completion.h b/src/include/RADOS/completion.h
deleted file mode 120000
index 100678fc2a5..00000000000
--- a/src/include/RADOS/completion.h
+++ /dev/null
@@ -1 +0,0 @@
-../../common/async/completion.h
\ No newline at end of file
diff --git a/src/include/buffer.h b/src/include/buffer.h
index 3d41eaa6e67..98358e450f6 100644
--- a/src/include/buffer.h
+++ b/src/include/buffer.h
@@ -50,13 +50,10 @@
 #include <exception>
 #include <type_traits>
 
-#include <boost/system/system_error.hpp>
-
 #include "page.h"
 #include "crc32c.h"
 #include "buffer_fwd.h"
 
-
 #ifdef __CEPH__
 # include "include/ceph_assert.h"
 #else
@@ -104,69 +101,30 @@ struct unique_leakable_ptr : public std::unique_ptr<T, ceph::nop_delete<T>> {
 namespace buffer CEPH_BUFFER_API {
 inline namespace v14_2_0 {
 
-  const boost::system::error_category& buffer_category() noexcept;
-  enum class errc { bad_alloc = 1,
-		    end_of_buffer,
-		    malformed_input };
-}
-}
-}
-
-namespace boost {
-namespace system {
-template<>
-struct is_error_code_enum<::ceph::buffer::errc> {
-  static const bool value = true;
-};
-}
-}
-
-namespace ceph {
-
-namespace buffer CEPH_BUFFER_API {
-inline namespace v14_2_0 {
-
   /*
    * exceptions
    */
 
-  //  explicit conversion:
-  inline boost::system::error_code make_error_code(errc e) noexcept {
-    return { static_cast<int>(e), buffer_category() };
-  }
-
-// implicit conversion:
-  inline boost::system::error_condition
-  make_error_condition(errc e) noexcept {
-    return { static_cast<int>(e), buffer_category() };
-  }
-
-  using error = boost::system::system_error;
-
+  struct error : public std::exception{
+    const char *what() const throw () override;
+  };
   struct bad_alloc : public error {
-    bad_alloc() : error(errc::bad_alloc) {}
-    bad_alloc(const char* what_arg) : error(errc::bad_alloc, what_arg) {}
-    bad_alloc(const std::string& what_arg) : error(errc::bad_alloc, what_arg) {}
+    const char *what() const throw () override;
   };
   struct end_of_buffer : public error {
-    end_of_buffer() : error(errc::end_of_buffer) {}
-    end_of_buffer(const char* what_arg) : error(errc::end_of_buffer, what_arg) {}
-    end_of_buffer(const std::string& what_arg)
-      : error(errc::end_of_buffer, what_arg) {}
+    const char *what() const throw () override;
   };
   struct malformed_input : public error {
-    malformed_input() : error(errc::malformed_input) {}
-    malformed_input(const char* what_arg)
-      : error(errc::malformed_input, what_arg) {}
-    malformed_input(const std::string& what_arg)
-      : error(errc::malformed_input, what_arg) {}
+    explicit malformed_input(const std::string& w) {
+      snprintf(buf, sizeof(buf), "buffer::malformed_input: %s", w.c_str());
+    }
+    const char *what() const throw () override;
+  private:
+    char buf[256];
   };
-  struct error_code : public error {
-    error_code(int r) : error(-r, boost::system::system_category()) {}
-    error_code(int r, const char* what_arg)
-      : error(-r, boost::system::system_category(), what_arg) {}
-    error_code(int r, const std::string& what_arg)
-      : error(-r, boost::system::system_category(), what_arg) {}
+  struct error_code : public malformed_input {
+    explicit error_code(int error);
+    int code;
   };
 
 
@@ -1354,5 +1312,4 @@ inline bufferhash& operator<<(bufferhash& l, const bufferlist &r) {
 
 } // namespace ceph
 
-
 #endif
diff --git a/src/include/expected.hpp b/src/include/expected.hpp
deleted file mode 100644
index 740c6ad2495..00000000000
--- a/src/include/expected.hpp
+++ /dev/null
@@ -1,2282 +0,0 @@
-///
-// expected - An implementation of std::expected with extensions
-// Written in 2017 by Simon Brand (@TartanLlama)
-//
-// To the extent possible under law, the author(s) have dedicated all
-// copyright and related and neighboring rights to this software to the
-// public domain worldwide. This software is distributed without any warranty.
-//
-// You should have received a copy of the CC0 Public Domain Dedication
-// along with this software. If not, see
-// <http://creativecommons.org/publicdomain/zero/1.0/>.
-///
-
-#ifndef TL_EXPECTED_HPP
-#define TL_EXPECTED_HPP
-
-#define TL_EXPECTED_VERSION_MAJOR 0
-#define TL_EXPECTED_VERSION_MINOR 2
-
-#include <exception>
-#include <functional>
-#include <type_traits>
-#include <utility>
-
-#if defined(__EXCEPTIONS) || defined(_CPPUNWIND)
-#define TL_EXPECTED_EXCEPTIONS_ENABLED
-#endif
-
-#if (defined(_MSC_VER) && _MSC_VER == 1900)
-/// \exclude
-#define TL_EXPECTED_MSVC2015
-#define TL_EXPECTED_MSVC2015_CONSTEXPR
-#else
-#define TL_EXPECTED_MSVC2015_CONSTEXPR constexpr
-#endif
-
-#if (defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ <= 9 &&              \
-     !defined(__clang__))
-/// \exclude
-#define TL_EXPECTED_GCC49
-#endif
-
-#if (defined(__GNUC__) && __GNUC__ == 5 && __GNUC_MINOR__ <= 4 &&              \
-     !defined(__clang__))
-/// \exclude
-#define TL_EXPECTED_GCC54
-#endif
-
-#if (defined(__GNUC__) && __GNUC__ == 5 && __GNUC_MINOR__ <= 5 &&              \
-     !defined(__clang__))
-/// \exclude
-#define TL_EXPECTED_GCC55
-#endif
-
-#if (defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ <= 9 &&              \
-     !defined(__clang__))
-// GCC < 5 doesn't support overloading on const&& for member functions
-/// \exclude
-#define TL_EXPECTED_NO_CONSTRR
-
-// GCC < 5 doesn't support some standard C++11 type traits
-/// \exclude
-#define TL_EXPECTED_IS_TRIVIALLY_COPY_CONSTRUCTIBLE(T)                         \
-  std::has_trivial_copy_constructor<T>
-/// \exclude
-#define TL_EXPECTED_IS_TRIVIALLY_COPY_ASSIGNABLE(T)                            \
-  std::has_trivial_copy_assign<T>
-
-// This one will be different for GCC 5.7 if it's ever supported
-/// \exclude
-#define TL_EXPECTED_IS_TRIVIALLY_DESTRUCTIBLE(T)                               \
-  std::is_trivially_destructible<T>
-
-// GCC 5 < v < 8 has a bug in is_trivially_copy_constructible which breaks std::vector
-// for non-copyable types
-#elif (defined(__GNUC__) && __GNUC__ < 8 &&                                                \
-     !defined(__clang__))
-#ifndef TL_GCC_LESS_8_TRIVIALLY_COPY_CONSTRUCTIBLE_MUTEX
-#define TL_GCC_LESS_8_TRIVIALLY_COPY_CONSTRUCTIBLE_MUTEX
-namespace tl {
-  namespace detail {
-      template<class T>
-      struct is_trivially_copy_constructible : std::is_trivially_copy_constructible<T>{};
-#ifdef _GLIBCXX_VECTOR
-      template<class T, class A>
-      struct is_trivially_copy_constructible<std::vector<T,A>>
-          : std::is_trivially_copy_constructible<T>{};
-#endif
-  }
-}
-#endif
-
-#define TL_EXPECTED_IS_TRIVIALLY_COPY_CONSTRUCTIBLE(T)                                     \
-  tl::detail::is_trivially_copy_constructible<T>
-#define TL_EXPECTED_IS_TRIVIALLY_COPY_ASSIGNABLE(T)                                        \
-  std::is_trivially_copy_assignable<T>
-#define TL_EXPECTED_IS_TRIVIALLY_DESTRUCTIBLE(T) std::is_trivially_destructible<T>
-#else
-/// \exclude
-#define TL_EXPECTED_IS_TRIVIALLY_COPY_CONSTRUCTIBLE(T)                         \
-  std::is_trivially_copy_constructible<T>
-/// \exclude
-#define TL_EXPECTED_IS_TRIVIALLY_COPY_ASSIGNABLE(T)                            \
-  std::is_trivially_copy_assignable<T>
-/// \exclude
-#define TL_EXPECTED_IS_TRIVIALLY_DESTRUCTIBLE(T)                               \
-  std::is_trivially_destructible<T>
-#endif
-
-#if __cplusplus > 201103L
-/// \exclude
-#define TL_EXPECTED_CXX14
-#endif
-
-#ifdef TL_EXPECTED_GCC49
-#define TL_EXPECTED_GCC49_CONSTEXPR
-#else
-#define TL_EXPECTED_GCC49_CONSTEXPR constexpr
-#endif
-
-#if (__cplusplus == 201103L || defined(TL_EXPECTED_MSVC2015) ||                \
-     defined(TL_EXPECTED_GCC49))
-/// \exclude
-#define TL_EXPECTED_11_CONSTEXPR
-#else
-/// \exclude
-#define TL_EXPECTED_11_CONSTEXPR constexpr
-#endif
-
-namespace tl {
-template <class T, class E> class expected;
-
-#ifndef TL_MONOSTATE_INPLACE_MUTEX
-#define TL_MONOSTATE_INPLACE_MUTEX
-/// \brief Used to represent an expected with no data
-class monostate {};
-
-/// \brief A tag type to tell expected to construct its value in-place
-struct in_place_t {
-  explicit in_place_t() = default;
-};
-/// \brief A tag to tell expected to construct its value in-place
-static constexpr in_place_t in_place{};
-#endif
-
-/// Used as a wrapper to store the unexpected value
-template <class E> class unexpected {
-public:
-  static_assert(!std::is_same<E, void>::value, "E must not be void");
-
-  unexpected() = delete;
-  constexpr explicit unexpected(const E &e) : m_val(e) {}
-
-  constexpr explicit unexpected(E &&e) : m_val(std::move(e)) {}
-
-  /// \returns the contained value
-  /// \group unexpected_value
-  constexpr const E &value() const & { return m_val; }
-  /// \group unexpected_value
-  TL_EXPECTED_11_CONSTEXPR E &value() & { return m_val; }
-  /// \group unexpected_value
-  TL_EXPECTED_11_CONSTEXPR E &&value() && { return std::move(m_val); }
-  /// \exclude
-  constexpr const E &&value() const && { return std::move(m_val); }
-
-private:
-  E m_val;
-};
-
-/// \brief Compares two unexpected objects
-/// \details Simply compares lhs.value() to rhs.value()
-/// \group unexpected_relop
-template <class E>
-constexpr bool operator==(const unexpected<E> &lhs, const unexpected<E> &rhs) {
-  return lhs.value() == rhs.value();
-}
-/// \group unexpected_relop
-template <class E>
-constexpr bool operator!=(const unexpected<E> &lhs, const unexpected<E> &rhs) {
-  return lhs.value() != rhs.value();
-}
-/// \group unexpected_relop
-template <class E>
-constexpr bool operator<(const unexpected<E> &lhs, const unexpected<E> &rhs) {
-  return lhs.value() < rhs.value();
-}
-/// \group unexpected_relop
-template <class E>
-constexpr bool operator<=(const unexpected<E> &lhs, const unexpected<E> &rhs) {
-  return lhs.value() <= rhs.value();
-}
-/// \group unexpected_relop
-template <class E>
-constexpr bool operator>(const unexpected<E> &lhs, const unexpected<E> &rhs) {
-  return lhs.value() > rhs.value();
-}
-/// \group unexpected_relop
-template <class E>
-constexpr bool operator>=(const unexpected<E> &lhs, const unexpected<E> &rhs) {
-  return lhs.value() >= rhs.value();
-}
-
-/// Create an `unexpected` from `e`, deducing the return type
-///
-/// *Example:*
-/// auto e1 = tl::make_unexpected(42);
-/// unexpected<int> e2 (42); //same semantics
-template <class E>
-unexpected<typename std::decay<E>::type> make_unexpected(E &&e) {
-  return unexpected<typename std::decay<E>::type>(std::forward<E>(e));
-}
-
-/// \brief A tag type to tell expected to construct the unexpected value
-struct unexpect_t {
-  unexpect_t() = default;
-};
-/// \brief A tag to tell expected to construct the unexpected value
-static constexpr unexpect_t unexpect{};
-
-/// \exclude
-namespace detail {
-template<typename E>
-[[noreturn]] TL_EXPECTED_11_CONSTEXPR void throw_exception(E &&e) {
-#ifdef TL_EXPECTED_EXCEPTIONS_ENABLED
-    throw std::forward<E>(e);
-#else
-  #ifdef _MSC_VER
-    __assume(0);
-  #else
-    __builtin_unreachable();
-  #endif
-#endif
-}
-
-#ifndef TL_TRAITS_MUTEX
-#define TL_TRAITS_MUTEX
-// C++14-style aliases for brevity
-template <class T> using remove_const_t = typename std::remove_const<T>::type;
-template <class T>
-using remove_reference_t = typename std::remove_reference<T>::type;
-template <class T> using decay_t = typename std::decay<T>::type;
-template <bool E, class T = void>
-using enable_if_t = typename std::enable_if<E, T>::type;
-template <bool B, class T, class F>
-using conditional_t = typename std::conditional<B, T, F>::type;
-
-// std::conjunction from C++17
-template <class...> struct conjunction : std::true_type {};
-template <class B> struct conjunction<B> : B {};
-template <class B, class... Bs>
-struct conjunction<B, Bs...>
-    : std::conditional<bool(B::value), conjunction<Bs...>, B>::type {};
-
-// std::invoke from C++17
-// https://stackoverflow.com/questions/38288042/c11-14-invoke-workaround
-template <typename Fn, typename... Args,
-          typename = enable_if_t<std::is_member_pointer<decay_t<Fn>>{}>,
-          int = 0>
-constexpr auto invoke(Fn &&f, Args &&... args) noexcept(
-    noexcept(std::mem_fn(f)(std::forward<Args>(args)...)))
-    -> decltype(std::mem_fn(f)(std::forward<Args>(args)...)) {
-  return std::mem_fn(f)(std::forward<Args>(args)...);
-}
-
-template <typename Fn, typename... Args,
-          typename = enable_if_t<!std::is_member_pointer<decay_t<Fn>>{}>>
-constexpr auto invoke(Fn &&f, Args &&... args) noexcept(
-    noexcept(std::forward<Fn>(f)(std::forward<Args>(args)...)))
-    -> decltype(std::forward<Fn>(f)(std::forward<Args>(args)...)) {
-  return std::forward<Fn>(f)(std::forward<Args>(args)...);
-}
-
-// std::invoke_result from C++17
-template <class F, class, class... Us> struct invoke_result_impl;
-
-template <class F, class... Us>
-struct invoke_result_impl<
-    F, decltype(detail::invoke(std::declval<F>(), std::declval<Us>()...), void()),
-    Us...> {
-  using type = decltype(detail::invoke(std::declval<F>(), std::declval<Us>()...));
-};
-
-template <class F, class... Us>
-using invoke_result = invoke_result_impl<F, void, Us...>;
-
-template <class F, class... Us>
-using invoke_result_t = typename invoke_result<F, Us...>::type;
-#endif
-
-// Trait for checking if a type is a tl::expected
-template <class T> struct is_expected_impl : std::false_type {};
-template <class T, class E>
-struct is_expected_impl<expected<T, E>> : std::true_type {};
-template <class T> using is_expected = is_expected_impl<decay_t<T>>;
-
-template <class T, class E, class U>
-using expected_enable_forward_value = detail::enable_if_t<
-    std::is_constructible<T, U &&>::value &&
-    !std::is_same<detail::decay_t<U>, in_place_t>::value &&
-    !std::is_same<expected<T, E>, detail::decay_t<U>>::value &&
-    !std::is_same<unexpected<E>, detail::decay_t<U>>::value>;
-
-template <class T, class E, class U, class G, class UR, class GR>
-using expected_enable_from_other = detail::enable_if_t<
-    std::is_constructible<T, UR>::value &&
-    std::is_constructible<E, GR>::value &&
-    !std::is_constructible<T, expected<U, G> &>::value &&
-    !std::is_constructible<T, expected<U, G> &&>::value &&
-    !std::is_constructible<T, const expected<U, G> &>::value &&
-    !std::is_constructible<T, const expected<U, G> &&>::value &&
-    !std::is_convertible<expected<U, G> &, T>::value &&
-    !std::is_convertible<expected<U, G> &&, T>::value &&
-    !std::is_convertible<const expected<U, G> &, T>::value &&
-    !std::is_convertible<const expected<U, G> &&, T>::value>;
-
-template <class T, class U>
-using is_void_or = conditional_t<std::is_void<T>::value, std::true_type, U>;
-
-template <class T>
-using is_copy_constructible_or_void =
-    is_void_or<T, std::is_copy_constructible<T>>;
-
-template <class T>
-using is_move_constructible_or_void =
-    is_void_or<T, std::is_move_constructible<T>>;
-
-template <class T>
-using is_copy_assignable_or_void =
-    is_void_or<T, std::is_copy_assignable<T>>;
-
-
-template <class T>
-using is_move_assignable_or_void =
-    is_void_or<T, std::is_move_assignable<T>>;
-    
-
-} // namespace detail
-
-/// \exclude
-namespace detail {
-struct no_init_t {};
-static constexpr no_init_t no_init{};
-
-// Implements the storage of the values, and ensures that the destructor is
-// trivial if it can be.
-//
-// This specialization is for where neither `T` or `E` is trivially
-// destructible, so the destructors must be called on destruction of the
-// `expected`
-template <class T, class E, bool = std::is_trivially_destructible<T>::value,
-          bool = std::is_trivially_destructible<E>::value>
-struct expected_storage_base {
-  constexpr expected_storage_base() : m_val(T{}), m_has_val(true) {}
-  constexpr expected_storage_base(no_init_t) : m_no_init(), m_has_val(false) {}
-
-  template <class... Args,
-            detail::enable_if_t<std::is_constructible<T, Args &&...>::value> * =
-                nullptr>
-  constexpr expected_storage_base(in_place_t, Args &&... args)
-      : m_val(std::forward<Args>(args)...), m_has_val(true) {}
-
-  template <class U, class... Args,
-            detail::enable_if_t<std::is_constructible<
-                T, std::initializer_list<U> &, Args &&...>::value> * = nullptr>
-  constexpr expected_storage_base(in_place_t, std::initializer_list<U> il,
-                                  Args &&... args)
-      : m_val(il, std::forward<Args>(args)...), m_has_val(true) {}
-  template <class... Args,
-            detail::enable_if_t<std::is_constructible<E, Args &&...>::value> * =
-                nullptr>
-  constexpr explicit expected_storage_base(unexpect_t, Args &&... args)
-      : m_unexpect(std::forward<Args>(args)...), m_has_val(false) {}
-
-  template <class U, class... Args,
-            detail::enable_if_t<std::is_constructible<
-                E, std::initializer_list<U> &, Args &&...>::value> * = nullptr>
-  constexpr explicit expected_storage_base(unexpect_t,
-                                           std::initializer_list<U> il,
-                                           Args &&... args)
-      : m_unexpect(il, std::forward<Args>(args)...), m_has_val(false) {}
-
-  ~expected_storage_base() {
-    if (m_has_val) {
-      m_val.~T();
-    } else {
-      m_unexpect.~unexpected<E>();
-    }
-  }
-  union {
-    char m_no_init;
-    T m_val;
-    unexpected<E> m_unexpect;
-  };
-  bool m_has_val;
-};
-
-// This specialization is for when both `T` and `E` are trivially-destructible,
-// so the destructor of the `expected` can be trivial.
-template <class T, class E> struct expected_storage_base<T, E, true, true> {
-  constexpr expected_storage_base() : m_val(T{}), m_has_val(true) {}
-  constexpr expected_storage_base(no_init_t) : m_no_init(), m_has_val(false) {}
-
-  template <class... Args,
-            detail::enable_if_t<std::is_constructible<T, Args &&...>::value> * =
-                nullptr>
-  constexpr expected_storage_base(in_place_t, Args &&... args)
-      : m_val(std::forward<Args>(args)...), m_has_val(true) {}
-
-  template <class U, class... Args,
-            detail::enable_if_t<std::is_constructible<
-                T, std::initializer_list<U> &, Args &&...>::value> * = nullptr>
-  constexpr expected_storage_base(in_place_t, std::initializer_list<U> il,
-                                  Args &&... args)
-      : m_val(il, std::forward<Args>(args)...), m_has_val(true) {}
-  template <class... Args,
-            detail::enable_if_t<std::is_constructible<E, Args &&...>::value> * =
-                nullptr>
-  constexpr explicit expected_storage_base(unexpect_t, Args &&... args)
-      : m_unexpect(std::forward<Args>(args)...), m_has_val(false) {}
-
-  template <class U, class... Args,
-            detail::enable_if_t<std::is_constructible<
-                E, std::initializer_list<U> &, Args &&...>::value> * = nullptr>
-  constexpr explicit expected_storage_base(unexpect_t,
-                                           std::initializer_list<U> il,
-                                           Args &&... args)
-      : m_unexpect(il, std::forward<Args>(args)...), m_has_val(false) {}
-
-  ~expected_storage_base() = default;
-  union {
-    char m_no_init;
-    T m_val;
-    unexpected<E> m_unexpect;
-  };
-  bool m_has_val;
-};
-
-// T is trivial, E is not.
-template <class T, class E> struct expected_storage_base<T, E, true, false> {
-  constexpr expected_storage_base() : m_val(T{}), m_has_val(true) {}
-  TL_EXPECTED_MSVC2015_CONSTEXPR expected_storage_base(no_init_t)
-      : m_no_init(), m_has_val(false) {}
-
-  template <class... Args,
-            detail::enable_if_t<std::is_constructible<T, Args &&...>::value> * =
-                nullptr>
-  constexpr expected_storage_base(in_place_t, Args &&... args)
-      : m_val(std::forward<Args>(args)...), m_has_val(true) {}
-
-  template <class U, class... Args,
-            detail::enable_if_t<std::is_constructible<
-                T, std::initializer_list<U> &, Args &&...>::value> * = nullptr>
-  constexpr expected_storage_base(in_place_t, std::initializer_list<U> il,
-                                  Args &&... args)
-      : m_val(il, std::forward<Args>(args)...), m_has_val(true) {}
-  template <class... Args,
-            detail::enable_if_t<std::is_constructible<E, Args &&...>::value> * =
-                nullptr>
-  constexpr explicit expected_storage_base(unexpect_t, Args &&... args)
-      : m_unexpect(std::forward<Args>(args)...), m_has_val(false) {}
-
-  template <class U, class... Args,
-            detail::enable_if_t<std::is_constructible<
-                E, std::initializer_list<U> &, Args &&...>::value> * = nullptr>
-  constexpr explicit expected_storage_base(unexpect_t,
-                                           std::initializer_list<U> il,
-                                           Args &&... args)
-      : m_unexpect(il, std::forward<Args>(args)...), m_has_val(false) {}
-
-  ~expected_storage_base() {
-    if (!m_has_val) {
-      m_unexpect.~unexpected<E>();
-    }
-  }
-
-  union {
-    char m_no_init;
-    T m_val;
-    unexpected<E> m_unexpect;
-  };
-  bool m_has_val;
-};
-
-// E is trivial, T is not.
-template <class T, class E> struct expected_storage_base<T, E, false, true> {
-  constexpr expected_storage_base() : m_val(T{}), m_has_val(true) {}
-  constexpr expected_storage_base(no_init_t) : m_no_init(), m_has_val(false) {}
-
-  template <class... Args,
-            detail::enable_if_t<std::is_constructible<T, Args &&...>::value> * =
-                nullptr>
-  constexpr expected_storage_base(in_place_t, Args &&... args)
-      : m_val(std::forward<Args>(args)...), m_has_val(true) {}
-
-  template <class U, class... Args,
-            detail::enable_if_t<std::is_constructible<
-                T, std::initializer_list<U> &, Args &&...>::value> * = nullptr>
-  constexpr expected_storage_base(in_place_t, std::initializer_list<U> il,
-                                  Args &&... args)
-      : m_val(il, std::forward<Args>(args)...), m_has_val(true) {}
-  template <class... Args,
-            detail::enable_if_t<std::is_constructible<E, Args &&...>::value> * =
-                nullptr>
-  constexpr explicit expected_storage_base(unexpect_t, Args &&... args)
-      : m_unexpect(std::forward<Args>(args)...), m_has_val(false) {}
-
-  template <class U, class... Args,
-            detail::enable_if_t<std::is_constructible<
-                E, std::initializer_list<U> &, Args &&...>::value> * = nullptr>
-  constexpr explicit expected_storage_base(unexpect_t,
-                                           std::initializer_list<U> il,
-                                           Args &&... args)
-      : m_unexpect(il, std::forward<Args>(args)...), m_has_val(false) {}
-
-  ~expected_storage_base() {
-    if (m_has_val) {
-      m_val.~T();
-    }
-  }
-  union {
-    char m_no_init;
-    T m_val;
-    unexpected<E> m_unexpect;
-  };
-  bool m_has_val;
-};
-
-// `T` is `void`, `E` is trivially-destructible
-template <class E> struct expected_storage_base<void, E, false, true> {
-  TL_EXPECTED_MSVC2015_CONSTEXPR expected_storage_base() : m_has_val(true) {}
-  constexpr expected_storage_base(no_init_t) : m_val(), m_has_val(false) {}
-
-  constexpr expected_storage_base(in_place_t) : m_has_val(true) {}
-
-  template <class... Args,
-            detail::enable_if_t<std::is_constructible<E, Args &&...>::value> * =
-                nullptr>
-  constexpr explicit expected_storage_base(unexpect_t, Args &&... args)
-      : m_unexpect(std::forward<Args>(args)...), m_has_val(false) {}
-
-  template <class U, class... Args,
-            detail::enable_if_t<std::is_constructible<
-                E, std::initializer_list<U> &, Args &&...>::value> * = nullptr>
-  constexpr explicit expected_storage_base(unexpect_t,
-                                           std::initializer_list<U> il,
-                                           Args &&... args)
-      : m_unexpect(il, std::forward<Args>(args)...), m_has_val(false) {}
-
-  ~expected_storage_base() = default;
-  struct dummy {};
-  union {
-    dummy m_val;
-    unexpected<E> m_unexpect;
-  };
-  bool m_has_val;
-};
-
-// `T` is `void`, `E` is not trivially-destructible
-template <class E> struct expected_storage_base<void, E, false, false> {
-  constexpr expected_storage_base() : m_dummy(), m_has_val(true) {}
-  constexpr expected_storage_base(no_init_t) : m_dummy(), m_has_val(false) {}
-
-  constexpr expected_storage_base(in_place_t) : m_dummy(), m_has_val(true) {}
-
-  template <class... Args,
-            detail::enable_if_t<std::is_constructible<E, Args &&...>::value> * =
-                nullptr>
-  constexpr explicit expected_storage_base(unexpect_t, Args &&... args)
-      : m_unexpect(std::forward<Args>(args)...), m_has_val(false) {}
-
-  template <class U, class... Args,
-            detail::enable_if_t<std::is_constructible<
-                E, std::initializer_list<U> &, Args &&...>::value> * = nullptr>
-  constexpr explicit expected_storage_base(unexpect_t,
-                                           std::initializer_list<U> il,
-                                           Args &&... args)
-      : m_unexpect(il, std::forward<Args>(args)...), m_has_val(false) {}
-
-  ~expected_storage_base() {
-    if (!m_has_val) {
-      m_unexpect.~unexpected<E>();
-    }
-  }
-
-  union {
-    char m_dummy;
-    unexpected<E> m_unexpect;
-  };
-  bool m_has_val;
-};
-
-// This base class provides some handy member functions which can be used in
-// further derived classes
-template <class T, class E>
-struct expected_operations_base : expected_storage_base<T, E> {
-  using expected_storage_base<T, E>::expected_storage_base;
-
-  template <class... Args> void construct(Args &&... args) noexcept {
-    new (std::addressof(this->m_val)) T(std::forward<Args>(args)...);
-    this->m_has_val = true;
-  }
-
-  template <class Rhs> void construct_with(Rhs &&rhs) noexcept {
-    new (std::addressof(this->m_val)) T(std::forward<Rhs>(rhs).get());
-    this->m_has_val = true;
-  }
-
-  template <class... Args> void construct_error(Args &&... args) noexcept {
-    new (std::addressof(this->m_unexpect))
-        unexpected<E>(std::forward<Args>(args)...);
-    this->m_has_val = false;
-  }
-
-  #ifdef TL_EXPECTED_EXCEPTIONS_ENABLED
-
-  // These assign overloads ensure that the most efficient assignment
-  // implementation is used while maintaining the strong exception guarantee.
-  // The problematic case is where rhs has a value, but *this does not.
-  //
-  // This overload handles the case where we can just copy-construct `T`
-  // directly into place without throwing.
-  template <class U = T,
-            detail::enable_if_t<std::is_nothrow_copy_constructible<U>::value>
-                * = nullptr>
-  void assign(const expected_operations_base &rhs) noexcept {
-    if (!this->m_has_val && rhs.m_has_val) {
-      geterr().~unexpected<E>();
-      construct(rhs.get());
-    } else {
-      assign_common(rhs);
-    }
-  }
-
-  // This overload handles the case where we can attempt to create a copy of
-  // `T`, then no-throw move it into place if the copy was successful.
-  template <class U = T,
-            detail::enable_if_t<!std::is_nothrow_copy_constructible<U>::value &&
-                                std::is_nothrow_move_constructible<U>::value>
-                * = nullptr>
-  void assign(const expected_operations_base &rhs) noexcept {
-    if (!this->m_has_val && rhs.m_has_val) {
-      T tmp = rhs.get();
-      geterr().~unexpected<E>();
-      construct(std::move(tmp));
-    } else {
-      assign_common(rhs);
-    }
-  }
-
-  // This overload is the worst-case, where we have to move-construct the
-  // unexpected value into temporary storage, then try to copy the T into place.
-  // If the construction succeeds, then everything is fine, but if it throws,
-  // then we move the old unexpected value back into place before rethrowing the
-  // exception.
-  template <class U = T,
-            detail::enable_if_t<!std::is_nothrow_copy_constructible<U>::value &&
-                                !std::is_nothrow_move_constructible<U>::value>
-                * = nullptr>
-  void assign(const expected_operations_base &rhs) {
-    if (!this->m_has_val && rhs.m_has_val) {
-      auto tmp = std::move(geterr());
-      geterr().~unexpected<E>();
-
-      try {
-        construct(rhs.get());
-      } catch (...) {
-        geterr() = std::move(tmp);
-        throw;
-      }
-    } else {
-      assign_common(rhs);
-    }
-  }
-
-  // These overloads do the same as above, but for rvalues
-  template <class U = T,
-            detail::enable_if_t<std::is_nothrow_move_constructible<U>::value>
-                * = nullptr>
-  void assign(expected_operations_base &&rhs) noexcept {
-    if (!this->m_has_val && rhs.m_has_val) {
-      geterr().~unexpected<E>();
-      construct(std::move(rhs).get());
-    } else {
-      assign_common(std::move(rhs));
-    }
-  }
-
-  template <class U = T,
-            detail::enable_if_t<!std::is_nothrow_move_constructible<U>::value>
-                * = nullptr>
-  void assign(expected_operations_base &&rhs) {
-    if (!this->m_has_val && rhs.m_has_val) {
-      auto tmp = std::move(geterr());
-      geterr().~unexpected<E>();
-      try {
-        construct(std::move(rhs).get());
-      } catch (...) {
-        geterr() = std::move(tmp);
-        throw;
-      }
-    } else {
-      assign_common(std::move(rhs));
-    }
-  }
-
-  #else
-
-  // If exceptions are disabled then we can just copy-construct
-  void assign(const expected_operations_base &rhs) noexcept {
-    if (!this->m_has_val && rhs.m_has_val) {
-      geterr().~unexpected<E>();
-      construct(rhs.get());
-    } else {
-      assign_common(rhs);
-    }
-  }
-
-  void assign(expected_operations_base &&rhs) noexcept {
-    if (!this->m_has_val && rhs.m_has_val) {
-      geterr().~unexpected<E>();
-      construct(std::move(rhs).get());
-    } else {
-      assign_common(rhs);
-    }
-  }
-
-  #endif
-
-  // The common part of move/copy assigning
-  template <class Rhs> void assign_common(Rhs &&rhs) {
-    if (this->m_has_val) {
-      if (rhs.m_has_val) {
-        get() = std::forward<Rhs>(rhs).get();
-      } else {
-		destroy_val();
-        construct_error(std::forward<Rhs>(rhs).geterr());
-      }
-    } else {
-      if (!rhs.m_has_val) {
-        geterr() = std::forward<Rhs>(rhs).geterr();
-      }
-    }
-  }
-
-  bool has_value() const { return this->m_has_val; }
-
-  TL_EXPECTED_11_CONSTEXPR T &get() & { return this->m_val; }
-  constexpr const T &get() const & { return this->m_val; }
-  TL_EXPECTED_11_CONSTEXPR T &&get() && { return std::move(this->m_val); }
-#ifndef TL_EXPECTED_NO_CONSTRR
-  constexpr const T &&get() const && { return std::move(this->m_val); }
-#endif
-
-  TL_EXPECTED_11_CONSTEXPR unexpected<E> &geterr() & {
-    return this->m_unexpect;
-  }
-  constexpr const unexpected<E> &geterr() const & { return this->m_unexpect; }
-  TL_EXPECTED_11_CONSTEXPR unexpected<E> &&geterr() && {
-    return std::move(this->m_unexpect);
-  }
-#ifndef TL_EXPECTED_NO_CONSTRR
-  constexpr const unexpected<E> &&geterr() const && {
-    return std::move(this->m_unexpect);
-  }
-#endif
-
-  constexpr void destroy_val() {
-	get().~T();
-  }
-};
-
-// This base class provides some handy member functions which can be used in
-// further derived classes
-template <class E>
-struct expected_operations_base<void, E> : expected_storage_base<void, E> {
-  using expected_storage_base<void, E>::expected_storage_base;
-
-  template <class... Args> void construct() noexcept { this->m_has_val = true; }
-
-  // This function doesn't use its argument, but needs it so that code in
-  // levels above this can work independently of whether T is void
-  template <class Rhs> void construct_with(Rhs &&) noexcept {
-    this->m_has_val = true;
-  }
-
-  template <class... Args> void construct_error(Args &&... args) noexcept {
-    new (std::addressof(this->m_unexpect))
-        unexpected<E>(std::forward<Args>(args)...);
-    this->m_has_val = false;
-  }
-
-  template <class Rhs> void assign(Rhs &&rhs) noexcept {
-    if (!this->m_has_val) {
-      if (rhs.m_has_val) {
-        geterr().~unexpected<E>();
-        construct();
-      } else {
-        geterr() = std::forward<Rhs>(rhs).geterr();
-      }
-    } else {
-      if (!rhs.m_has_val) {
-        construct_error(std::forward<Rhs>(rhs).geterr());
-      }
-    }
-  }
-
-  bool has_value() const { return this->m_has_val; }
-
-  TL_EXPECTED_11_CONSTEXPR unexpected<E> &geterr() & {
-    return this->m_unexpect;
-  }
-  constexpr const unexpected<E> &geterr() const & { return this->m_unexpect; }
-  TL_EXPECTED_11_CONSTEXPR unexpected<E> &&geterr() && {
-    return std::move(this->m_unexpect);
-  }
-#ifndef TL_EXPECTED_NO_CONSTRR
-  constexpr const unexpected<E> &&geterr() const && {
-    return std::move(this->m_unexpect);
-  }
-#endif
-
-  constexpr void destroy_val() {
-	  //no-op
-  }
-};
-
-// This class manages conditionally having a trivial copy constructor
-// This specialization is for when T and E are trivially copy constructible
-template <class T, class E,
-          bool = is_void_or<T, TL_EXPECTED_IS_TRIVIALLY_COPY_CONSTRUCTIBLE(T)>::
-              value &&TL_EXPECTED_IS_TRIVIALLY_COPY_CONSTRUCTIBLE(E)::value>
-struct expected_copy_base : expected_operations_base<T, E> {
-  using expected_operations_base<T, E>::expected_operations_base;
-};
-
-// This specialization is for when T or E are not trivially copy constructible
-template <class T, class E>
-struct expected_copy_base<T, E, false> : expected_operations_base<T, E> {
-  using expected_operations_base<T, E>::expected_operations_base;
-
-  expected_copy_base() = default;
-  expected_copy_base(const expected_copy_base &rhs)
-      : expected_operations_base<T, E>(no_init) {
-    if (rhs.has_value()) {
-      this->construct_with(rhs);
-    } else {
-      this->construct_error(rhs.geterr());
-    }
-  }
-
-  expected_copy_base(expected_copy_base &&rhs) = default;
-  expected_copy_base &operator=(const expected_copy_base &rhs) = default;
-  expected_copy_base &operator=(expected_copy_base &&rhs) = default;
-};
-
-// This class manages conditionally having a trivial move constructor
-// Unfortunately there's no way to achieve this in GCC < 5 AFAIK, since it
-// doesn't implement an analogue to std::is_trivially_move_constructible. We
-// have to make do with a non-trivial move constructor even if T is trivially
-// move constructible
-#ifndef TL_EXPECTED_GCC49
-template <class T, class E,
-          bool = is_void_or<T, std::is_trivially_move_constructible<T>>::value
-              &&std::is_trivially_move_constructible<E>::value>
-struct expected_move_base : expected_copy_base<T, E> {
-  using expected_copy_base<T, E>::expected_copy_base;
-};
-#else
-template <class T, class E, bool = false> struct expected_move_base;
-#endif
-template <class T, class E>
-struct expected_move_base<T, E, false> : expected_copy_base<T, E> {
-  using expected_copy_base<T, E>::expected_copy_base;
-
-  expected_move_base() = default;
-  expected_move_base(const expected_move_base &rhs) = default;
-
-  expected_move_base(expected_move_base &&rhs) noexcept(
-      std::is_nothrow_move_constructible<T>::value)
-      : expected_copy_base<T, E>(no_init) {
-    if (rhs.has_value()) {
-      this->construct_with(std::move(rhs));
-    } else {
-      this->construct_error(std::move(rhs.geterr()));
-    }
-  }
-  expected_move_base &operator=(const expected_move_base &rhs) = default;
-  expected_move_base &operator=(expected_move_base &&rhs) = default;
-};
-
-// This class manages conditionally having a trivial copy assignment operator
-template <class T, class E,
-          bool = is_void_or<
-              T, conjunction<TL_EXPECTED_IS_TRIVIALLY_COPY_ASSIGNABLE(T),
-                             TL_EXPECTED_IS_TRIVIALLY_COPY_CONSTRUCTIBLE(T),
-                             TL_EXPECTED_IS_TRIVIALLY_DESTRUCTIBLE(T)>>::value
-              &&TL_EXPECTED_IS_TRIVIALLY_COPY_ASSIGNABLE(E)::value
-                  &&TL_EXPECTED_IS_TRIVIALLY_COPY_CONSTRUCTIBLE(E)::value
-                      &&TL_EXPECTED_IS_TRIVIALLY_DESTRUCTIBLE(E)::value>
-struct expected_copy_assign_base : expected_move_base<T, E> {
-  using expected_move_base<T, E>::expected_move_base;
-};
-
-template <class T, class E>
-struct expected_copy_assign_base<T, E, false> : expected_move_base<T, E> {
-  using expected_move_base<T, E>::expected_move_base;
-
-  expected_copy_assign_base() = default;
-  expected_copy_assign_base(const expected_copy_assign_base &rhs) = default;
-
-  expected_copy_assign_base(expected_copy_assign_base &&rhs) = default;
-  expected_copy_assign_base &operator=(const expected_copy_assign_base &rhs) {
-    this->assign(rhs);
-    return *this;
-  }
-  expected_copy_assign_base &
-  operator=(expected_copy_assign_base &&rhs) = default;
-};
-
-// This class manages conditionally having a trivial move assignment operator
-// Unfortunately there's no way to achieve this in GCC < 5 AFAIK, since it
-// doesn't implement an analogue to std::is_trivially_move_assignable. We have
-// to make do with a non-trivial move assignment operator even if T is trivially
-// move assignable
-#ifndef TL_EXPECTED_GCC49
-template <class T, class E,
-          bool =
-              is_void_or<T, conjunction<std::is_trivially_destructible<T>,
-                                        std::is_trivially_move_constructible<T>,
-                                        std::is_trivially_move_assignable<T>>>::
-                  value &&std::is_trivially_destructible<E>::value
-                      &&std::is_trivially_move_constructible<E>::value
-                          &&std::is_trivially_move_assignable<E>::value>
-struct expected_move_assign_base : expected_copy_assign_base<T, E> {
-  using expected_copy_assign_base<T, E>::expected_copy_assign_base;
-};
-#else
-template <class T, class E, bool = false> struct expected_move_assign_base;
-#endif
-
-template <class T, class E>
-struct expected_move_assign_base<T, E, false>
-    : expected_copy_assign_base<T, E> {
-  using expected_copy_assign_base<T, E>::expected_copy_assign_base;
-
-  expected_move_assign_base() = default;
-  expected_move_assign_base(const expected_move_assign_base &rhs) = default;
-
-  expected_move_assign_base(expected_move_assign_base &&rhs) = default;
-
-  expected_move_assign_base &
-  operator=(const expected_move_assign_base &rhs) = default;
-
-  expected_move_assign_base &
-  operator=(expected_move_assign_base &&rhs) noexcept(
-      std::is_nothrow_move_constructible<T>::value
-          &&std::is_nothrow_move_assignable<T>::value) {
-    this->assign(std::move(rhs));
-    return *this;
-  }
-};
-
-// expected_delete_ctor_base will conditionally delete copy and move
-// constructors depending on whether T is copy/move constructible
-template <class T, class E,
-          bool EnableCopy = (is_copy_constructible_or_void<T>::value &&
-                             std::is_copy_constructible<E>::value),
-          bool EnableMove = (is_move_constructible_or_void<T>::value &&
-                             std::is_move_constructible<E>::value)>
-struct expected_delete_ctor_base {
-  expected_delete_ctor_base() = default;
-  expected_delete_ctor_base(const expected_delete_ctor_base &) = default;
-  expected_delete_ctor_base(expected_delete_ctor_base &&) noexcept = default;
-  expected_delete_ctor_base &
-  operator=(const expected_delete_ctor_base &) = default;
-  expected_delete_ctor_base &
-  operator=(expected_delete_ctor_base &&) noexcept = default;
-};
-
-template <class T, class E>
-struct expected_delete_ctor_base<T, E, true, false> {
-  expected_delete_ctor_base() = default;
-  expected_delete_ctor_base(const expected_delete_ctor_base &) = default;
-  expected_delete_ctor_base(expected_delete_ctor_base &&) noexcept = delete;
-  expected_delete_ctor_base &
-  operator=(const expected_delete_ctor_base &) = default;
-  expected_delete_ctor_base &
-  operator=(expected_delete_ctor_base &&) noexcept = default;
-};
-
-template <class T, class E>
-struct expected_delete_ctor_base<T, E, false, true> {
-  expected_delete_ctor_base() = default;
-  expected_delete_ctor_base(const expected_delete_ctor_base &) = delete;
-  expected_delete_ctor_base(expected_delete_ctor_base &&) noexcept = default;
-  expected_delete_ctor_base &
-  operator=(const expected_delete_ctor_base &) = default;
-  expected_delete_ctor_base &
-  operator=(expected_delete_ctor_base &&) noexcept = default;
-};
-
-template <class T, class E>
-struct expected_delete_ctor_base<T, E, false, false> {
-  expected_delete_ctor_base() = default;
-  expected_delete_ctor_base(const expected_delete_ctor_base &) = delete;
-  expected_delete_ctor_base(expected_delete_ctor_base &&) noexcept = delete;
-  expected_delete_ctor_base &
-  operator=(const expected_delete_ctor_base &) = default;
-  expected_delete_ctor_base &
-  operator=(expected_delete_ctor_base &&) noexcept = default;
-};
-
-// expected_delete_assign_base will conditionally delete copy and move
-// constructors depending on whether T and E are copy/move constructible +
-// assignable
-template <class T, class E,
-          bool EnableCopy = (is_copy_constructible_or_void<T>::value &&
-                             std::is_copy_constructible<E>::value &&
-                             is_copy_assignable_or_void<T>::value &&
-                             std::is_copy_assignable<E>::value),
-          bool EnableMove = (is_move_constructible_or_void<T>::value &&
-                             std::is_move_constructible<E>::value &&
-                             is_move_assignable_or_void<T>::value &&
-                             std::is_move_assignable<E>::value)>
-struct expected_delete_assign_base {
-  expected_delete_assign_base() = default;
-  expected_delete_assign_base(const expected_delete_assign_base &) = default;
-  expected_delete_assign_base(expected_delete_assign_base &&) noexcept =
-      default;
-  expected_delete_assign_base &
-  operator=(const expected_delete_assign_base &) = default;
-  expected_delete_assign_base &
-  operator=(expected_delete_assign_base &&) noexcept = default;
-};
-
-template <class T, class E>
-struct expected_delete_assign_base<T, E, true, false> {
-  expected_delete_assign_base() = default;
-  expected_delete_assign_base(const expected_delete_assign_base &) = default;
-  expected_delete_assign_base(expected_delete_assign_base &&) noexcept =
-      default;
-  expected_delete_assign_base &
-  operator=(const expected_delete_assign_base &) = default;
-  expected_delete_assign_base &
-  operator=(expected_delete_assign_base &&) noexcept = delete;
-};
-
-template <class T, class E>
-struct expected_delete_assign_base<T, E, false, true> {
-  expected_delete_assign_base() = default;
-  expected_delete_assign_base(const expected_delete_assign_base &) = default;
-  expected_delete_assign_base(expected_delete_assign_base &&) noexcept =
-      default;
-  expected_delete_assign_base &
-  operator=(const expected_delete_assign_base &) = delete;
-  expected_delete_assign_base &
-  operator=(expected_delete_assign_base &&) noexcept = default;
-};
-
-template <class T, class E>
-struct expected_delete_assign_base<T, E, false, false> {
-  expected_delete_assign_base() = default;
-  expected_delete_assign_base(const expected_delete_assign_base &) = default;
-  expected_delete_assign_base(expected_delete_assign_base &&) noexcept =
-      default;
-  expected_delete_assign_base &
-  operator=(const expected_delete_assign_base &) = delete;
-  expected_delete_assign_base &
-  operator=(expected_delete_assign_base &&) noexcept = delete;
-};
-
-// This is needed to be able to construct the expected_default_ctor_base which
-// follows, while still conditionally deleting the default constructor.
-struct default_constructor_tag {
-  explicit constexpr default_constructor_tag() = default;
-};
-
-// expected_default_ctor_base will ensure that expected has a deleted default
-// consturctor if T is not default constructible.
-// This specialization is for when T is default constructible
-template <class T, class E,
-          bool Enable =
-              std::is_default_constructible<T>::value || std::is_void<T>::value>
-struct expected_default_ctor_base {
-  constexpr expected_default_ctor_base() noexcept = default;
-  constexpr expected_default_ctor_base(
-      expected_default_ctor_base const &) noexcept = default;
-  constexpr expected_default_ctor_base(expected_default_ctor_base &&) noexcept =
-      default;
-  expected_default_ctor_base &
-  operator=(expected_default_ctor_base const &) noexcept = default;
-  expected_default_ctor_base &
-  operator=(expected_default_ctor_base &&) noexcept = default;
-
-  constexpr explicit expected_default_ctor_base(default_constructor_tag) {}
-};
-
-// This specialization is for when T is not default constructible
-template <class T, class E> struct expected_default_ctor_base<T, E, false> {
-  constexpr expected_default_ctor_base() noexcept = delete;
-  constexpr expected_default_ctor_base(
-      expected_default_ctor_base const &) noexcept = default;
-  constexpr expected_default_ctor_base(expected_default_ctor_base &&) noexcept =
-      default;
-  expected_default_ctor_base &
-  operator=(expected_default_ctor_base const &) noexcept = default;
-  expected_default_ctor_base &
-  operator=(expected_default_ctor_base &&) noexcept = default;
-
-  constexpr explicit expected_default_ctor_base(default_constructor_tag) {}
-};
-} // namespace detail
-
-template <class E> class bad_expected_access : public std::exception {
-public:
-  explicit bad_expected_access(E e) : m_val(std::move(e)) {}
-
-  virtual const char *what() const noexcept override {
-    return "Bad expected access";
-  }
-
-  const E &error() const & { return m_val; }
-  E &error() & { return m_val; }
-  const E &&error() const && { return std::move(m_val); }
-  E &&error() && { return std::move(m_val); }
-
-private:
-  E m_val;
-};
-
-/// An `expected<T, E>` object is an object that contains the storage for
-/// another object and manages the lifetime of this contained object `T`.
-/// Alternatively it could contain the storage for another unexpected object
-/// `E`. The contained object may not be initialized after the expected object
-/// has been initialized, and may not be destroyed before the expected object
-/// has been destroyed. The initialization state of the contained object is
-/// tracked by the expected object.
-template <class T, class E>
-class expected : private detail::expected_move_assign_base<T, E>,
-                 private detail::expected_delete_ctor_base<T, E>,
-                 private detail::expected_delete_assign_base<T, E>,
-                 private detail::expected_default_ctor_base<T, E> {
-  static_assert(!std::is_reference<T>::value, "T must not be a reference");
-  static_assert(!std::is_same<T, std::remove_cv<in_place_t>>::value,
-                "T must not be in_place_t");
-  static_assert(!std::is_same<T, std::remove_cv<unexpect_t>>::value,
-                "T must not be unexpect_t");
-  static_assert(!std::is_same<T, std::remove_cv<unexpected<E>>>::value,
-                "T must not be unexpected<E>");
-  static_assert(!std::is_reference<E>::value, "E must not be a reference");
-
-  T *valptr() { return std::addressof(this->m_val); }
-  const T *valptr() const { return std::addressof(this->m_val); }    
-  unexpected<E> *errptr() { return std::addressof(this->m_unexpect); }
-  const unexpected<E> *errptr() const { return std::addressof(this->m_unexpect); }    
-
-  template <class U = T,
-            detail::enable_if_t<!std::is_void<U>::value> * = nullptr>
-  U &val() {
-    return this->m_val;
-  }
-  unexpected<E> &err() { return this->m_unexpect; }
-
-  template <class U = T,
-            detail::enable_if_t<!std::is_void<U>::value> * = nullptr>
-  const U &val() const {
-    return this->m_val;
-  }
-  const unexpected<E> &err() const { return this->m_unexpect; }
-
-  using impl_base = detail::expected_move_assign_base<T, E>;
-  using ctor_base = detail::expected_default_ctor_base<T, E>;
-
-public:
-  typedef T value_type;
-  typedef E error_type;
-  typedef unexpected<E> unexpected_type;
-
-#if defined(TL_EXPECTED_CXX14) && !defined(TL_EXPECTED_GCC49) &&               \
-    !defined(TL_EXPECTED_GCC54) && !defined(TL_EXPECTED_GCC55)
-  /// \group and_then
-  /// Carries out some operation which returns an expected on the stored object
-  /// if there is one. \requires `std::invoke(std::forward<F>(f), value())`
-  /// returns an `expected<U>` for some `U`. \returns Let `U` be the result
-  /// of `std::invoke(std::forward<F>(f), value())`. Returns an
-  /// `expected<U>`. The return value is empty if `*this` is empty,
-  /// otherwise the return value of `std::invoke(std::forward<F>(f), value())`
-  /// is returned.
-  /// \synopsis template <class F>\nconstexpr auto and_then(F &&f) &;
-  template <class F> TL_EXPECTED_11_CONSTEXPR auto and_then(F &&f) & {
-    return and_then_impl(*this, std::forward<F>(f));
-  }
-
-  /// \group and_then
-  /// \synopsis template <class F>\nconstexpr auto and_then(F &&f) &&;
-  template <class F> TL_EXPECTED_11_CONSTEXPR auto and_then(F &&f) && {
-    return and_then_impl(std::move(*this), std::forward<F>(f));
-  }
-
-  /// \group and_then
-  /// \synopsis template <class F>\nconstexpr auto and_then(F &&f) const &;
-  template <class F> constexpr auto and_then(F &&f) const & {
-    return and_then_impl(*this, std::forward<F>(f));
-  }
-
-#ifndef TL_EXPECTED_NO_CONSTRR
-  /// \group and_then
-  /// \synopsis template <class F>\nconstexpr auto and_then(F &&f) const &&;
-  template <class F> constexpr auto and_then(F &&f) const && {
-    return and_then_impl(std::move(*this), std::forward<F>(f));
-  }
-#endif
-
-#else
-  /// \group and_then
-  /// Carries out some operation which returns an expected on the stored object
-  /// if there is one. \requires `std::invoke(std::forward<F>(f), value())`
-  /// returns an `expected<U>` for some `U`. \returns Let `U` be the result
-  /// of `std::invoke(std::forward<F>(f), value())`. Returns an
-  /// `expected<U>`. The return value is empty if `*this` is empty,
-  /// otherwise the return value of `std::invoke(std::forward<F>(f), value())`
-  /// is returned.
-  /// \synopsis template <class F>\nconstexpr auto and_then(F &&f) &;
-  template <class F>
-  TL_EXPECTED_11_CONSTEXPR auto
-  and_then(F &&f) & -> decltype(and_then_impl(*this, std::forward<F>(f))) {
-    return and_then_impl(*this, std::forward<F>(f));
-  }
-
-  /// \group and_then
-  /// \synopsis template <class F>\nconstexpr auto and_then(F &&f) &&;
-  template <class F>
-  TL_EXPECTED_11_CONSTEXPR auto and_then(F &&f) && -> decltype(
-      and_then_impl(std::move(*this), std::forward<F>(f))) {
-    return and_then_impl(std::move(*this), std::forward<F>(f));
-  }
-
-  /// \group and_then
-  /// \synopsis template <class F>\nconstexpr auto and_then(F &&f) const &;
-  template <class F>
-  constexpr auto and_then(F &&f) const & -> decltype(
-      and_then_impl(*this, std::forward<F>(f))) {
-    return and_then_impl(*this, std::forward<F>(f));
-  }
-
-#ifndef TL_EXPECTED_NO_CONSTRR
-  /// \group and_then
-  /// \synopsis template <class F>\nconstexpr auto and_then(F &&f) const &&;
-  template <class F>
-  constexpr auto and_then(F &&f) const && -> decltype(
-      and_then_impl(std::move(*this), std::forward<F>(f))) {
-    return and_then_impl(std::move(*this), std::forward<F>(f));
-  }
-#endif
-#endif
-
-#if defined(TL_EXPECTED_CXX14) && !defined(TL_EXPECTED_GCC49) &&               \
-    !defined(TL_EXPECTED_GCC54) && !defined(TL_EXPECTED_GCC55)
-  /// \brief Carries out some operation on the stored object if there is one.
-  /// \returns Let `U` be the result of `std::invoke(std::forward<F>(f),
-  /// value())`. If `U` is `void`, returns an `expected<monostate,E>, otherwise
-  //  returns an `expected<U,E>`. If `*this` is unexpected, the
-  /// result is `*this`, otherwise an `expected<U,E>` is constructed from the
-  /// return value of `std::invoke(std::forward<F>(f), value())` and is
-  /// returned.
-  ///
-  /// \group map
-  /// \synopsis template <class F> constexpr auto map(F &&f) &;
-  template <class F> TL_EXPECTED_11_CONSTEXPR auto map(F &&f) & {
-    return expected_map_impl(*this, std::forward<F>(f));
-  }
-
-  /// \group map
-  /// \synopsis template <class F> constexpr auto map(F &&f) &&;
-  template <class F> TL_EXPECTED_11_CONSTEXPR auto map(F &&f) && {
-    return expected_map_impl(std::move(*this), std::forward<F>(f));
-  }
-
-  /// \group map
-  /// \synopsis template <class F> constexpr auto map(F &&f) const &;
-  template <class F> constexpr auto map(F &&f) const & {
-    return expected_map_impl(*this, std::forward<F>(f));
-  }
-
-  /// \group map
-  /// \synopsis template <class F> constexpr auto map(F &&f) const &&;
-  template <class F> constexpr auto map(F &&f) const && {
-    return expected_map_impl(std::move(*this), std::forward<F>(f));
-  }
-#else
-  /// \brief Carries out some operation on the stored object if there is one.
-  /// \returns Let `U` be the result of `std::invoke(std::forward<F>(f),
-  /// value())`. If `U` is `void`, returns an `expected<monostate,E>, otherwise
-  //  returns an `expected<U,E>`. If `*this` is unexpected, the
-  /// result is `*this`, otherwise an `expected<U,E>` is constructed from the
-  /// return value of `std::invoke(std::forward<F>(f), value())` and is
-  /// returned.
-  ///
-  /// \group map
-  /// \synopsis template <class F> constexpr auto map(F &&f) &;
-  template <class F>
-  TL_EXPECTED_11_CONSTEXPR decltype(
-      expected_map_impl(std::declval<expected &>(), std::declval<F &&>()))
-  map(F &&f) & {
-    return expected_map_impl(*this, std::forward<F>(f));
-  }
-
-  /// \group map
-  /// \synopsis template <class F> constexpr auto map(F &&f) &&;
-  template <class F>
-  TL_EXPECTED_11_CONSTEXPR decltype(
-      expected_map_impl(std::declval<expected>(), std::declval<F &&>()))
-  map(F &&f) && {
-    return expected_map_impl(std::move(*this), std::forward<F>(f));
-  }
-
-  /// \group map
-  /// \synopsis template <class F> constexpr auto map(F &&f) const &;
-  template <class F>
-  constexpr decltype(expected_map_impl(std::declval<const expected &>(),
-                                       std::declval<F &&>()))
-  map(F &&f) const & {
-    return expected_map_impl(*this, std::forward<F>(f));
-  }
-
-#ifndef TL_EXPECTED_NO_CONSTRR
-  /// \group map
-  /// \synopsis template <class F> constexpr auto map(F &&f) const &&;
-  template <class F>
-  constexpr decltype(expected_map_impl(std::declval<const expected &&>(),
-                                       std::declval<F &&>()))
-  map(F &&f) const && {
-    return expected_map_impl(std::move(*this), std::forward<F>(f));
-  }
-#endif
-#endif
-
-#if defined(TL_EXPECTED_CXX14) && !defined(TL_EXPECTED_GCC49) &&               \
-    !defined(TL_EXPECTED_GCC54) && !defined(TL_EXPECTED_GCC55)
-  /// \brief Carries out some operation on the stored unexpected object if there
-  /// is one.
-  /// \returns Let `U` be the result of `std::invoke(std::forward<F>(f),
-  /// value())`. If `U` is `void`, returns an `expected<T,monostate>`, otherwise
-  /// returns an `expected<T,U>`. If `*this` has an expected
-  /// value, the result is `*this`, otherwise an `expected<T,U>` is constructed
-  /// from `make_unexpected(std::invoke(std::forward<F>(f), value()))` and is
-  /// returned.
-  ///
-  /// \group map_error
-  /// \synopsis template <class F> constexpr auto map_error(F &&f) &;
-  template <class F> TL_EXPECTED_11_CONSTEXPR auto map_error(F &&f) & {
-    return map_error_impl(*this, std::forward<F>(f));
-  }
-
-  /// \group map_error
-  /// \synopsis template <class F> constexpr auto map_error(F &&f) &&;
-  template <class F> TL_EXPECTED_11_CONSTEXPR auto map_error(F &&f) && {
-    return map_error_impl(std::move(*this), std::forward<F>(f));
-  }
-
-  /// \group map_error
-  /// \synopsis template <class F> constexpr auto map_error(F &&f) const &;
-  template <class F> constexpr auto map_error(F &&f) const & {
-    return map_error_impl(*this, std::forward<F>(f));
-  }
-
-  /// \group map_error
-  /// \synopsis template <class F> constexpr auto map_error(F &&f) const &&;
-  template <class F> constexpr auto map_error(F &&f) const && {
-    return map_error_impl(std::move(*this), std::forward<F>(f));
-  }
-#else
-  /// \brief Carries out some operation on the stored unexpected object if there
-  /// is one.
-  /// \returns Let `U` be the result of `std::invoke(std::forward<F>(f),
-  /// value())`. Returns an `expected<T,U>`. If `*this` has an expected
-  /// value, the result is `*this`, otherwise an `expected<T,U>` is constructed
-  /// from `make_unexpected(std::invoke(std::forward<F>(f), value()))` and is
-  /// returned.
-  ///
-  /// \group map_error
-  /// \synopsis template <class F> constexpr auto map_error(F &&f) &;
-  template <class F>
-  TL_EXPECTED_11_CONSTEXPR decltype(map_error_impl(std::declval<expected &>(),
-                                                   std::declval<F &&>()))
-  map_error(F &&f) & {
-    return map_error_impl(*this, std::forward<F>(f));
-  }
-
-  /// \group map_error
-  /// \synopsis template <class F> constexpr auto map_error(F &&f) &&;
-  template <class F>
-  TL_EXPECTED_11_CONSTEXPR decltype(map_error_impl(std::declval<expected &&>(),
-                                                   std::declval<F &&>()))
-  map_error(F &&f) && {
-    return map_error_impl(std::move(*this), std::forward<F>(f));
-  }
-
-  /// \group map_error
-  /// \synopsis template <class F> constexpr auto map_error(F &&f) const &;
-  template <class F>
-  constexpr decltype(map_error_impl(std::declval<const expected &>(),
-                                    std::declval<F &&>()))
-  map_error(F &&f) const & {
-    return map_error_impl(*this, std::forward<F>(f));
-  }
-
-#ifndef TL_EXPECTED_NO_CONSTRR
-  /// \group map_error
-  /// \synopsis template <class F> constexpr auto map_error(F &&f) const &&;
-  template <class F>
-  constexpr decltype(map_error_impl(std::declval<const expected &&>(),
-                                    std::declval<F &&>()))
-  map_error(F &&f) const && {
-    return map_error_impl(std::move(*this), std::forward<F>(f));
-  }
-#endif
-#endif
-
-  /// \brief Calls `f` if the expectd is in the unexpected state
-  /// \requires `F` is invokable with `E`, and `std::invoke_result_t<F>`
-  /// must be void or convertible to `expcted<T,E>`.
-  /// \effects If `*this` has a value, returns `*this`.
-  /// Otherwise, if `f` returns `void`, calls `std::forward<F>(f)(E)` and returns
-  /// `std::nullopt`. Otherwise, returns `std::forward<F>(f)(E)`.
-  ///
-  /// \group or_else
-  template <class F> expected TL_EXPECTED_11_CONSTEXPR or_else(F &&f) & {
-    return or_else_impl(*this, std::forward<F>(f));
-  }
-
-  template <class F> expected TL_EXPECTED_11_CONSTEXPR or_else(F &&f) && {
-    return or_else_impl(std::move(*this), std::forward<F>(f));
-  }
-
-  template <class F> expected constexpr or_else(F &&f) const & {
-    return or_else_impl(*this, std::forward<F>(f));
-  }
-
-#ifndef TL_EXPECTED_NO_CONSTRR
-  template <class F> expected constexpr or_else(F &&f) const && {
-    return or_else_impl(std::move(*this), std::forward<F>(f));
-  }
-#endif
-  constexpr expected() = default;
-  constexpr expected(const expected &rhs) = default;
-  constexpr expected(expected &&rhs) = default;
-  expected &operator=(const expected &rhs) = default;
-  expected &operator=(expected &&rhs) = default;
-
-  template <class... Args,
-            detail::enable_if_t<std::is_constructible<T, Args &&...>::value> * =
-                nullptr>
-  constexpr expected(in_place_t, Args &&... args)
-      : impl_base(in_place, std::forward<Args>(args)...),
-        ctor_base(detail::default_constructor_tag{}) {}
-
-  template <class U, class... Args,
-            detail::enable_if_t<std::is_constructible<
-                T, std::initializer_list<U> &, Args &&...>::value> * = nullptr>
-  constexpr expected(in_place_t, std::initializer_list<U> il, Args &&... args)
-      : impl_base(in_place, il, std::forward<Args>(args)...),
-        ctor_base(detail::default_constructor_tag{}) {}
-
-  /// \group unexpected_ctor
-  /// \synopsis EXPLICIT constexpr expected(const unexpected<G> &e);
-  template <class G = E,
-            detail::enable_if_t<std::is_constructible<E, const G &>::value> * =
-                nullptr,
-            detail::enable_if_t<!std::is_convertible<const G &, E>::value> * =
-                nullptr>
-  explicit constexpr expected(const unexpected<G> &e)
-      : impl_base(unexpect, e.value()),
-        ctor_base(detail::default_constructor_tag{}) {}
-
-  /// \exclude
-  template <
-      class G = E,
-      detail::enable_if_t<std::is_constructible<E, const G &>::value> * =
-          nullptr,
-      detail::enable_if_t<std::is_convertible<const G &, E>::value> * = nullptr>
-  constexpr expected(unexpected<G> const &e)
-      : impl_base(unexpect, e.value()),
-        ctor_base(detail::default_constructor_tag{}) {}
-
-  /// \group unexpected_ctor
-  /// \synopsis EXPLICIT constexpr expected(unexpected<G> &&e);
-  template <
-      class G = E,
-      detail::enable_if_t<std::is_constructible<E, G &&>::value> * = nullptr,
-      detail::enable_if_t<!std::is_convertible<G &&, E>::value> * = nullptr>
-  explicit constexpr expected(unexpected<G> &&e) noexcept(
-      std::is_nothrow_constructible<E, G &&>::value)
-      : impl_base(unexpect, std::move(e.value())),
-        ctor_base(detail::default_constructor_tag{}) {}
-
-  /// \exclude
-  template <
-      class G = E,
-      detail::enable_if_t<std::is_constructible<E, G &&>::value> * = nullptr,
-      detail::enable_if_t<std::is_convertible<G &&, E>::value> * = nullptr>
-  constexpr expected(unexpected<G> &&e) noexcept(
-      std::is_nothrow_constructible<E, G &&>::value)
-      : impl_base(unexpect, std::move(e.value())),
-        ctor_base(detail::default_constructor_tag{}) {}
-
-  template <class... Args,
-            detail::enable_if_t<std::is_constructible<E, Args &&...>::value> * =
-                nullptr>
-  constexpr explicit expected(unexpect_t, Args &&... args)
-      : impl_base(unexpect, std::forward<Args>(args)...),
-        ctor_base(detail::default_constructor_tag{}) {}
-
-  /// \exclude
-  template <class U, class... Args,
-            detail::enable_if_t<std::is_constructible<
-                E, std::initializer_list<U> &, Args &&...>::value> * = nullptr>
-  constexpr explicit expected(unexpect_t, std::initializer_list<U> il,
-                              Args &&... args)
-      : impl_base(unexpect, il, std::forward<Args>(args)...),
-        ctor_base(detail::default_constructor_tag{}) {}
-
-  template <class U, class G,
-            detail::enable_if_t<!(std::is_convertible<U const &, T>::value &&
-                                  std::is_convertible<G const &, E>::value)> * =
-                nullptr,
-            detail::expected_enable_from_other<T, E, U, G, const U &, const G &>
-                * = nullptr>
-  explicit TL_EXPECTED_11_CONSTEXPR expected(const expected<U, G> &rhs)
-      : ctor_base(detail::default_constructor_tag{}) {
-    if (rhs.has_value()) {
-      this->construct(*rhs);
-    } else {
-      this->construct_error(rhs.error());        
-    }
-  }
-
-  /// \exclude
-  template <class U, class G,
-            detail::enable_if_t<(std::is_convertible<U const &, T>::value &&
-                                 std::is_convertible<G const &, E>::value)> * =
-                nullptr,
-            detail::expected_enable_from_other<T, E, U, G, const U &, const G &>
-                * = nullptr>
-  TL_EXPECTED_11_CONSTEXPR expected(const expected<U, G> &rhs)
-      : ctor_base(detail::default_constructor_tag{}) {
-    if (rhs.has_value()) {
-      this->construct(*rhs);
-    } else {
-      this->construct_error(rhs.error());        
-    }      
-  }
-
-  template <
-      class U, class G,
-      detail::enable_if_t<!(std::is_convertible<U &&, T>::value &&
-                            std::is_convertible<G &&, E>::value)> * = nullptr,
-      detail::expected_enable_from_other<T, E, U, G, U &&, G &&> * = nullptr>
-  explicit TL_EXPECTED_11_CONSTEXPR expected(expected<U, G> &&rhs)
-      : ctor_base(detail::default_constructor_tag{}) {
-    if (rhs.has_value()) {
-      this->construct(std::move(*rhs));
-    } else {
-      this->construct_error(std::move(rhs.error()));        
-    }            
-  }
-
-  /// \exclude
-  template <
-      class U, class G,
-      detail::enable_if_t<(std::is_convertible<U &&, T>::value &&
-                           std::is_convertible<G &&, E>::value)> * = nullptr,
-      detail::expected_enable_from_other<T, E, U, G, U &&, G &&> * = nullptr>
-  TL_EXPECTED_11_CONSTEXPR expected(expected<U, G> &&rhs)
-      : ctor_base(detail::default_constructor_tag{}) {
-    if (rhs.has_value()) {
-      this->construct(std::move(*rhs));
-    } else {
-      this->construct_error(std::move(rhs.error()));        
-    }                  
-  }
-
-  template <
-      class U = T,
-      detail::enable_if_t<!std::is_convertible<U &&, T>::value> * = nullptr,
-      detail::expected_enable_forward_value<T, E, U> * = nullptr>
-  explicit TL_EXPECTED_MSVC2015_CONSTEXPR expected(U &&v)
-      : expected(in_place, std::forward<U>(v)) {}
-
-  /// \exclude
-  template <
-      class U = T,
-      detail::enable_if_t<std::is_convertible<U &&, T>::value> * = nullptr,
-      detail::expected_enable_forward_value<T, E, U> * = nullptr>
-  TL_EXPECTED_MSVC2015_CONSTEXPR expected(U &&v)
-      : expected(in_place, std::forward<U>(v)) {}
-
-  template <
-      class U = T, class G = T,
-      detail::enable_if_t<std::is_nothrow_constructible<T, U &&>::value> * =
-          nullptr,
-      detail::enable_if_t<!std::is_void<G>::value> * = nullptr,
-      detail::enable_if_t<
-          (!std::is_same<expected<T, E>, detail::decay_t<U>>::value &&
-           !detail::conjunction<std::is_scalar<T>,
-                                std::is_same<T, detail::decay_t<U>>>::value &&
-           std::is_constructible<T, U>::value &&
-           std::is_assignable<G &, U>::value &&
-           std::is_nothrow_move_constructible<E>::value)> * = nullptr>
-  expected &operator=(U &&v) {
-    if (has_value()) {
-      val() = std::forward<U>(v);
-    } else {
-      err().~unexpected<E>();
-      ::new (valptr()) T(std::forward<U>(v));
-      this->m_has_val = true;
-    }
-
-    return *this;
-  }
-
-  /// \exclude
-  template <
-      class U = T, class G = T,
-      detail::enable_if_t<!std::is_nothrow_constructible<T, U &&>::value> * =
-          nullptr,
-      detail::enable_if_t<!std::is_void<U>::value> * = nullptr,
-      detail::enable_if_t<
-          (!std::is_same<expected<T, E>, detail::decay_t<U>>::value &&
-           !detail::conjunction<std::is_scalar<T>,
-                                std::is_same<T, detail::decay_t<U>>>::value &&
-           std::is_constructible<T, U>::value &&
-           std::is_assignable<G &, U>::value &&
-           std::is_nothrow_move_constructible<E>::value)> * = nullptr>
-  expected &operator=(U &&v) {
-    if (has_value()) {
-      val() = std::forward<U>(v);
-    } else {
-      auto tmp = std::move(err());
-      err().~unexpected<E>();
-
-      #ifdef TL_EXPECTED_EXCEPTIONS_ENABLED
-      try {
-        ::new (valptr()) T(std::move(v));
-        this->m_has_val = true;
-      } catch (...) {
-        err() = std::move(tmp);
-        throw;
-      }
-      #else
-        ::new (valptr()) T(std::move(v));
-        this->m_has_val = true;
-      #endif
-    }
-
-    return *this;
-  }
-
-  template <class G = E,
-            detail::enable_if_t<std::is_nothrow_copy_constructible<G>::value &&
-                                std::is_assignable<G &, G>::value> * = nullptr>
-  expected &operator=(const unexpected<G> &rhs) {
-    if (!has_value()) {
-      err() = rhs;
-    } else {
-      this->destroy_val();
-      ::new (errptr()) unexpected<E>(rhs);
-      this->m_has_val = false;
-    }
-
-    return *this;
-  }
-
-  template <class G = E,
-            detail::enable_if_t<std::is_nothrow_move_constructible<G>::value &&
-                                std::is_move_assignable<G>::value> * = nullptr>
-  expected &operator=(unexpected<G> &&rhs) noexcept {
-    if (!has_value()) {
-      err() = std::move(rhs);
-    } else {
-      this->destroy_val();
-      ::new (errptr()) unexpected<E>(std::move(rhs));
-      this->m_has_val = false;
-    }
-
-    return *this;
-  }
-
-  template <class... Args, detail::enable_if_t<std::is_nothrow_constructible<
-                               T, Args &&...>::value> * = nullptr>
-  void emplace(Args &&... args) {
-    if (has_value()) {
-      val() = T(std::forward<Args>(args)...);
-    } else {
-      err().~unexpected<E>();
-      ::new (valptr()) T(std::forward<Args>(args)...);
-      this->m_has_val = true;
-    }
-  }
-
-  /// \exclude
-  template <class... Args, detail::enable_if_t<!std::is_nothrow_constructible<
-                               T, Args &&...>::value> * = nullptr>
-  void emplace(Args &&... args) {
-    if (has_value()) {
-      val() = T(std::forward<Args>(args)...);
-    } else {
-      auto tmp = std::move(err());
-      err().~unexpected<E>();
-
-      #ifdef TL_EXPECTED_EXCEPTIONS_ENABLED
-      try {
-        ::new (valptr()) T(std::forward<Args>(args)...);
-        this->m_has_val = true;
-      } catch (...) {
-        err() = std::move(tmp);
-        throw;
-      }
-      #else
-      ::new (valptr()) T(std::forward<Args>(args)...);
-      this->m_has_val = true;
-      #endif
-    }
-  }
-
-  template <class U, class... Args,
-            detail::enable_if_t<std::is_nothrow_constructible<
-                T, std::initializer_list<U> &, Args &&...>::value> * = nullptr>
-  void emplace(std::initializer_list<U> il, Args &&... args) {
-    if (has_value()) {
-      T t(il, std::forward<Args>(args)...);
-      val() = std::move(t);
-    } else {
-      err().~unexpected<E>();
-      ::new (valptr()) T(il, std::forward<Args>(args)...);
-      this->m_has_val = true;
-    }
-  }
-
-  /// \exclude
-  template <class U, class... Args,
-            detail::enable_if_t<!std::is_nothrow_constructible<
-                T, std::initializer_list<U> &, Args &&...>::value> * = nullptr>
-  void emplace(std::initializer_list<U> il, Args &&... args) {
-    if (has_value()) {
-      T t(il, std::forward<Args>(args)...);
-      val() = std::move(t);
-    } else {
-      auto tmp = std::move(err());
-      err().~unexpected<E>();
-
-      #ifdef TL_EXPECTED_EXCEPTIONS_ENABLED
-      try {
-        ::new (valptr()) T(il, std::forward<Args>(args)...);
-        this->m_has_val = true;
-      } catch (...) {
-        err() = std::move(tmp);
-        throw;
-      }
-      #else
-      ::new (valptr()) T(il, std::forward<Args>(args)...);
-      this->m_has_val = true;
-      #endif
-    }
-  }
-
-  // TODO SFINAE
-  void swap(expected &rhs) noexcept(
-      std::is_nothrow_move_constructible<T>::value &&noexcept(
-          swap(std::declval<T &>(), std::declval<T &>())) &&
-      std::is_nothrow_move_constructible<E>::value &&
-      noexcept(swap(std::declval<E &>(), std::declval<E &>()))) {
-    if (has_value() && rhs.has_value()) {
-      using std::swap;
-      swap(val(), rhs.val());
-    } else if (!has_value() && rhs.has_value()) {
-      using std::swap;
-      swap(err(), rhs.err());
-    } else if (has_value()) {
-      auto temp = std::move(rhs.err());
-      ::new (rhs.valptr()) T(val());
-      ::new (errptr()) unexpected_type(std::move(temp));
-      std::swap(this->m_has_val, rhs.m_has_val);
-    } else {
-      auto temp = std::move(this->err());
-      ::new (valptr()) T(rhs.val());
-      ::new (errptr()) unexpected_type(std::move(temp));
-      std::swap(this->m_has_val, rhs.m_has_val);
-    }
-  }
-
-  /// \returns a pointer to the stored value
-  /// \requires a value is stored
-  /// \group pointer
-  constexpr const T *operator->() const { return valptr(); }
-  /// \group pointer
-  TL_EXPECTED_11_CONSTEXPR T *operator->() { return valptr(); }
-
-  /// \returns the stored value
-  /// \requires a value is stored
-  /// \group deref
-  template <class U = T,
-            detail::enable_if_t<!std::is_void<U>::value> * = nullptr>
-  constexpr const U &operator*() const & {
-    return val();
-  }
-  /// \group deref
-  template <class U = T,
-            detail::enable_if_t<!std::is_void<U>::value> * = nullptr>
-  TL_EXPECTED_11_CONSTEXPR U &operator*() & {
-    return val();
-  }
-  /// \group deref
-  template <class U = T,
-            detail::enable_if_t<!std::is_void<U>::value> * = nullptr>
-  constexpr const U &&operator*() const && {
-    return std::move(val());
-  }
-  /// \group deref
-  template <class U = T,
-            detail::enable_if_t<!std::is_void<U>::value> * = nullptr>
-  TL_EXPECTED_11_CONSTEXPR U &&operator*() && {
-    return std::move(val());
-  }
-
-  /// \returns whether or not the optional has a value
-  /// \group has_value
-  constexpr bool has_value() const noexcept { return this->m_has_val; }
-  /// \group has_value
-  constexpr explicit operator bool() const noexcept { return this->m_has_val; }
-
-  /// \returns the contained value if there is one, otherwise throws
-  /// [bad_expected_access]
-  ///
-  /// \group value
-  template <class U = T,
-            detail::enable_if_t<!std::is_void<U>::value> * = nullptr>
-  TL_EXPECTED_11_CONSTEXPR const U &value() const & {
-    if (!has_value())
-      detail::throw_exception(bad_expected_access<E>(err().value()));
-    return val();
-  }
-  /// \group value
-  template <class U = T,
-            detail::enable_if_t<!std::is_void<U>::value> * = nullptr>
-  TL_EXPECTED_11_CONSTEXPR U &value() & {
-    if (!has_value())
-      detail::throw_exception(bad_expected_access<E>(err().value()));
-    return val();
-  }
-  /// \group value
-  template <class U = T,
-            detail::enable_if_t<!std::is_void<U>::value> * = nullptr>
-  TL_EXPECTED_11_CONSTEXPR const U &&value() const && {
-    if (!has_value())
-      detail::throw_exception(bad_expected_access<E>(err().value()));
-    return std::move(val());
-  }
-  /// \group value
-  template <class U = T,
-            detail::enable_if_t<!std::is_void<U>::value> * = nullptr>
-  TL_EXPECTED_11_CONSTEXPR U &&value() && {
-    if (!has_value())
-      detail::throw_exception(bad_expected_access<E>(err().value()));
-    return std::move(val());
-  }
-
-  /// \returns the unexpected value
-  /// \requires there is an unexpected value
-  /// \group error
-  constexpr const E &error() const & { return err().value(); }
-  /// \group error
-  TL_EXPECTED_11_CONSTEXPR E &error() & { return err().value(); }
-  /// \group error
-  constexpr const E &&error() const && { return std::move(err().value()); }
-  /// \group error
-  TL_EXPECTED_11_CONSTEXPR E &&error() && { return std::move(err().value()); }
-
-  /// \returns the stored value if there is one, otherwise returns `u`
-  /// \group value_or
-  template <class U> constexpr T value_or(U &&v) const & {
-    static_assert(std::is_copy_constructible<T>::value &&
-                      std::is_convertible<U &&, T>::value,
-                  "T must be copy-constructible and convertible to from U&&");
-    return bool(*this) ? **this : static_cast<T>(std::forward<U>(v));
-  }
-  /// \group value_or
-  template <class U> TL_EXPECTED_11_CONSTEXPR T value_or(U &&v) && {
-    static_assert(std::is_move_constructible<T>::value &&
-                      std::is_convertible<U &&, T>::value,
-                  "T must be move-constructible and convertible to from U&&");
-    return bool(*this) ? std::move(**this) : static_cast<T>(std::forward<U>(v));
-  }
-};
-
-/// \exclude
-namespace detail {
-template <class Exp> using exp_t = typename detail::decay_t<Exp>::value_type;
-template <class Exp> using err_t = typename detail::decay_t<Exp>::error_type;
-template <class Exp, class Ret> using ret_t = expected<Ret, err_t<Exp>>;
-
-#ifdef TL_EXPECTED_CXX14
-template <class Exp, class F,
-          detail::enable_if_t<!std::is_void<exp_t<Exp>>::value> * = nullptr,
-          class Ret = decltype(detail::invoke(std::declval<F>(),
-                                              *std::declval<Exp>()))>
-constexpr auto and_then_impl(Exp &&exp, F &&f) {
-  static_assert(detail::is_expected<Ret>::value, "F must return an expected");
-
-  return exp.has_value()
-             ? detail::invoke(std::forward<F>(f), *std::forward<Exp>(exp))
-             : Ret(unexpect, exp.error());
-}
-
-template <class Exp, class F,
-          detail::enable_if_t<std::is_void<exp_t<Exp>>::value> * = nullptr,
-          class Ret = decltype(detail::invoke(std::declval<F>()))>
-constexpr auto and_then_impl(Exp &&exp, F &&f) {
-  static_assert(detail::is_expected<Ret>::value, "F must return an expected");
-
-  return exp.has_value() ? detail::invoke(std::forward<F>(f))
-                         : Ret(unexpect, exp.error());
-}
-#else
-template <class> struct TC;
-template <class Exp, class F,
-          class Ret = decltype(detail::invoke(std::declval<F>(),
-                                              *std::declval<Exp>())),
-          detail::enable_if_t<!std::is_void<exp_t<Exp>>::value> * = nullptr>
-auto and_then_impl(Exp &&exp, F &&f) -> Ret {
-  static_assert(detail::is_expected<Ret>::value, "F must return an expected");
-
-  return exp.has_value()
-             ? detail::invoke(std::forward<F>(f), *std::forward<Exp>(exp))
-             : Ret(unexpect, exp.error());
-}
-
-template <class Exp, class F,
-          class Ret = decltype(detail::invoke(std::declval<F>())),
-          detail::enable_if_t<std::is_void<exp_t<Exp>>::value> * = nullptr>
-constexpr auto and_then_impl(Exp &&exp, F &&f) -> Ret {
-  static_assert(detail::is_expected<Ret>::value, "F must return an expected");
-
-  return exp.has_value() ? detail::invoke(std::forward<F>(f))
-                         : Ret(unexpect, exp.error());
-}
-#endif
-
-#ifdef TL_EXPECTED_CXX14
-template <class Exp, class F,
-          detail::enable_if_t<!std::is_void<exp_t<Exp>>::value> * = nullptr,          
-          class Ret = decltype(detail::invoke(std::declval<F>(),
-                                              *std::declval<Exp>())),
-          detail::enable_if_t<!std::is_void<Ret>::value> * = nullptr>
-constexpr auto expected_map_impl(Exp &&exp, F &&f) {
-  using result = ret_t<Exp, detail::decay_t<Ret>>;
-  return exp.has_value() ? result(detail::invoke(std::forward<F>(f),
-                                                 *std::forward<Exp>(exp)))
-                         : result(unexpect, std::forward<Exp>(exp).error());
-}
-
-template <class Exp, class F,
-          detail::enable_if_t<!std::is_void<exp_t<Exp>>::value> * = nullptr,          
-          class Ret = decltype(detail::invoke(std::declval<F>(),
-                                              *std::declval<Exp>())),
-          detail::enable_if_t<std::is_void<Ret>::value> * = nullptr>
-auto expected_map_impl(Exp &&exp, F &&f) {
-  using result = expected<void, err_t<Exp>>;
-  if (exp.has_value()) {
-    detail::invoke(std::forward<F>(f), *std::forward<Exp>(exp));
-    return result();
-  }
-
-  return result(unexpect, std::forward<Exp>(exp).error());
-}
-
-template <class Exp, class F,
-          detail::enable_if_t<std::is_void<exp_t<Exp>>::value> * = nullptr,
-          class Ret = decltype(detail::invoke(std::declval<F>())),
-          detail::enable_if_t<!std::is_void<Ret>::value> * = nullptr>
-constexpr auto expected_map_impl(Exp &&exp, F &&f) {
-  using result = ret_t<Exp, detail::decay_t<Ret>>;
-  return exp.has_value() ? result(detail::invoke(std::forward<F>(f)))
-                         : result(unexpect, std::forward<Exp>(exp).error());
-}
-
-template <class Exp, class F,
-          detail::enable_if_t<std::is_void<exp_t<Exp>>::value> * = nullptr,          
-          class Ret = decltype(detail::invoke(std::declval<F>())),
-          detail::enable_if_t<std::is_void<Ret>::value> * = nullptr>
-auto expected_map_impl(Exp &&exp, F &&f) {
-  using result = expected<void, err_t<Exp>>;
-  if (exp.has_value()) {
-    detail::invoke(std::forward<F>(f));
-    return result();
-  }
-
-  return result(unexpect, std::forward<Exp>(exp).error());
-}    
-#else
-template <class Exp, class F,
-          detail::enable_if_t<!std::is_void<exp_t<Exp>>::value> * = nullptr,          
-          class Ret = decltype(detail::invoke(std::declval<F>(),
-                                              *std::declval<Exp>())),
-          detail::enable_if_t<!std::is_void<Ret>::value> * = nullptr>
-
-constexpr auto expected_map_impl(Exp &&exp, F &&f)
-    -> ret_t<Exp, detail::decay_t<Ret>> {
-  using result = ret_t<Exp, detail::decay_t<Ret>>;
-
-  return exp.has_value() ? result(detail::invoke(std::forward<F>(f),
-                                                 *std::forward<Exp>(exp)))
-                         : result(unexpect, std::forward<Exp>(exp).error());
-}
-
-template <class Exp, class F,
-          detail::enable_if_t<!std::is_void<exp_t<Exp>>::value> * = nullptr,                    
-          class Ret = decltype(detail::invoke(std::declval<F>(),
-                                              *std::declval<Exp>())),
-          detail::enable_if_t<std::is_void<Ret>::value> * = nullptr>
-
-auto expected_map_impl(Exp &&exp, F &&f) -> expected<void, err_t<Exp>> {
-  if (exp.has_value()) {
-    detail::invoke(std::forward<F>(f), *std::forward<Exp>(exp));
-    return {};
-  }
-
-  return unexpected<err_t<Exp>>(std::forward<Exp>(exp).error());
-}
-
-template <class Exp, class F,
-          detail::enable_if_t<std::is_void<exp_t<Exp>>::value> * = nullptr,                              
-          class Ret = decltype(detail::invoke(std::declval<F>())),
-          detail::enable_if_t<!std::is_void<Ret>::value> * = nullptr>
-
-constexpr auto expected_map_impl(Exp &&exp, F &&f)
-    -> ret_t<Exp, detail::decay_t<Ret>> {
-  using result = ret_t<Exp, detail::decay_t<Ret>>;
-
-  return exp.has_value() ? result(detail::invoke(std::forward<F>(f)))
-                         : result(unexpect, std::forward<Exp>(exp).error());
-}
-
-template <class Exp, class F,
-          detail::enable_if_t<std::is_void<exp_t<Exp>>::value> * = nullptr,                                        
-          class Ret = decltype(detail::invoke(std::declval<F>())),
-          detail::enable_if_t<std::is_void<Ret>::value> * = nullptr>
-
-auto expected_map_impl(Exp &&exp, F &&f) -> expected<void, err_t<Exp>> {
-  if (exp.has_value()) {
-    detail::invoke(std::forward<F>(f));
-    return {};
-  }
-
-  return unexpected<err_t<Exp>>(std::forward<Exp>(exp).error());
-}    
-#endif
-
-#if defined(TL_EXPECTED_CXX14) && !defined(TL_EXPECTED_GCC49) &&               \
-    !defined(TL_EXPECTED_GCC54) && !defined(TL_EXPECTED_GCC55)
-template <class Exp, class F,
-          detail::enable_if_t<!std::is_void<exp_t<Exp>>::value> * = nullptr,          
-          class Ret = decltype(detail::invoke(std::declval<F>(),
-                                              std::declval<Exp>().error())),
-          detail::enable_if_t<!std::is_void<Ret>::value> * = nullptr>
-constexpr auto map_error_impl(Exp &&exp, F &&f) {
-  using result = expected<exp_t<Exp>, detail::decay_t<Ret>>;
-  return exp.has_value()
-             ? result(*std::forward<Exp>(exp))
-             : result(unexpect, detail::invoke(std::forward<F>(f),
-                                               std::forward<Exp>(exp).error()));
-}
-template <class Exp, class F,
-          detail::enable_if_t<!std::is_void<exp_t<Exp>>::value> * = nullptr,                    
-          class Ret = decltype(detail::invoke(std::declval<F>(),
-                                              std::declval<Exp>().error())),
-          detail::enable_if_t<std::is_void<Ret>::value> * = nullptr>
-auto map_error_impl(Exp &&exp, F &&f) {
-  using result = expected<exp_t<Exp>, monostate>;
-  if (exp.has_value()) {
-    return result(*std::forward<Exp>(exp));
-  }
-
-  detail::invoke(std::forward<F>(f), std::forward<Exp>(exp).error());
-  return result(unexpect, monostate{});
-}
-template <class Exp, class F,
-          detail::enable_if_t<std::is_void<exp_t<Exp>>::value> * = nullptr,          
-          class Ret = decltype(detail::invoke(std::declval<F>(),
-                                              std::declval<Exp>().error())),
-          detail::enable_if_t<!std::is_void<Ret>::value> * = nullptr>
-constexpr auto map_error_impl(Exp &&exp, F &&f) {
-  using result = expected<exp_t<Exp>, detail::decay_t<Ret>>;
-  return exp.has_value()
-             ? result()
-             : result(unexpect, detail::invoke(std::forward<F>(f),
-                                               std::forward<Exp>(exp).error()));
-}
-template <class Exp, class F,
-          detail::enable_if_t<std::is_void<exp_t<Exp>>::value> * = nullptr,                    
-          class Ret = decltype(detail::invoke(std::declval<F>(),
-                                              std::declval<Exp>().error())),
-          detail::enable_if_t<std::is_void<Ret>::value> * = nullptr>
-auto map_error_impl(Exp &&exp, F &&f) {
-  using result = expected<exp_t<Exp>, monostate>;
-  if (exp.has_value()) {
-    return result();
-  }
-
-  detail::invoke(std::forward<F>(f), std::forward<Exp>(exp).error());
-  return result(unexpect, monostate{});
-}    
-#else
-template <class Exp, class F,
-          detail::enable_if_t<!std::is_void<exp_t<Exp>>::value> * = nullptr,                              
-          class Ret = decltype(detail::invoke(std::declval<F>(),
-                                              std::declval<Exp>().error())),
-          detail::enable_if_t<!std::is_void<Ret>::value> * = nullptr>
-constexpr auto map_error_impl(Exp &&exp, F &&f)
-    -> expected<exp_t<Exp>, detail::decay_t<Ret>> {
-  using result = expected<exp_t<Exp>, detail::decay_t<Ret>>;
-
-  return exp.has_value()
-             ? result(*std::forward<Exp>(exp))
-             : result(unexpect, detail::invoke(std::forward<F>(f),
-                                               std::forward<Exp>(exp).error()));
-}
-
-template <class Exp, class F,
-          detail::enable_if_t<!std::is_void<exp_t<Exp>>::value> * = nullptr,                                        
-          class Ret = decltype(detail::invoke(std::declval<F>(),
-                                              std::declval<Exp>().error())),
-          detail::enable_if_t<std::is_void<Ret>::value> * = nullptr>
-auto map_error_impl(Exp &&exp, F &&f) -> expected<exp_t<Exp>, monostate> {
-  using result = expected<exp_t<Exp>, monostate>;
-  if (exp.has_value()) {
-    return result(*std::forward<Exp>(exp));
-  }
-
-  detail::invoke(std::forward<F>(f), std::forward<Exp>(exp).error());
-  return result(unexpect, monostate{});
-}
-
-template <class Exp, class F,
-          detail::enable_if_t<std::is_void<exp_t<Exp>>::value> * = nullptr,                              
-          class Ret = decltype(detail::invoke(std::declval<F>(),
-                                              std::declval<Exp>().error())),
-          detail::enable_if_t<!std::is_void<Ret>::value> * = nullptr>
-constexpr auto map_error_impl(Exp &&exp, F &&f)
-    -> expected<exp_t<Exp>, detail::decay_t<Ret>> {
-  using result = expected<exp_t<Exp>, detail::decay_t<Ret>>;
-
-  return exp.has_value()
-             ? result()
-             : result(unexpect, detail::invoke(std::forward<F>(f),
-                                               std::forward<Exp>(exp).error()));
-}
-
-template <class Exp, class F,
-          detail::enable_if_t<std::is_void<exp_t<Exp>>::value> * = nullptr,                                        
-          class Ret = decltype(detail::invoke(std::declval<F>(),
-                                              std::declval<Exp>().error())),
-          detail::enable_if_t<std::is_void<Ret>::value> * = nullptr>
-auto map_error_impl(Exp &&exp, F &&f) -> expected<exp_t<Exp>, monostate> {
-  using result = expected<exp_t<Exp>, monostate>;
-  if (exp.has_value()) {
-    return result();
-  }
-
-  detail::invoke(std::forward<F>(f), std::forward<Exp>(exp).error());
-  return result(unexpect, monostate{});
-}    
-#endif
-
-#ifdef TL_EXPECTED_CXX14
-template <class Exp, class F,
-          class Ret = decltype(detail::invoke(std::declval<F>(),
-                                              std::declval<Exp>().error())),
-          detail::enable_if_t<!std::is_void<Ret>::value> * = nullptr>
-constexpr auto or_else_impl(Exp &&exp, F &&f) {
-  static_assert(detail::is_expected<Ret>::value, "F must return an expected");
-  return exp.has_value()
-  ? std::forward<Exp>(exp)
-  : detail::invoke(std::forward<F>(f), std::forward<Exp>(exp).error());
-}
-
-template <class Exp, class F,
-          class Ret = decltype(detail::invoke(std::declval<F>(),
-                                              std::declval<Exp>().error())),
-          detail::enable_if_t<std::is_void<Ret>::value> * = nullptr>
-detail::decay_t<Exp> or_else_impl(Exp &&exp, F &&f) {
-  return exp.has_value()
-  ? std::forward<Exp>(exp)
-  : (detail::invoke(std::forward<F>(f), std::forward<Exp>(exp).error()),
-    std::forward<Exp>(exp));
-}
-#else
-template <class Exp, class F,
-          class Ret = decltype(detail::invoke(std::declval<F>(),
-                                              std::declval<Exp>().error())),
-  detail::enable_if_t<!std::is_void<Ret>::value> * = nullptr>
-auto or_else_impl(Exp &&exp, F &&f) -> Ret {
-  static_assert(detail::is_expected<Ret>::value, "F must return an expected");
-  return exp.has_value()
-         ? std::forward<Exp>(exp)
-         : detail::invoke(std::forward<F>(f), std::forward<Exp>(exp).error());
-}
-
-template <class Exp, class F,
-          class Ret = decltype(detail::invoke(std::declval<F>(),
-                                              std::declval<Exp>().error())),
-  detail::enable_if_t<std::is_void<Ret>::value> * = nullptr>
-detail::decay_t<Exp> or_else_impl(Exp &&exp, F &&f) {
-  return exp.has_value()
-         ? std::forward<Exp>(exp)
-         : (detail::invoke(std::forward<F>(f), std::forward<Exp>(exp).error()),
-            std::forward<Exp>(exp));
-}
-#endif
-} // namespace detail
-
-template <class T, class E, class U, class F>
-constexpr bool operator==(const expected<T, E> &lhs,
-                          const expected<U, F> &rhs) {
-  return (lhs.has_value() != rhs.has_value())
-             ? false
-             : (!lhs.has_value() ? lhs.error() == rhs.error() : *lhs == *rhs);
-}
-template <class T, class E, class U, class F>
-constexpr bool operator!=(const expected<T, E> &lhs,
-                          const expected<U, F> &rhs) {
-  return (lhs.has_value() != rhs.has_value())
-             ? true
-             : (!lhs.has_value() ? lhs.error() != rhs.error() : *lhs != *rhs);
-}
-
-template <class T, class E, class U>
-constexpr bool operator==(const expected<T, E> &x, const U &v) {
-  return x.has_value() ? *x == v : false;
-}
-template <class T, class E, class U>
-constexpr bool operator==(const U &v, const expected<T, E> &x) {
-  return x.has_value() ? *x == v : false;
-}
-template <class T, class E, class U>
-constexpr bool operator!=(const expected<T, E> &x, const U &v) {
-  return x.has_value() ? *x != v : true;
-}
-template <class T, class E, class U>
-constexpr bool operator!=(const U &v, const expected<T, E> &x) {
-  return x.has_value() ? *x != v : true;
-}
-
-template <class T, class E>
-constexpr bool operator==(const expected<T, E> &x, const unexpected<E> &e) {
-  return x.has_value() ? false : x.error() == e.value();
-}
-template <class T, class E>
-constexpr bool operator==(const unexpected<E> &e, const expected<T, E> &x) {
-  return x.has_value() ? false : x.error() == e.value();
-}
-template <class T, class E>
-constexpr bool operator!=(const expected<T, E> &x, const unexpected<E> &e) {
-  return x.has_value() ? true : x.error() != e.value();
-}
-template <class T, class E>
-constexpr bool operator!=(const unexpected<E> &e, const expected<T, E> &x) {
-  return x.has_value() ? true : x.error() != e.value();
-}
-
-// TODO is_swappable
-template <class T, class E,
-          detail::enable_if_t<std::is_move_constructible<T>::value &&
-                              std::is_move_constructible<E>::value> * = nullptr>
-void swap(expected<T, E> &lhs,
-          expected<T, E> &rhs) noexcept(noexcept(lhs.swap(rhs))) {
-  lhs.swap(rhs);
-}
-} // namespace tl
-
-#define TL_OPTIONAL_EXPECTED_MUTEX
-#endif
diff --git a/src/include/object.h b/src/include/object.h
index 96951e74de7..b0f21bc4558 100644
--- a/src/include/object.h
+++ b/src/include/object.h
@@ -17,11 +17,10 @@
 
 #include <cstdint>
 #include <cstdio>
-#include <iomanip>
 #include <iosfwd>
+#include <iomanip>
 #include <string>
-#include <string>
-#include <string_view>
+
 
 #include "include/rados.h"
 #include "include/unordered_map.h"
@@ -41,8 +40,6 @@ struct object_t {
   object_t(const char *s) : name(s) {}
   // cppcheck-suppress noExplicitConstructor
   object_t(const std::string& s) : name(s) {}
-  object_t(std::string&& s) : name(std::move(s)) {}
-  object_t(std::string_view s) : name(s) {}
 
   void swap(object_t& o) {
     name.swap(o.name);
diff --git a/src/libcephfs.cc b/src/libcephfs.cc
index 8cde7fb3219..8a6b14d1ed8 100644
--- a/src/libcephfs.cc
+++ b/src/libcephfs.cc
@@ -20,7 +20,6 @@
 #include "auth/Crypto.h"
 #include "client/Client.h"
 #include "librados/RadosClient.h"
-#include "common/async/context_pool.h"
 #include "common/ceph_argparse.h"
 #include "common/common_init.h"
 #include "common/config.h"
@@ -37,7 +36,6 @@
 #define DEFAULT_UMASK 002
 
 static mode_t umask_cb(void *);
-ceph::async::io_context_pool icp;
 
 struct ceph_mount_info
 {
@@ -85,9 +83,8 @@ public:
       cct->_log->start();
     }
 
-    icp.start(cct->_conf.get_val<std::uint64_t>("client_asio_thread_count"));
     {
-      MonClient mc_bootstrap(cct, icp);
+      MonClient mc_bootstrap(cct);
       ret = mc_bootstrap.get_monmap_and_config();
       if (ret < 0)
 	return ret;
@@ -96,7 +93,7 @@ public:
     common_init_finish(cct);
 
     //monmap
-    monclient = new MonClient(cct, icp);
+    monclient = new MonClient(cct);
     ret = -CEPHFS_ERROR_MON_MAP_BUILD; //defined in libcephfs.h;
     if (monclient->build_initial_monmap() < 0)
       goto fail;
@@ -106,7 +103,7 @@ public:
 
     //at last the client
     ret = -CEPHFS_ERROR_NEW_CLIENT; //defined in libcephfs.h;
-    client = new StandaloneClient(messenger, monclient, icp);
+    client = new StandaloneClient(messenger, monclient);
     if (!client)
       goto fail;
 
@@ -205,7 +202,6 @@ public:
       delete messenger;
       messenger = nullptr;
     }
-    icp.stop();
     if (monclient) {
       delete monclient;
       monclient = nullptr;
diff --git a/src/librados/AioCompletionImpl.h b/src/librados/AioCompletionImpl.h
index 7b49bc68d73..d3a674e8ee7 100644
--- a/src/librados/AioCompletionImpl.h
+++ b/src/librados/AioCompletionImpl.h
@@ -126,14 +126,14 @@ struct librados::AioCompletionImpl {
 };
 
 namespace librados {
-struct CB_AioComplete {
+struct C_AioComplete : public Context {
   AioCompletionImpl *c;
 
-  explicit CB_AioComplete(AioCompletionImpl *cc) : c(cc) {
+  explicit C_AioComplete(AioCompletionImpl *cc) : c(cc) {
     c->_get();
   }
 
-  void operator()() {
+  void finish(int r) override {
     rados_callback_t cb_complete = c->callback_complete;
     void *cb_complete_arg = c->callback_complete_arg;
     if (cb_complete)
@@ -160,14 +160,14 @@ struct CB_AioComplete {
   * flush where we only want to wait for things to be safe,
   * but allow users to specify any of the callbacks.
   */
-struct CB_AioCompleteAndSafe {
+struct C_AioCompleteAndSafe : public Context {
   AioCompletionImpl *c;
 
-  explicit CB_AioCompleteAndSafe(AioCompletionImpl *cc) : c(cc) {
+  explicit C_AioCompleteAndSafe(AioCompletionImpl *cc) : c(cc) {
     c->get();
   }
 
-  void operator()(int r = 0) {
+  void finish(int r) override {
     c->lock.lock();
     c->rval = r;
     c->complete = true;
@@ -190,6 +190,7 @@ struct CB_AioCompleteAndSafe {
     c->put_unlock();
   }
 };
+
 }
 
 #endif
diff --git a/src/librados/IoCtxImpl.cc b/src/librados/IoCtxImpl.cc
index 8d3ba1e16fb..ddcd4c6fea1 100644
--- a/src/librados/IoCtxImpl.cc
+++ b/src/librados/IoCtxImpl.cc
@@ -28,38 +28,34 @@
 #undef dout_prefix
 #define dout_prefix *_dout << "librados: "
 
-namespace bs = boost::system;
-namespace ca = ceph::async;
-namespace cb = ceph::buffer;
-
 namespace librados {
 namespace {
 
-struct CB_notify_Finish {
+struct C_notify_Finish : public Context {
   CephContext *cct;
   Context *ctx;
   Objecter *objecter;
   Objecter::LingerOp *linger_op;
+  bufferlist reply_bl;
   bufferlist *preply_bl;
   char **preply_buf;
   size_t *preply_buf_len;
 
-  CB_notify_Finish(CephContext *_cct, Context *_ctx, Objecter *_objecter,
-		   Objecter::LingerOp *_linger_op, bufferlist *_preply_bl,
-		   char **_preply_buf, size_t *_preply_buf_len)
+  C_notify_Finish(CephContext *_cct, Context *_ctx, Objecter *_objecter,
+                  Objecter::LingerOp *_linger_op, bufferlist *_preply_bl,
+                  char **_preply_buf, size_t *_preply_buf_len)
     : cct(_cct), ctx(_ctx), objecter(_objecter), linger_op(_linger_op),
       preply_bl(_preply_bl), preply_buf(_preply_buf),
-      preply_buf_len(_preply_buf_len) {}
-
-
-  CB_notify_Finish(const CB_notify_Finish&) = delete;
-  CB_notify_Finish operator =(const CB_notify_Finish&) = delete;
-  CB_notify_Finish(CB_notify_Finish&&) = delete;
-  CB_notify_Finish operator =(CB_notify_Finish&&) = delete;
+      preply_buf_len(_preply_buf_len)
+  {
+    linger_op->on_notify_finish = this;
+    linger_op->notify_result_bl = &reply_bl;
+  }
 
-  void operator()(bs::error_code ec, bufferlist&& reply_bl) {
+  void finish(int r) override
+  {
     ldout(cct, 10) << __func__ << " completed notify (linger op "
-                   << linger_op << "), ec = " << ec << dendl;
+                   << linger_op << "), r = " << r << dendl;
 
     // pass result back to user
     // NOTE: we do this regardless of what error code we return
@@ -76,21 +72,21 @@ struct CB_notify_Finish {
     if (preply_bl)
       preply_bl->claim(reply_bl);
 
-    ctx->complete(ceph::from_error_code(ec));
-    linger_op->on_notify_finish.reset();
+    ctx->complete(r);
   }
 };
 
-struct CB_aio_linger_cancel {
+struct C_aio_linger_cancel : public Context {
   Objecter *objecter;
   Objecter::LingerOp *linger_op;
 
-  CB_aio_linger_cancel(Objecter *_objecter, Objecter::LingerOp *_linger_op)
+  C_aio_linger_cancel(Objecter *_objecter, Objecter::LingerOp *_linger_op)
     : objecter(_objecter), linger_op(_linger_op)
   {
   }
 
-  void operator()() {
+  void finish(int r) override
+  {
     objecter->linger_cancel(linger_op);
   }
 };
@@ -108,9 +104,8 @@ struct C_aio_linger_Complete : public Context {
 
   void finish(int r) override {
     if (cancel || r < 0)
-      boost::asio::defer(c->io->client->finish_strand,
-			 CB_aio_linger_cancel(c->io->objecter,
-					      linger_op));
+      c->io->client->finisher.queue(new C_aio_linger_cancel(c->io->objecter,
+                                                            linger_op));
 
     c->lock.lock();
     c->rval = r;
@@ -119,7 +114,7 @@ struct C_aio_linger_Complete : public Context {
 
     if (c->callback_complete ||
 	c->callback_safe) {
-      boost::asio::defer(c->io->client->finish_strand, CB_AioComplete(c));
+      c->io->client->finisher.queue(new C_AioComplete(c));
     }
     c->put_unlock();
   }
@@ -166,11 +161,12 @@ struct C_aio_notify_Complete : public C_aio_linger_Complete {
 
 struct C_aio_notify_Ack : public Context {
   CephContext *cct;
+  C_notify_Finish *onfinish;
   C_aio_notify_Complete *oncomplete;
 
-  C_aio_notify_Ack(CephContext *_cct,
+  C_aio_notify_Ack(CephContext *_cct, C_notify_Finish *_onfinish,
                    C_aio_notify_Complete *_oncomplete)
-    : cct(_cct), oncomplete(_oncomplete)
+    : cct(_cct), onfinish(_onfinish), oncomplete(_oncomplete)
   {
   }
 
@@ -199,7 +195,7 @@ struct C_aio_selfmanaged_snap_op_Complete : public Context {
     c->cond.notify_all();
 
     if (c->callback_complete || c->callback_safe) {
-      boost::asio::defer(client->finish_strand, librados::CB_AioComplete(c));
+      client->finisher.queue(new librados::C_AioComplete(c));
     }
     c->put_unlock();
   }
@@ -309,7 +305,7 @@ void librados::IoCtxImpl::complete_aio_write(AioCompletionImpl *c)
     ldout(client->cct, 20) << " waking waiters on seq " << waiters->first << dendl;
     for (std::list<AioCompletionImpl*>::iterator it = waiters->second.begin();
 	 it != waiters->second.end(); ++it) {
-      boost::asio::defer(client->finish_strand, CB_AioCompleteAndSafe(*it));
+      client->finisher.queue(new C_AioCompleteAndSafe(*it));
       (*it)->put();
     }
     aio_write_waiters.erase(waiters++);
@@ -329,7 +325,7 @@ void librados::IoCtxImpl::flush_aio_writes_async(AioCompletionImpl *c)
   if (aio_write_list.empty()) {
     ldout(client->cct, 20) << "flush_aio_writes_async no writes. (tid "
 			   << seq << ")" << dendl;
-    boost::asio::defer(client->finish_strand, CB_AioCompleteAndSafe(c));
+    client->finisher.queue(new C_AioCompleteAndSafe(c));
   } else {
     ldout(client->cct, 20) << "flush_aio_writes_async " << aio_write_list.size()
 			   << " writes in flight; waiting on tid " << seq << dendl;
@@ -366,10 +362,14 @@ int librados::IoCtxImpl::snap_create(const char *snapName)
   ceph::condition_variable cond;
   bool done;
   Context *onfinish = new C_SafeCond(mylock, cond, &done, &reply);
-  objecter->create_pool_snap(poolid, sName, onfinish);
+  reply = objecter->create_pool_snap(poolid, sName, onfinish);
 
-  std::unique_lock l{mylock};
-  cond.wait(l, [&done] { return done; });
+  if (reply < 0) {
+    delete onfinish;
+  } else {
+    std::unique_lock l{mylock};
+    cond.wait(l, [&done] { return done; });
+  }
   return reply;
 }
 
@@ -382,14 +382,18 @@ int librados::IoCtxImpl::selfmanaged_snap_create(uint64_t *psnapid)
   bool done;
   Context *onfinish = new C_SafeCond(mylock, cond, &done, &reply);
   snapid_t snapid;
-  objecter->allocate_selfmanaged_snap(poolid, &snapid, onfinish);
+  reply = objecter->allocate_selfmanaged_snap(poolid, &snapid, onfinish);
 
-  {
-    std::unique_lock l{mylock};
-    cond.wait(l, [&done] { return done; });
+  if (reply < 0) {
+    delete onfinish;
+  } else {
+    {
+      std::unique_lock l{mylock};
+      cond.wait(l, [&done] { return done; });
+    }
+    if (reply == 0)
+      *psnapid = snapid;
   }
-  if (reply == 0)
-    *psnapid = snapid;
   return reply;
 }
 
@@ -398,8 +402,11 @@ void librados::IoCtxImpl::aio_selfmanaged_snap_create(uint64_t *snapid,
 {
   C_aio_selfmanaged_snap_create_Complete *onfinish =
     new C_aio_selfmanaged_snap_create_Complete(client, c, snapid);
-  objecter->allocate_selfmanaged_snap(poolid, &onfinish->snapid,
-				      onfinish);
+  int r = objecter->allocate_selfmanaged_snap(poolid, &onfinish->snapid,
+                                              onfinish);
+  if (r < 0) {
+    onfinish->complete(r);
+  }
 }
 
 int librados::IoCtxImpl::snap_remove(const char *snapName)
@@ -411,9 +418,14 @@ int librados::IoCtxImpl::snap_remove(const char *snapName)
   ceph::condition_variable cond;
   bool done;
   Context *onfinish = new C_SafeCond(mylock, cond, &done, &reply);
-  objecter->delete_pool_snap(poolid, sName, onfinish);
-  unique_lock l{mylock};
-  cond.wait(l, [&done] { return done; });
+  reply = objecter->delete_pool_snap(poolid, sName, onfinish);
+
+  if (reply < 0) {
+    delete onfinish; 
+  } else {
+    unique_lock l{mylock};
+    cond.wait(l, [&done] { return done; });
+  }
   return reply;
 }
 
@@ -1143,7 +1155,7 @@ struct AioGetxattrsData {
   AioGetxattrsData(librados::AioCompletionImpl *c, map<string, bufferlist>* attrset,
 		   librados::RadosClient *_client) :
     user_completion(c), user_attrset(attrset), client(_client) {}
-  struct librados::CB_AioCompleteAndSafe user_completion;
+  struct librados::C_AioCompleteAndSafe user_completion;
   map<string, bufferlist> result_attrset;
   map<std::string, bufferlist>* user_attrset;
   librados::RadosClient *client;
@@ -1162,7 +1174,7 @@ static void aio_getxattrs_complete(rados_completion_t c, void *arg) {
       (*cdata->user_attrset)[p->first] = p->second;
     }
   }
-  cdata->user_completion(rc);
+  cdata->user_completion.finish(rc);
   ((librados::AioCompletionImpl*)c)->put();
   delete cdata;
 }
@@ -1465,7 +1477,7 @@ int librados::IoCtxImpl::stat(const object_t& oid, uint64_t *psize, time_t *pmti
 
   ::ObjectOperation rd;
   prepare_assert_ops(&rd);
-  rd.stat(psize, &mtime, nullptr);
+  rd.stat(psize, &mtime, NULL);
   int r = operate_read(oid, &rd, NULL);
 
   if (r >= 0 && pmtime) {
@@ -1485,7 +1497,7 @@ int librados::IoCtxImpl::stat2(const object_t& oid, uint64_t *psize, struct time
 
   ::ObjectOperation rd;
   prepare_assert_ops(&rd);
-  rd.stat(psize, &mtime, nullptr);
+  rd.stat(psize, &mtime, NULL);
   int r = operate_read(oid, &rd, NULL);
   if (r < 0) {
     return r;
@@ -1556,7 +1568,7 @@ void librados::IoCtxImpl::set_sync_op_version(version_t ver)
   last_objver = ver;
 }
 
-struct WatchInfo {
+struct WatchInfo : public Objecter::WatchContext {
   librados::IoCtxImpl *ioctx;
   object_t oid;
   librados::WatchCtx *ctx;
@@ -1569,7 +1581,7 @@ struct WatchInfo {
     : ioctx(io), oid(o), ctx(c), ctx2(c2), internal(inter) {
     ioctx->get();
   }
-  ~WatchInfo() {
+  ~WatchInfo() override {
     ioctx->put();
     if (internal) {
       delete ctx;
@@ -1577,15 +1589,10 @@ struct WatchInfo {
     }
   }
 
-  WatchInfo(const WatchInfo&) = delete;
-  WatchInfo& operator =(const WatchInfo&) = delete;
-  WatchInfo(WatchInfo&&) = delete;
-  WatchInfo& operator =(WatchInfo&&) = delete;
-
   void handle_notify(uint64_t notify_id,
 		     uint64_t cookie,
 		     uint64_t notifier_id,
-		     bufferlist& bl) {
+		     bufferlist& bl) override {
     ldout(ioctx->client->cct, 10) << __func__ << " " << notify_id
 				  << " cookie " << cookie
 				  << " notifier_id " << notifier_id
@@ -1602,25 +1609,13 @@ struct WatchInfo {
       ioctx->notify_ack(oid, notify_id, cookie, empty);
     }
   }
-  void handle_error(uint64_t cookie, int err) {
+  void handle_error(uint64_t cookie, int err) override {
     ldout(ioctx->client->cct, 10) << __func__ << " cookie " << cookie
 				  << " err " << err
 				  << dendl;
     if (ctx2)
       ctx2->handle_error(cookie, err);
   }
-
-  void operator()(bs::error_code ec,
-		  uint64_t notify_id,
-		  uint64_t cookie,
-		  uint64_t notifier_id,
-		  bufferlist&& bl) {
-    if (ec) {
-      handle_error(cookie, ceph::from_error_code(ec));
-    } else {
-      handle_notify(notify_id, cookie, notifier_id, bl);
-    }
-  }
 };
 
 int librados::IoCtxImpl::watch(const object_t& oid, uint64_t *handle,
@@ -1643,15 +1638,9 @@ int librados::IoCtxImpl::watch(const object_t& oid, uint64_t *handle,
 
   Objecter::LingerOp *linger_op = objecter->linger_register(oid, oloc, 0);
   *handle = linger_op->get_cookie();
-  auto wi = std::make_unique<WatchInfo>(this, oid, ctx, ctx2, internal);
-  linger_op->handle =
-    [wi = std::move(wi)](bs::error_code ec,
-			 uint64_t notify_id,
-			 uint64_t cookie,
-			 uint64_t notifier_id,
-			 bufferlist&& bl) mutable {
-      (*wi)(ec, notify_id, cookie, notifier_id, std::move(bl));
-    };
+  linger_op->watch_context = new WatchInfo(this,
+					   oid, ctx, ctx2, internal);
+
   prepare_assert_ops(&wr);
   wr.watch(*handle, CEPH_OSD_WATCH_OP_WATCH, timeout);
   bufferlist bl;
@@ -1695,15 +1684,7 @@ int librados::IoCtxImpl::aio_watch(const object_t& oid,
 
   ::ObjectOperation wr;
   *handle = linger_op->get_cookie();
-  auto wi = std::make_unique<WatchInfo>(this, oid, ctx, ctx2, internal);
-  linger_op->handle =
-    [wi = std::move(wi)](bs::error_code ec,
-			 uint64_t notify_id,
-			 uint64_t cookie,
-			 uint64_t notifier_id,
-			 bufferlist&& bl) mutable {
-      (*wi)(ec, notify_id, cookie, notifier_id, std::move(bl));
-    };
+  linger_op->watch_context = new WatchInfo(this, oid, ctx, ctx2, internal);
 
   prepare_assert_ops(&wr);
   wr.watch(*handle, CEPH_OSD_WATCH_OP_WATCH, timeout);
@@ -1731,13 +1712,8 @@ int librados::IoCtxImpl::notify_ack(
 
 int librados::IoCtxImpl::watch_check(uint64_t cookie)
 {
-  auto linger_op = reinterpret_cast<Objecter::LingerOp*>(cookie);
-  auto r = objecter->linger_check(linger_op);
-  if (r)
-    return 1 + std::chrono::duration_cast<
-      std::chrono::milliseconds>(*r).count();
-  else
-    return ceph::from_error_code(r.error());
+  Objecter::LingerOp *linger_op = reinterpret_cast<Objecter::LingerOp*>(cookie);
+  return objecter->linger_check(linger_op);
 }
 
 int librados::IoCtxImpl::unwatch(uint64_t cookie)
@@ -1782,15 +1758,11 @@ int librados::IoCtxImpl::notify(const object_t& oid, bufferlist& bl,
   Objecter::LingerOp *linger_op = objecter->linger_register(oid, oloc, 0);
 
   C_SaferCond notify_finish_cond;
-  linger_op->on_notify_finish =
-    Objecter::LingerOp::OpComp::create(
-      objecter->service.get_executor(),
-      [c = std::make_unique<CB_notify_Finish>(client->cct, &notify_finish_cond,
-					      objecter, linger_op, preply_bl,
-					      preply_buf, preply_buf_len)]
-      (bs::error_code ec, cb::list bl) {
-	std::move(*c)(ec, std::move(bl));
-      });
+  Context *notify_finish = new C_notify_Finish(client->cct, &notify_finish_cond,
+                                               objecter, linger_op, preply_bl,
+                                               preply_buf, preply_buf_len);
+  (void) notify_finish;
+
   uint32_t timeout = notify_timeout;
   if (timeout_ms)
     timeout = timeout_ms / 1000;
@@ -1840,17 +1812,11 @@ int librados::IoCtxImpl::aio_notify(const object_t& oid, AioCompletionImpl *c,
   c->io = this;
 
   C_aio_notify_Complete *oncomplete = new C_aio_notify_Complete(c, linger_op);
-  linger_op->on_notify_finish =
-    Objecter::LingerOp::OpComp::create(
-      objecter->service.get_executor(),
-      [c = std::make_unique<CB_notify_Finish>(client->cct, oncomplete,
-					      objecter, linger_op,
-					      preply_bl, preply_buf,
-					      preply_buf_len)]
-      (bs::error_code ec, cb::list&& bl) {
-	std::move(*c)(ec, std::move(bl));
-      });
-  Context *onack = new C_aio_notify_Ack(client->cct, oncomplete);
+  C_notify_Finish *onnotify = new C_notify_Finish(client->cct, oncomplete,
+                                                  objecter, linger_op,
+                                                  preply_bl, preply_buf,
+                                                  preply_buf_len);
+  Context *onack = new C_aio_notify_Ack(client->cct, onnotify, oncomplete);
 
   uint32_t timeout = notify_timeout;
   if (timeout_ms)
@@ -1934,7 +1900,7 @@ void librados::IoCtxImpl::C_aio_stat_Ack::finish(int r)
   }
 
   if (c->callback_complete) {
-    boost::asio::defer(c->io->client->finish_strand, CB_AioComplete(c));
+    c->io->client->finisher.queue(new C_AioComplete(c));
   }
 
   c->put_unlock();
@@ -1962,7 +1928,7 @@ void librados::IoCtxImpl::C_aio_stat2_Ack::finish(int r)
   }
 
   if (c->callback_complete) {
-    boost::asio::defer(c->io->client->finish_strand, CB_AioComplete(c));
+    c->io->client->finisher.queue(new C_AioComplete(c));
   }
 
   c->put_unlock();
@@ -1998,7 +1964,7 @@ void librados::IoCtxImpl::C_aio_Complete::finish(int r)
 
   if (c->callback_complete ||
       c->callback_safe) {
-    boost::asio::defer(c->io->client->finish_strand, CB_AioComplete(c));
+    c->io->client->finisher.queue(new C_AioComplete(c));
   }
 
   if (c->aio_write_seq) {
@@ -2062,7 +2028,6 @@ int librados::IoCtxImpl::application_enable(const std::string& app_name,
 
   r = c->get_return_value();
   c->release();
-  c->put();
   if (r < 0) {
     return r;
   }
@@ -2078,10 +2043,7 @@ void librados::IoCtxImpl::application_enable_async(const std::string& app_name,
   // preserved until Luminous is configured as minimim version.
   if (!client->get_required_monitor_features().contains_all(
         ceph::features::mon::FEATURE_LUMINOUS)) {
-    boost::asio::defer(client->finish_strand,
-		       [cb = CB_PoolAsync_Safe(c)]() mutable {
-			 cb(-EOPNOTSUPP);
-		       });
+    client->finisher.queue(new C_PoolAsync_Safe(c), -EOPNOTSUPP);
     return;
   }
 
@@ -2099,7 +2061,7 @@ void librados::IoCtxImpl::application_enable_async(const std::string& app_name,
   cmds.push_back(cmd.str());
   bufferlist inbl;
   client->mon_command_async(cmds, inbl, nullptr, nullptr,
-                            make_lambda_context(CB_PoolAsync_Safe(c)));
+                            new C_PoolAsync_Safe(c));
 }
 
 int librados::IoCtxImpl::application_list(std::set<std::string> *app_names)
diff --git a/src/librados/ListObjectImpl.h b/src/librados/ListObjectImpl.h
index 7396c12108d..95c2e21a4ec 100644
--- a/src/librados/ListObjectImpl.h
+++ b/src/librados/ListObjectImpl.h
@@ -11,15 +11,13 @@
  * Foundation.  See file COPYING.
  *
  */
+#include <string>
 
 #ifndef CEPH_LIBRADOS_LISTOBJECTIMPL_H
 #define CEPH_LIBRADOS_LISTOBJECTIMPL_H
 
-#include <string>
 #include <include/rados/librados.hpp>
 
-#include "include/cmp.h"
-
 namespace librados {
 struct ListObjectImpl {
   std::string nspace;
diff --git a/src/librados/PoolAsyncCompletionImpl.h b/src/librados/PoolAsyncCompletionImpl.h
index 73420fe359c..b52d7fada20 100644
--- a/src/librados/PoolAsyncCompletionImpl.h
+++ b/src/librados/PoolAsyncCompletionImpl.h
@@ -16,9 +16,7 @@
 #define CEPH_LIBRADOS_POOLASYNCCOMPLETIONIMPL_H
 
 #include "common/ceph_mutex.h"
-
-#include <boost/intrusive_ptr.hpp>
-
+#include "include/Context.h"
 #include "include/rados/librados.h"
 #include "include/rados/librados.hpp"
 
@@ -31,68 +29,67 @@ namespace librados {
     bool released = false;
     bool done = false;
 
-    rados_callback_t callback = nullptr;
-    void *callback_arg = nullptr;
+    rados_callback_t callback = 0;
+    void *callback_arg = nullptr;;
 
     PoolAsyncCompletionImpl() = default;
 
     int set_callback(void *cb_arg, rados_callback_t cb) {
-      std::scoped_lock l(lock);
+      std::scoped_lock l{lock};
       callback = cb;
       callback_arg = cb_arg;
       return 0;
     }
     int wait() {
-      std::unique_lock l(lock);
-      while (!done)
-	cond.wait(l);
+      std::unique_lock l{lock};
+      cond.wait(l, [this] { return done;});
       return 0;
     }
     int is_complete() {
-      std::scoped_lock l(lock);
+      std::scoped_lock l{lock};
       return done;
     }
     int get_return_value() {
-      std::scoped_lock l(lock);
+      std::scoped_lock l{lock};
       return rval;
     }
     void get() {
-      std::scoped_lock l(lock);
+      std::scoped_lock l{lock};
       ceph_assert(ref > 0);
       ref++;
     }
     void release() {
-      std::scoped_lock l(lock);
+      lock.lock();
       ceph_assert(!released);
       released = true;
+      put_unlock();
     }
     void put() {
-      std::unique_lock l(lock);
+      lock.lock();
+      put_unlock();
+    }
+    void put_unlock() {
+      ceph_assert(ref > 0);
       int n = --ref;
-      l.unlock();
+      lock.unlock();
       if (!n)
 	delete this;
     }
   };
 
-  inline void intrusive_ptr_add_ref(PoolAsyncCompletionImpl* p) {
-    p->get();
-  }
-  inline void intrusive_ptr_release(PoolAsyncCompletionImpl* p) {
-    p->put();
-  }
-
-  class CB_PoolAsync_Safe {
-    boost::intrusive_ptr<PoolAsyncCompletionImpl> p;
+  class C_PoolAsync_Safe : public Context {
+    PoolAsyncCompletionImpl *c;
 
   public:
-    explicit CB_PoolAsync_Safe(boost::intrusive_ptr<PoolAsyncCompletionImpl> p)
-      : p(p) {}
-    ~CB_PoolAsync_Safe() = default;
-
-    void operator()(int r) {
-      auto c(std::move(p));
-      std::unique_lock l(c->lock);
+    explicit C_PoolAsync_Safe(PoolAsyncCompletionImpl *_c) : c(_c) {
+      c->get();
+    }
+    ~C_PoolAsync_Safe() override {
+      c->put();
+    }
+  
+    void finish(int r) override {
+      c->lock.lock();
       c->rval = r;
       c->done = true;
       c->cond.notify_all();
@@ -100,10 +97,12 @@ namespace librados {
       if (c->callback) {
 	rados_callback_t cb = c->callback;
 	void *cb_arg = c->callback_arg;
-	l.unlock();
-	cb(c.get(), cb_arg);
-	l.lock();
+	c->lock.unlock();
+	cb(c, cb_arg);
+	c->lock.lock();
       }
+
+      c->lock.unlock();
     }
   };
 }
diff --git a/src/librados/RadosClient.cc b/src/librados/RadosClient.cc
index 15fef61f7ea..372419816d8 100644
--- a/src/librados/RadosClient.cc
+++ b/src/librados/RadosClient.cc
@@ -28,7 +28,6 @@
 #include "common/ceph_json.h"
 #include "common/errno.h"
 #include "common/ceph_json.h"
-#include "common/async/waiter.h"
 #include "include/buffer.h"
 #include "include/stringify.h"
 #include "include/util.h"
@@ -57,13 +56,22 @@
 #undef dout_prefix
 #define dout_prefix *_dout << "librados: "
 
-namespace ca = ceph::async;
-namespace cb = ceph::buffer;
-namespace bc = boost::container;
-namespace bs = boost::system;
-
 librados::RadosClient::RadosClient(CephContext *cct_)
-  : Dispatcher(cct_->get()) {}
+  : Dispatcher(cct_->get()),
+    cct_deleter{cct_, [](CephContext *p) {p->put();}},
+    conf(cct_->_conf),
+    state(DISCONNECTED),
+    monclient(cct_),
+    mgrclient(cct_, nullptr, &monclient.monmap),
+    messenger(NULL),
+    instance_id(0),
+    objecter(NULL),
+    timer(cct, lock),
+    refcnt(1),
+    log_last_version(0), log_cb(NULL), log_cb2(NULL), log_cb_arg(NULL),
+    finisher(cct, "radosclient", "fn-radosclient")
+{
+}
 
 int64_t librados::RadosClient::lookup_pool(const char *name)
 {
@@ -225,7 +233,7 @@ int librados::RadosClient::connect()
   }
 
   {
-    MonClient mc_bootstrap(cct, poolctx);
+    MonClient mc_bootstrap(cct);
     err = mc_bootstrap.get_monmap_and_config();
     if (err < 0)
       return err;
@@ -233,8 +241,6 @@ int librados::RadosClient::connect()
 
   common_init_finish(cct);
 
-  poolctx.start(cct->_conf.get_val<std::uint64_t>("librados_thread_count"));
-
   // get monmap
   err = monclient.build_initial_monmap();
   if (err < 0)
@@ -255,9 +261,9 @@ int librados::RadosClient::connect()
   ldout(cct, 1) << "starting objecter" << dendl;
 
   objecter = new (std::nothrow) Objecter(cct, messenger, &monclient,
-					 poolctx,
-					 cct->_conf->rados_mon_op_timeout,
-					 cct->_conf->rados_osd_op_timeout);
+			  &finisher,
+			  cct->_conf->rados_mon_op_timeout,
+			  cct->_conf->rados_osd_op_timeout);
   if (!objecter)
     goto out;
   objecter->set_balanced_budget();
@@ -312,6 +318,10 @@ int librados::RadosClient::connect()
   objecter->start();
   lock.lock();
 
+  timer.init();
+
+  finisher.start();
+
   state = CONNECTED;
   instance_id = monclient.get_global_id();
 
@@ -354,9 +364,12 @@ void librados::RadosClient::shutdown()
       // make sure watch callbacks are flushed
       watch_flush();
     }
+    finisher.wait_for_empty();
+    finisher.stop();
   }
   state = DISCONNECTED;
   instance_id = 0;
+  timer.shutdown();   // will drop+retake lock
   l.unlock();
   if (need_objecter) {
     objecter->shutdown();
@@ -368,40 +381,41 @@ void librados::RadosClient::shutdown()
     messenger->shutdown();
     messenger->wait();
   }
-  poolctx.stop();
   ldout(cct, 1) << "shutdown" << dendl;
 }
 
 int librados::RadosClient::watch_flush()
 {
   ldout(cct, 10) << __func__ << " enter" << dendl;
-  ca::waiter<> w;
-  objecter->linger_callback_flush(w);
-
-  w.wait();
+  ceph::mutex mylock = ceph::make_mutex("RadosClient::watch_flush::mylock");
+  ceph::condition_variable cond;
+  bool done;
+  objecter->linger_callback_flush(new C_SafeCond(mylock, cond, &done));
 
+  std::unique_lock l{mylock};
+  cond.wait(l, [&done] { return done; });
   ldout(cct, 10) << __func__ << " exit" << dendl;
   return 0;
 }
 
-struct CB_aio_watch_flush_Complete {
+struct C_aio_watch_flush_Complete : public Context {
   librados::RadosClient *client;
   librados::AioCompletionImpl *c;
 
-  CB_aio_watch_flush_Complete(librados::RadosClient *_client, librados::AioCompletionImpl *_c)
+  C_aio_watch_flush_Complete(librados::RadosClient *_client, librados::AioCompletionImpl *_c)
     : client(_client), c(_c) {
     c->get();
   }
 
-  void operator()() {
+  void finish(int r) override {
     c->lock.lock();
-    c->rval = 0;
+    c->rval = r;
     c->complete = true;
     c->cond.notify_all();
 
     if (c->callback_complete ||
 	c->callback_safe) {
-      boost::asio::defer(client->finish_strand, librados::CB_AioComplete(c));
+      client->finisher.queue(new librados::C_AioComplete(c));
     }
     c->put_unlock();
   }
@@ -410,7 +424,8 @@ struct CB_aio_watch_flush_Complete {
 int librados::RadosClient::async_watch_flush(AioCompletionImpl *c)
 {
   ldout(cct, 10) << __func__ << " enter" << dendl;
-  objecter->linger_callback_flush(CB_aio_watch_flush_Complete(this, c));
+  Context *oncomplete = new C_aio_watch_flush_Complete(this, c);
+  objecter->linger_callback_flush(oncomplete);
   ldout(cct, 10) << __func__ << " exit" << dendl;
   return 0;
 }
@@ -588,10 +603,15 @@ int librados::RadosClient::wait_for_osdmap()
 
 int librados::RadosClient::wait_for_latest_osdmap()
 {
-  ca::waiter<bs::error_code> w;
-  objecter->wait_for_latest_osdmap(w);
-  auto ec = w.wait();
-  return ceph::from_error_code(ec);
+  ceph::mutex mylock = ceph::make_mutex("RadosClient::wait_for_latest_osdmap");
+  ceph::condition_variable cond;
+  bool done;
+
+  objecter->wait_for_latest_osdmap(new C_SafeCond(mylock, cond, &done));
+
+  std::unique_lock l{mylock};
+  cond.wait(l, [&done] {return done;});
+  return 0;
 }
 
 int librados::RadosClient::pool_list(std::list<std::pair<int64_t, string> >& v)
@@ -609,24 +629,20 @@ int librados::RadosClient::pool_list(std::list<std::pair<int64_t, string> >& v)
 
 int librados::RadosClient::get_pool_stats(std::list<string>& pools,
 					  map<string,::pool_stat_t> *result,
-					  bool *pper_pool)
+					  bool *per_pool)
 {
-  ca::waiter<bs::error_code,
-	     bc::flat_map<std::string, ::pool_stat_t>, bool> w;
-
-  std::vector<std::string> v(pools.begin(), pools.end());
-  objecter->get_pool_stats(v, w);
-
-  auto [ec, res, per_pool] = w.wait();
-  if (ec)
-    return ceph::from_error_code(ec);
+  ceph::mutex mylock = ceph::make_mutex("RadosClient::get_pool_stats::mylock");
+  ceph::condition_variable cond;
+  bool done;
+  int ret = 0;
 
-  if (per_pool)
-    *pper_pool = per_pool;
-  if (result)
-    result->insert(res.begin(), res.end());
+  objecter->get_pool_stats(pools, result, per_pool,
+			   new C_SafeCond(mylock, cond, &done,
+					  &ret));
 
-  return 0;
+  unique_lock l{mylock};
+  cond.wait(l, [&done] { return done;});
+  return ret;
 }
 
 bool librados::RadosClient::get_pool_is_selfmanaged_snaps_mode(
@@ -688,10 +704,14 @@ int librados::RadosClient::pool_create(string& name,
   ceph::condition_variable cond;
   bool done;
   Context *onfinish = new C_SafeCond(mylock, cond, &done, &reply);
-  objecter->create_pool(name, onfinish, crush_rule);
+  reply = objecter->create_pool(name, onfinish, crush_rule);
 
-  std::unique_lock l{mylock};
-  cond.wait(l, [&done] { return done; });
+  if (reply < 0) {
+    delete onfinish;
+  } else {
+    std::unique_lock l{mylock};
+    cond.wait(l, [&done] { return done; });
+  }
   return reply;
 }
 
@@ -703,8 +723,11 @@ int librados::RadosClient::pool_create_async(string& name,
   if (r < 0)
     return r;
 
-  Context *onfinish = make_lambda_context(CB_PoolAsync_Safe(c));
-  objecter->create_pool(name, onfinish, crush_rule);
+  Context *onfinish = new C_PoolAsync_Safe(c);
+  r = objecter->create_pool(name, onfinish, crush_rule);
+  if (r < 0) {
+    delete onfinish;
+  }
   return r;
 }
 
@@ -743,10 +766,14 @@ int librados::RadosClient::pool_delete(const char *name)
   bool done;
   int ret;
   Context *onfinish = new C_SafeCond(mylock, cond, &done, &ret);
-  objecter->delete_pool(name, onfinish);
+  ret = objecter->delete_pool(name, onfinish);
 
-  std::unique_lock l{mylock};
-  cond.wait(l, [&done] { return done;});
+  if (ret < 0) {
+    delete onfinish;
+  } else {
+    std::unique_lock l{mylock};
+    cond.wait(l, [&done] { return done;});
+  }
   return ret;
 }
 
@@ -756,8 +783,11 @@ int librados::RadosClient::pool_delete_async(const char *name, PoolAsyncCompleti
   if (r < 0)
     return r;
 
-  Context *onfinish = make_lambda_context(CB_PoolAsync_Safe(c));
-  objecter->delete_pool(name, onfinish);
+  Context *onfinish = new C_PoolAsync_Safe(c);
+  r = objecter->delete_pool(name, onfinish);
+  if (r < 0) {
+    delete onfinish;
+  }
   return r;
 }
 
@@ -813,20 +843,7 @@ void librados::RadosClient::mon_command_async(const vector<string>& cmd,
                                               Context *on_finish)
 {
   std::lock_guard l{lock};
-  monclient.start_mon_command(cmd, inbl,
-			      [outs, outbl,
-			       on_finish = std::unique_ptr<Context>(on_finish)]
-			      (bs::error_code e,
-			       std::string&& s,
-			       ceph::bufferlist&& b) mutable {
-				if (outs)
-				  *outs = std::move(s);
-				if (outbl)
-				  *outbl = std::move(b);
-				if (on_finish)
-				  on_finish.release()->complete(
-				    ceph::from_error_code(e));
-			      });
+  monclient.start_mon_command(cmd, inbl, outbl, outs, on_finish);
 }
 
 int librados::RadosClient::mgr_command(const vector<string>& cmd,
@@ -880,68 +897,80 @@ int librados::RadosClient::mon_command(int rank, const vector<string>& cmd,
 				       const bufferlist &inbl,
 				       bufferlist *outbl, string *outs)
 {
-  ca::waiter<bs::error_code, std::string, ceph::bufferlist> w;
-  monclient.start_mon_command(rank, cmd, inbl, w);
-  auto&& [ec, s, bl] = w.wait();
-
-  if (outs)
-    *outs = std::move(s);
-  if (outbl)
-    *outbl = std::move(bl);
-
-  return ceph::from_error_code(ec);
+  ceph::mutex mylock = ceph::make_mutex("RadosClient::mon_command::mylock");
+  ceph::condition_variable cond;
+  bool done;
+  int rval;
+  {
+    std::lock_guard l{mylock};
+    monclient.start_mon_command(rank, cmd, inbl, outbl, outs,
+				new C_SafeCond(mylock, cond, &done, &rval));
+  }
+  std::unique_lock l{mylock};
+  cond.wait(l, [&done] { return done;});
+  return rval;
 }
 
 int librados::RadosClient::mon_command(string name, const vector<string>& cmd,
 				       const bufferlist &inbl,
 				       bufferlist *outbl, string *outs)
 {
-  ca::waiter<bs::error_code, std::string, ceph::bufferlist> w;
-  monclient.start_mon_command(name, cmd, inbl, w);
-  auto&& [ec, s, bl] = w.wait();
-
-  if (outs)
-    *outs = std::move(s);
-  if (outbl)
-    *outbl = std::move(bl);
-
-  return ceph::from_error_code(ec);
+  ceph::mutex mylock = ceph::make_mutex("RadosClient::mon_command::mylock");
+  ceph::condition_variable cond;
+  bool done;
+  int rval;
+  {
+    std::lock_guard l{mylock};
+    monclient.start_mon_command(name, cmd, inbl, outbl, outs,
+				new C_SafeCond(mylock, cond, &done, &rval));
+  }
+  std::unique_lock l{mylock};
+  cond.wait(l, [&done] { return done;});
+  return rval;
 }
 
 int librados::RadosClient::osd_command(int osd, vector<string>& cmd,
 				       const bufferlist& inbl,
 				       bufferlist *poutbl, string *prs)
 {
+  ceph::mutex mylock = ceph::make_mutex("RadosClient::osd_command::mylock");
+  ceph::condition_variable cond;
+  bool done;
+  int ret;
   ceph_tid_t tid;
 
   if (osd < 0)
     return -EINVAL;
 
-
-  ca::waiter<bs::error_code, std::string, cb::list> w;
-  // XXX do anything with tid?
-  objecter->osd_command(osd, std::move(cmd), cb::list(inbl), &tid, w);
-  auto [ec, s, bl] = w.wait();
-  if (poutbl)
-    *poutbl = std::move(bl);
-  if (prs)
-    *prs = std::move(s);
-  return ceph::from_error_code(ec);
+  {
+    std::lock_guard l{mylock};
+    // XXX do anything with tid?
+    objecter->osd_command(osd, cmd, inbl, &tid, poutbl, prs,
+			  new C_SafeCond(mylock, cond, &done, &ret));
+  }
+  std::unique_lock l{mylock};
+  cond.wait(l, [&done] { return done;});
+  return ret;
 }
 
 int librados::RadosClient::pg_command(pg_t pgid, vector<string>& cmd,
 				      const bufferlist& inbl,
 				      bufferlist *poutbl, string *prs)
 {
+  ceph::mutex mylock = ceph::make_mutex("RadosClient::pg_command::mylock");
+  ceph::condition_variable cond;
+  bool done;
+  int ret;
   ceph_tid_t tid;
-  ca::waiter<bs::error_code, std::string, cb::list> w;
-  objecter->pg_command(pgid, std::move(cmd), inbl, &tid, w);
-  auto [ec, s, bl] = w.wait();
-  if (poutbl)
-    *poutbl = std::move(bl);
-  if (prs)
-    *prs = std::move(s);
-  return ceph::from_error_code(ec);
+
+  {
+    std::lock_guard l{lock};
+    objecter->pg_command(pgid, cmd, inbl, &tid, poutbl, prs,
+			 new C_SafeCond(mylock, cond, &done, &ret));
+  }
+  std::unique_lock l{mylock};
+  cond.wait(l, [&done] { return done;});
+  return ret;
 }
 
 int librados::RadosClient::monitor_log(const string& level,
@@ -1140,24 +1169,3 @@ int librados::RadosClient::get_inconsistent_pgs(int64_t pool_id,
   }
   return 0;
 }
-
-namespace {
-const char *config_keys[] = {
-  "librados_thread_count",
-  NULL
-};
-}
-
-const char** librados::RadosClient::get_tracked_conf_keys() const
-{
-  return config_keys;
-}
-
-void librados::RadosClient::handle_conf_change(const ConfigProxy& conf,
-					       const std::set<std::string> &changed)
-{
-  if (changed.count("librados_thread_count")) {
-    poolctx.stop();
-    poolctx.start(conf.get_val<std::uint64_t>("librados_thread_count"));
-  }
-}
diff --git a/src/librados/RadosClient.h b/src/librados/RadosClient.h
index eb97a765faf..0f2b153def4 100644
--- a/src/librados/RadosClient.h
+++ b/src/librados/RadosClient.h
@@ -14,57 +14,48 @@
 #ifndef CEPH_LIBRADOS_RADOSCLIENT_H
 #define CEPH_LIBRADOS_RADOSCLIENT_H
 
-#include <functional>
-#include <memory>
-#include <string>
-
-#include "msg/Dispatcher.h"
-
-#include "common/async/context_pool.h"
 #include "common/config_fwd.h"
 #include "common/Cond.h"
+#include "common/Timer.h"
 #include "common/ceph_mutex.h"
 #include "common/ceph_time.h"
-#include "common/config_obs.h"
 #include "include/rados/librados.h"
 #include "include/rados/librados.hpp"
 #include "mon/MonClient.h"
 #include "mgr/MgrClient.h"
+#include "msg/Dispatcher.h"
 
 #include "IoCtxImpl.h"
 
+struct AuthAuthorizer;
 struct Context;
 class CephContext;
+struct Connection;
 class Message;
 class MLog;
 class Messenger;
 class AioCompletionImpl;
 
-class librados::RadosClient : public Dispatcher,
-			      public md_config_obs_t
+class librados::RadosClient : public Dispatcher
 {
-public:
-  using Dispatcher::cct;
-private:
   std::unique_ptr<CephContext,
-		  std::function<void(CephContext*)> > cct_deleter{
-    cct, [](CephContext *p) {p->put();}};
+		  std::function<void(CephContext*)> > cct_deleter;
 
 public:
-  const ConfigProxy& conf{cct->_conf};
-  ceph::async::io_context_pool poolctx;
+  using Dispatcher::cct;
+  const ConfigProxy& conf;
 private:
   enum {
     DISCONNECTED,
     CONNECTING,
     CONNECTED,
-  } state{DISCONNECTED};
+  } state;
 
-  MonClient monclient{cct, poolctx};
-  MgrClient mgrclient{cct, nullptr, &monclient.monmap};
-  Messenger *messenger{nullptr};
+  MonClient monclient;
+  MgrClient mgrclient;
+  Messenger *messenger;
 
-  uint64_t instance_id{0};
+  uint64_t instance_id;
 
   bool _dispatch(Message *m);
   bool ms_dispatch(Message *m) override;
@@ -74,16 +65,17 @@ private:
   void ms_handle_remote_reset(Connection *con) override;
   bool ms_handle_refused(Connection *con) override;
 
-  Objecter *objecter{nullptr};
+  Objecter *objecter;
 
   ceph::mutex lock = ceph::make_mutex("librados::RadosClient::lock");
   ceph::condition_variable cond;
-  int refcnt{1};
+  SafeTimer timer;
+  int refcnt;
 
-  version_t log_last_version{0};
-  rados_log_callback_t log_cb{nullptr};
-  rados_log_callback2_t log_cb2{nullptr};
-  void *log_cb_arg{nullptr};
+  version_t log_last_version;
+  rados_log_callback_t log_cb;
+  rados_log_callback2_t log_cb2;
+  void *log_cb_arg;
   string log_watch;
 
   bool service_daemon = false;
@@ -93,11 +85,11 @@ private:
   int wait_for_osdmap();
 
 public:
-  boost::asio::io_context::strand finish_strand{poolctx.get_io_context()};
+  Finisher finisher;
 
-  explicit RadosClient(CephContext *cct);
+  explicit RadosClient(CephContext *cct_);
   ~RadosClient() override;
-  int ping_monitor(std::string mon_id, std::string *result);
+  int ping_monitor(string mon_id, string *result);
   int connect();
   void shutdown();
 
@@ -185,9 +177,6 @@ public:
   mon_feature_t get_required_monitor_features() const;
 
   int get_inconsistent_pgs(int64_t pool_id, std::vector<std::string>* pgs);
-  const char** get_tracked_conf_keys() const override;
-  void handle_conf_change(const ConfigProxy& conf,
-                          const std::set <std::string> &changed) override;
 };
 
 #endif
diff --git a/src/librados/librados_c.cc b/src/librados/librados_c.cc
index bbdd9120955..41cdbf6a42f 100644
--- a/src/librados/librados_c.cc
+++ b/src/librados/librados_c.cc
@@ -10,7 +10,6 @@
 #include "common/common_init.h"
 #include "common/TracepointProvider.h"
 #include "common/hobject.h"
-#include "common/async/waiter.h"
 #include "include/rados/librados.h"
 #include "include/types.h"
 #include <include/stringify.h>
@@ -2005,14 +2004,15 @@ extern "C" int _rados_object_list(rados_ioctx_t io,
   // Zero out items so that they will be safe to free later
   memset(result_items, 0, sizeof(rados_object_list_item) * result_item_count);
 
+  std::list<librados::ListObjectImpl> result;
+  hobject_t next_hash;
+
   bufferlist filter_bl;
   if (filter_buf != nullptr) {
     filter_bl.append(filter_buf, filter_buf_len);
   }
 
-  ceph::async::waiter<boost::system::error_code,
-		      std::vector<librados::ListObjectImpl>,
-		      hobject_t> w;
+  C_SaferCond cond;
   ctx->objecter->enumerate_objects(
       ctx->poolid,
       ctx->oloc.nspace,
@@ -2020,22 +2020,24 @@ extern "C" int _rados_object_list(rados_ioctx_t io,
       *((hobject_t*)finish),
       result_item_count,
       filter_bl,
-      w);
+      &result,
+      &next_hash,
+      &cond);
 
   hobject_t *next_hobj = (hobject_t*)(*next);
   ceph_assert(next_hobj);
 
-  auto [ec, result, next_hash] = w.wait();
-
-  if (ec) {
+  int r = cond.wait();
+  if (r < 0) {
     *next_hobj = hobject_t::get_max();
-    return ceph::from_error_code(ec);
+    return r;
   }
 
   ceph_assert(result.size() <= result_item_count);  // Don't overflow!
 
   int k = 0;
-  for (auto i = result.begin(); i != result.end(); ++i) {
+  for (std::list<librados::ListObjectImpl>::iterator i = result.begin();
+       i != result.end(); ++i) {
     rados_object_list_item &item = result_items[k++];
     do_out_buffer(i->oid, &item.oid, &item.oid_length);
     do_out_buffer(i->nspace, &item.nspace, &item.nspace_length);
@@ -2512,7 +2514,7 @@ struct AioGetxattrData {
   bufferlist bl;
   char* user_buf;
   size_t len;
-  struct librados::CB_AioCompleteAndSafe user_completion;
+  struct librados::C_AioCompleteAndSafe user_completion;
 };
 
 static void rados_aio_getxattr_complete(rados_completion_t c, void *arg) {
@@ -2527,7 +2529,7 @@ static void rados_aio_getxattr_complete(rados_completion_t c, void *arg) {
       rc = cdata->bl.length();
     }
   }
-  cdata->user_completion(rc);
+  cdata->user_completion.finish(rc);
   delete cdata;
 }
 
@@ -2566,7 +2568,7 @@ struct AioGetxattrsData {
   }
   librados::RadosXattrsIter *it;
   rados_xattrs_iter_t *iter;
-  struct librados::CB_AioCompleteAndSafe user_completion;
+  struct librados::C_AioCompleteAndSafe user_completion;
 };
 }
 
@@ -2574,12 +2576,12 @@ static void rados_aio_getxattrs_complete(rados_completion_t c, void *arg) {
   AioGetxattrsData *cdata = reinterpret_cast<AioGetxattrsData*>(arg);
   int rc = _rados_aio_get_return_value(c);
   if (rc) {
-    cdata->user_completion(rc);
+    cdata->user_completion.finish(rc);
   } else {
     cdata->it->i = cdata->it->attrset.begin();
     *cdata->iter = cdata->it;
     cdata->it = 0;
-    cdata->user_completion(0);
+    cdata->user_completion.finish(0);
   }
   delete cdata;
 }
@@ -3149,7 +3151,7 @@ LIBRADOS_C_API_BASE_DEFAULT(rados_write_op_assert_version);
 extern "C" void _rados_write_op_assert_exists(rados_write_op_t write_op)
 {
   tracepoint(librados, rados_write_op_assert_exists_enter, write_op);
-  ((::ObjectOperation *)write_op)->stat(nullptr, nullptr, nullptr);
+  ((::ObjectOperation *)write_op)->stat(NULL, (ceph::real_time *)NULL, NULL);
   tracepoint(librados, rados_write_op_assert_exists_exit);
 }
 LIBRADOS_C_API_BASE_DEFAULT(rados_write_op_assert_exists);
@@ -3574,7 +3576,7 @@ LIBRADOS_C_API_BASE_DEFAULT(rados_read_op_assert_version);
 extern "C" void _rados_read_op_assert_exists(rados_read_op_t read_op)
 {
   tracepoint(librados, rados_read_op_assert_exists_enter, read_op);
-  ((::ObjectOperation *)read_op)->stat(nullptr, nullptr, nullptr);
+  ((::ObjectOperation *)read_op)->stat(NULL, (ceph::real_time *)NULL, NULL);
   tracepoint(librados, rados_read_op_assert_exists_exit);
 }
 LIBRADOS_C_API_BASE_DEFAULT(rados_read_op_assert_exists);
@@ -3800,7 +3802,7 @@ extern "C" void _rados_read_op_getxattrs(rados_read_op_t read_op,
   tracepoint(librados, rados_read_op_getxattrs_enter, read_op, prval);
   librados::RadosXattrsIter *xattrs_iter = new librados::RadosXattrsIter;
   ((::ObjectOperation *)read_op)->getxattrs(&xattrs_iter->attrset, prval);
-  ((::ObjectOperation *)read_op)->set_handler(new C_XattrsIter(xattrs_iter));
+  ((::ObjectOperation *)read_op)->add_handler(new C_XattrsIter(xattrs_iter));
   *iter = xattrs_iter;
   tracepoint(librados, rados_read_op_getxattrs_exit, *iter);
 }
@@ -3824,7 +3826,7 @@ extern "C" void _rados_read_op_omap_get_vals(rados_read_op_t read_op,
     &omap_iter->values,
     nullptr,
     prval);
-  ((::ObjectOperation *)read_op)->set_handler(new C_OmapIter(omap_iter));
+  ((::ObjectOperation *)read_op)->add_handler(new C_OmapIter(omap_iter));
   *iter = omap_iter;
   tracepoint(librados, rados_read_op_omap_get_vals_exit, *iter);
 }
@@ -3849,7 +3851,7 @@ extern "C" void _rados_read_op_omap_get_vals2(rados_read_op_t read_op,
     &omap_iter->values,
     (bool*)pmore,
     prval);
-  ((::ObjectOperation *)read_op)->set_handler(new C_OmapIter(omap_iter));
+  ((::ObjectOperation *)read_op)->add_handler(new C_OmapIter(omap_iter));
   *iter = omap_iter;
   tracepoint(librados, rados_read_op_omap_get_vals_exit, *iter);
 }
@@ -3881,7 +3883,7 @@ extern "C" void _rados_read_op_omap_get_keys(rados_read_op_t read_op,
   ((::ObjectOperation *)read_op)->omap_get_keys(
     start_after ? start_after : "",
     max_return, &ctx->keys, nullptr, prval);
-  ((::ObjectOperation *)read_op)->set_handler(ctx);
+  ((::ObjectOperation *)read_op)->add_handler(ctx);
   *iter = omap_iter;
   tracepoint(librados, rados_read_op_omap_get_keys_exit, *iter);
 }
@@ -3901,7 +3903,7 @@ extern "C" void _rados_read_op_omap_get_keys2(rados_read_op_t read_op,
     start_after ? start_after : "",
     max_return, &ctx->keys,
     (bool*)pmore, prval);
-  ((::ObjectOperation *)read_op)->set_handler(ctx);
+  ((::ObjectOperation *)read_op)->add_handler(ctx);
   *iter = omap_iter;
   tracepoint(librados, rados_read_op_omap_get_keys_exit, *iter);
 }
@@ -3916,7 +3918,7 @@ static void internal_rados_read_op_omap_get_vals_by_keys(rados_read_op_t read_op
   ((::ObjectOperation *)read_op)->omap_get_vals_by_keys(to_get,
                                                         &omap_iter->values,
                                                         prval);
-  ((::ObjectOperation *)read_op)->set_handler(new C_OmapIter(omap_iter));
+  ((::ObjectOperation *)read_op)->add_handler(new C_OmapIter(omap_iter));
   *iter = omap_iter;
 }
 
diff --git a/src/librados/librados_cxx.cc b/src/librados/librados_cxx.cc
index 4e240fe0c02..a021c26280d 100644
--- a/src/librados/librados_cxx.cc
+++ b/src/librados/librados_cxx.cc
@@ -21,7 +21,6 @@
 #include "common/common_init.h"
 #include "common/TracepointProvider.h"
 #include "common/hobject.h"
-#include "common/async/waiter.h"
 #include "include/rados/librados.h"
 #include "include/rados/librados.hpp"
 #include "include/types.h"
@@ -156,11 +155,10 @@ void librados::ObjectOperation::assert_exists()
 {
   ceph_assert(impl);
   ::ObjectOperation *o = &impl->o;
-  o->stat(nullptr, nullptr, nullptr);
+  o->stat(NULL, (ceph::real_time*) NULL, NULL);
 }
 
-void librados::ObjectOperation::exec(const char *cls, const char *method,
-				     bufferlist& inbl)
+void librados::ObjectOperation::exec(const char *cls, const char *method, bufferlist& inbl)
 {
   ceph_assert(impl);
   ::ObjectOperation *o = &impl->o;
@@ -1972,7 +1970,7 @@ struct AioGetxattrDataPP {
   AioGetxattrDataPP(librados::AioCompletionImpl *c, bufferlist *_bl) :
     bl(_bl), completion(c) {}
   bufferlist *bl;
-  struct librados::CB_AioCompleteAndSafe completion;
+  struct librados::C_AioCompleteAndSafe completion;
 };
 
 static void rados_aio_getxattr_completepp(rados_completion_t c, void *arg) {
@@ -1981,7 +1979,7 @@ static void rados_aio_getxattr_completepp(rados_completion_t c, void *arg) {
   if (rc >= 0) {
     rc = cdata->bl->length();
   }
-  cdata->completion(rc);
+  cdata->completion.finish(rc);
   delete cdata;
 }
 
@@ -3004,9 +3002,9 @@ int librados::IoCtx::object_list(const ObjectCursor &start,
   ceph_assert(next != nullptr);
   result->clear();
 
-  ceph::async::waiter<boost::system::error_code,
-		      std::vector<librados::ListObjectImpl>,
-		      hobject_t>  w;
+  C_SaferCond cond;
+  hobject_t next_hash;
+  std::list<librados::ListObjectImpl> obj_result;
   io_ctx_impl->objecter->enumerate_objects(
       io_ctx_impl->poolid,
       io_ctx_impl->oloc.nspace,
@@ -3014,17 +3012,19 @@ int librados::IoCtx::object_list(const ObjectCursor &start,
       *((hobject_t*)finish.c_cursor),
       result_item_count,
       filter,
-      w);
+      &obj_result,
+      &next_hash,
+      &cond);
 
-  auto [ec, obj_result, next_hash] = w.wait();
-  if (ec) {
+  int r = cond.wait();
+  if (r < 0) {
     next->set((rados_object_list_cursor)(new hobject_t(hobject_t::get_max())));
-    return ceph::from_error_code(ec);
+    return r;
   }
 
   next->set((rados_object_list_cursor)(new hobject_t(next_hash)));
 
-  for (auto i = obj_result.begin();
+  for (std::list<librados::ListObjectImpl>::iterator i = obj_result.begin();
        i != obj_result.end(); ++i) {
     ObjectItem oi;
     oi.oid = i->oid;
diff --git a/src/libradosstriper/MultiAioCompletionImpl.cc b/src/libradosstriper/MultiAioCompletionImpl.cc
index acf9e0b6b8b..901bb136664 100644
--- a/src/libradosstriper/MultiAioCompletionImpl.cc
+++ b/src/libradosstriper/MultiAioCompletionImpl.cc
@@ -58,3 +58,13 @@ void libradosstriper::MultiAioCompletionImpl::finish_adding_requests()
   if (!pending_safe)
     safe();
 }
+
+void intrusive_ptr_add_ref(libradosstriper::MultiAioCompletionImpl* ptr)
+{
+  ptr->get();
+}
+
+void intrusive_ptr_release(libradosstriper::MultiAioCompletionImpl* ptr)
+{
+  ptr->put();
+}
diff --git a/src/libradosstriper/MultiAioCompletionImpl.h b/src/libradosstriper/MultiAioCompletionImpl.h
index 3ac3aae4492..32f7b9a84cf 100644
--- a/src/libradosstriper/MultiAioCompletionImpl.h
+++ b/src/libradosstriper/MultiAioCompletionImpl.h
@@ -20,9 +20,7 @@
 #include "common/ceph_mutex.h"
 #include "include/radosstriper/libradosstriper.hpp"
 
-namespace libradosstriper {
-
-struct MultiAioCompletionImpl {
+struct libradosstriper::MultiAioCompletionImpl {
 
   ceph::mutex lock = ceph::make_mutex("MultiAioCompletionImpl lock", false);
   ceph::condition_variable cond;
@@ -153,17 +151,10 @@ struct MultiAioCompletionImpl {
   void complete_request(ssize_t r);
   void safe_request(ssize_t r);
   void finish_adding_requests();
-};
 
-inline void intrusive_ptr_add_ref(MultiAioCompletionImpl* ptr)
-{
-  ptr->get();
-}
+};
 
-inline void intrusive_ptr_release(MultiAioCompletionImpl* ptr)
-{
-  ptr->put();
-}
-}
+void intrusive_ptr_add_ref(libradosstriper::MultiAioCompletionImpl*);
+void intrusive_ptr_release(libradosstriper::MultiAioCompletionImpl*);
 
 #endif // CEPH_LIBRADOSSTRIPERSTRIPER_MULTIAIOCOMPLETIONIMPL_H
diff --git a/src/libradosstriper/RadosStriperImpl.h b/src/libradosstriper/RadosStriperImpl.h
index 8226a9ba2a2..160db7b6f8d 100644
--- a/src/libradosstriper/RadosStriperImpl.h
+++ b/src/libradosstriper/RadosStriperImpl.h
@@ -17,8 +17,6 @@
 
 #include <string>
 
-#include <boost/intrusive_ptr.hpp>
-
 #include "include/rados/librados.h"
 #include "include/rados/librados.hpp"
 #include "include/radosstriper/libradosstriper.h"
@@ -28,7 +26,6 @@
 #include "librados/IoCtxImpl.h"
 #include "librados/AioCompletionImpl.h"
 #include "common/RefCountedObj.h"
-#include "common/ceph_context.h"
 
 namespace libradosstriper {
 
diff --git a/src/librbd/Journal.cc b/src/librbd/Journal.cc
index 4a32d93c94b..66849c461fa 100644
--- a/src/librbd/Journal.cc
+++ b/src/librbd/Journal.cc
@@ -1645,7 +1645,7 @@ int Journal<I>::check_resync_requested(bool *do_resync) {
     decode(client_data, bl_it);
   } catch (const buffer::error &err) {
     lderr(cct) << this << " " << __func__ << ": "
-               << "failed to decode client data: " << err.what() << dendl;
+               << "failed to decode client data: " << err << dendl;
     return -EINVAL;
   }
 
diff --git a/src/mds/CDir.cc b/src/mds/CDir.cc
index 540c8461b05..771eb4a890a 100644
--- a/src/mds/CDir.cc
+++ b/src/mds/CDir.cc
@@ -1903,9 +1903,9 @@ void CDir::_omap_fetched(bufferlist& hdrbl, map<string, bufferlist>& omap,
       decode(got_fnode, p);
     } catch (const buffer::error &err) {
       derr << "Corrupt fnode in dirfrag " << dirfrag()
-	   << ": " << err.what() << dendl;
+        << ": " << err << dendl;
       clog->warn() << "Corrupt fnode header in " << dirfrag() << ": "
-		   << err.what() << " (" << get_path() << ")";
+		  << err << " (" << get_path() << ")";
       go_bad(complete);
       return;
     }
@@ -1971,7 +1971,7 @@ void CDir::_omap_fetched(bufferlist& hdrbl, map<string, bufferlist>& omap,
     } catch (const buffer::error &err) {
       cache->mds->clog->warn() << "Corrupt dentry '" << dname << "' in "
                                   "dir frag " << dirfrag() << ": "
-                               << err.what() << "(" << get_path() << ")";
+                               << err << "(" << get_path() << ")";
 
       // Remember that this dentry is damaged.  Subsequent operations
       // that try to act directly on it will get their EIOs, but this
@@ -2200,7 +2200,7 @@ void CDir::_omap_commit(int op_prio)
 
       // don't create new dirfrag blindly
       if (!is_new() && !state_test(CDir::STATE_FRAGMENTING))
-	op.stat(nullptr, nullptr, nullptr);
+	op.stat(NULL, (ceph::real_time*) NULL, NULL);
 
       if (!to_set.empty())
 	op.omap_set(to_set);
@@ -2238,7 +2238,7 @@ void CDir::_omap_commit(int op_prio)
 
   // don't create new dirfrag blindly
   if (!is_new() && !state_test(CDir::STATE_FRAGMENTING))
-    op.stat(nullptr, nullptr, nullptr);
+    op.stat(NULL, (ceph::real_time*)NULL, NULL);
 
   /*
    * save the header at the last moment.. If we were to send it off before other
diff --git a/src/mds/CInode.cc b/src/mds/CInode.cc
index 8e917aad64c..e3c4ef1e24a 100644
--- a/src/mds/CInode.cc
+++ b/src/mds/CInode.cc
@@ -1286,7 +1286,7 @@ void CInode::_fetched(bufferlist& bl, bufferlist& bl2, Context *fin)
       fin->complete(0);
     }
   } catch (buffer::error &err) {
-    derr << "Corrupt inode " << ino() << ": " << err.what() << dendl;
+    derr << "Corrupt inode " << ino() << ": " << err << dendl;
     fin->complete(-EINVAL);
     return;
   }
diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index 2a9248ab558..c32118dd48c 100644
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -8607,7 +8607,7 @@ void MDCache::_open_ino_backtrace_fetched(inodeno_t ino, bufferlist& bl, int err
       decode(backtrace, bl);
     } catch (const buffer::error &decode_exc) {
       derr << "corrupt backtrace on ino x0" << std::hex << ino
-           << std::dec << ": " << decode_exc.what() << dendl;
+           << std::dec << ": " << decode_exc << dendl;
       open_ino_finish(ino, info, -EIO);
       return;
     }
diff --git a/src/mds/MDSDaemon.cc b/src/mds/MDSDaemon.cc
index eb7626d64e5..45356177b99 100644
--- a/src/mds/MDSDaemon.cc
+++ b/src/mds/MDSDaemon.cc
@@ -61,8 +61,7 @@
 #define dout_prefix *_dout << "mds." << name << ' '
 
 // cons/des
-MDSDaemon::MDSDaemon(std::string_view n, Messenger *m, MonClient *mc,
-		     boost::asio::io_context& ioctx) :
+MDSDaemon::MDSDaemon(std::string_view n, Messenger *m, MonClient *mc) :
   Dispatcher(m->cct),
   timer(m->cct, mds_lock),
   gss_ktfile_client(m->cct->_conf.get_val<std::string>("gss_ktab_client_file")),
@@ -70,7 +69,6 @@ MDSDaemon::MDSDaemon(std::string_view n, Messenger *m, MonClient *mc,
   name(n),
   messenger(m),
   monc(mc),
-  ioctx(ioctx),
   mgrc(m->cct, m, &mc->monmap),
   log_client(m->cct, messenger, &mc->monmap, LogClient::NO_FLAGS),
   starttime(mono_clock::now())
@@ -847,12 +845,10 @@ void MDSDaemon::handle_mds_map(const cref_t<MMDSMap> &m)
 
     // Did I previously not hold a rank?  Initialize!
     if (mds_rank == NULL) {
-      mds_rank = new MDSRankDispatcher(
-	whoami, mds_lock, clog,
-	timer, beacon, mdsmap, messenger, monc, &mgrc,
-	new LambdaContext([this](int r){respawn();}),
-	new LambdaContext([this](int r){suicide();}),
-	ioctx);
+      mds_rank = new MDSRankDispatcher(whoami, mds_lock, clog,
+          timer, beacon, mdsmap, messenger, monc, &mgrc,
+          new LambdaContext([this](int r){respawn();}),
+          new LambdaContext([this](int r){suicide();}));
       dout(10) <<  __func__ << ": initializing MDS rank "
                << mds_rank->get_nodeid() << dendl;
       mds_rank->init();
diff --git a/src/mds/MDSDaemon.h b/src/mds/MDSDaemon.h
index 1b9e79c4543..a4568ee9c73 100644
--- a/src/mds/MDSDaemon.h
+++ b/src/mds/MDSDaemon.h
@@ -42,9 +42,7 @@ class MonClient;
 
 class MDSDaemon : public Dispatcher {
  public:
-  MDSDaemon(std::string_view n, Messenger *m, MonClient *mc,
-	    boost::asio::io_context& ioctx);
-
+  MDSDaemon(std::string_view n, Messenger *m, MonClient *mc);
   ~MDSDaemon() override;
 
   mono_time get_starttime() const {
@@ -136,7 +134,6 @@ class MDSDaemon : public Dispatcher {
 
   Messenger    *messenger;
   MonClient    *monc;
-  boost::asio::io_context& ioctx;
   MgrClient     mgrc;
   std::unique_ptr<MDSMap> mdsmap;
   LogClient    log_client;
diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index f214416555b..924768f6bbe 100644
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -16,7 +16,6 @@
 
 #include "common/debug.h"
 #include "common/errno.h"
-#include "common/async/waiter.h"
 
 #include "messages/MClientRequestForward.h"
 #include "messages/MMDSLoadTargets.h"
@@ -485,11 +484,10 @@ MDSRank::MDSRank(
     MonClient *monc_,
     MgrClient *mgrc,
     Context *respawn_hook_,
-    Context *suicide_hook_,
-    boost::asio::io_context& ioctx) :
+    Context *suicide_hook_) :
     cct(msgr->cct), mds_lock(mds_lock_), clog(clog_),
     timer(timer_), mdsmap(mdsmap_),
-    objecter(new Objecter(g_ceph_context, msgr, monc_, ctxpool, 0, 0)),
+    objecter(new Objecter(g_ceph_context, msgr, monc_, nullptr, 0, 0)),
     damage_table(whoami_), sessionmap(this),
     op_tracker(g_ceph_context, g_conf()->mds_enable_op_tracker,
                g_conf()->osd_num_op_tracker_shard),
@@ -580,7 +578,6 @@ MDSRank::~MDSRank()
 
 void MDSRankDispatcher::init()
 {
-  ctxpool.start(cct->_conf.get_val<std::uint64_t>("mds_asio_thread_count"));
   objecter->init();
   messenger->add_dispatcher_head(objecter);
 
@@ -829,7 +826,6 @@ void MDSRankDispatcher::shutdown()
     objecter->shutdown();
 
   monc->shutdown();
-  ctxpool.finish();
 
   op_tracker.on_shutdown();
 
@@ -1654,6 +1650,7 @@ void MDSRank::calc_recovery_set()
   dout(1) << " recovery set is " << rs << dendl;
 }
 
+
 void MDSRank::replay_start()
 {
   dout(1) << "replay_start" << dendl;
@@ -1664,13 +1661,18 @@ void MDSRank::replay_start()
   calc_recovery_set();
 
   // Check if we need to wait for a newer OSD map before starting
-  auto fin = new C_IO_Wrapper(
-    this, new C_MDS_BootStart(this, MDS_BOOT_INITIAL));
-  objecter->wait_for_map(
-    mdsmap->get_last_failure_osd_epoch(), lambdafy(fin));
-
-  dout(1) << " waiting for osdmap " << mdsmap->get_last_failure_osd_epoch()
-	  << " (which blacklists prior instance)" << dendl;
+  Context *fin = new C_IO_Wrapper(this, new C_MDS_BootStart(this, MDS_BOOT_INITIAL));
+  bool const ready = objecter->wait_for_map(
+      mdsmap->get_last_failure_osd_epoch(),
+      fin);
+
+  if (ready) {
+    delete fin;
+    boot_start();
+  } else {
+    dout(1) << " waiting for osdmap " << mdsmap->get_last_failure_osd_epoch()
+	    << " (which blacklists prior instance)" << dendl;
+  }
 }
 
 
@@ -1721,11 +1723,23 @@ void MDSRank::standby_replay_restart()
     /* We are transitioning out of standby: wait for OSD map update
        before making final pass */
     dout(1) << "standby_replay_restart (final takeover pass)" << dendl;
-    auto fin = new C_IO_Wrapper(
-      this, new C_MDS_StandbyReplayRestart(this));
-    objecter->wait_for_map(mdsmap->get_last_failure_osd_epoch(), lambdafy(fin));
-    dout(1) << " waiting for osdmap " << mdsmap->get_last_failure_osd_epoch()
-	    << " (which blacklists prior instance)" << dendl;
+    Context *fin = new C_IO_Wrapper(this, new C_MDS_StandbyReplayRestart(this));
+    bool ready = objecter->wait_for_map(mdsmap->get_last_failure_osd_epoch(), fin);
+    if (ready) {
+      delete fin;
+      mdlog->get_journaler()->reread_head_and_probe(
+        new C_MDS_StandbyReplayRestartFinish(
+          this,
+	  mdlog->get_journaler()->get_read_pos()));
+
+      dout(1) << " opening purge_queue (async)" << dendl;
+      purge_queue.open(NULL);
+      dout(1) << " opening open_file_table (async)" << dendl;
+      mdcache->open_file_table.load(nullptr);
+    } else {
+      dout(1) << " waiting for osdmap " << mdsmap->get_last_failure_osd_epoch()
+              << " (which blacklists prior instance)" << dendl;
+    }
   }
 }
 
@@ -2438,10 +2452,12 @@ int MDSRankDispatcher::handle_asok_command(std::string_view command,
       std::lock_guard l(mds_lock);
       set_osd_epoch_barrier(target_epoch);
     }
-    ceph::async::waiter<boost::system::error_code> w;
-    objecter->wait_for_map(target_epoch, w);
-    dout(4) << __func__ << ": waiting for OSD epoch " << target_epoch << dendl;
-    w.wait();
+    C_SaferCond cond;
+    bool already_got = objecter->wait_for_map(target_epoch, &cond);
+    if (!already_got) {
+      dout(4) << __func__ << ": waiting for OSD epoch " << target_epoch << dendl;
+      cond.wait();
+    }
   } else if (command == "session ls") {
     std::lock_guard l(mds_lock);
 
@@ -3388,7 +3404,7 @@ bool MDSRank::evict_client(int64_t session_id,
 
     Context *on_blacklist_done = new LambdaContext([this, fn](int r) {
       objecter->wait_for_latest_osdmap(
-      lambdafy((new C_OnFinisher(
+       new C_OnFinisher(
          new LambdaContext([this, fn](int r) {
               std::lock_guard l(mds_lock);
               auto epoch = objecter->with_osdmap([](const OSDMap &o){
@@ -3399,7 +3415,7 @@ bool MDSRank::evict_client(int64_t session_id,
 
               fn();
             }), finisher)
-      )));
+       );
     });
 
     dout(4) << "Sending mon blacklist command: " << cmd[0] << dendl;
@@ -3466,10 +3482,9 @@ MDSRankDispatcher::MDSRankDispatcher(
     MonClient *monc_,
     MgrClient *mgrc,
     Context *respawn_hook_,
-    Context *suicide_hook_,
-    boost::asio::io_context& ictx)
+    Context *suicide_hook_)
   : MDSRank(whoami_, mds_lock_, clog_, timer_, beacon_, mdsmap_,
-            msgr, monc_, mgrc, respawn_hook_, suicide_hook_, ictx)
+            msgr, monc_, mgrc, respawn_hook_, suicide_hook_)
 {
     g_conf().add_observer(this);
 }
@@ -3646,7 +3661,6 @@ const char** MDSRankDispatcher::get_tracked_conf_keys() const
     "mds_recall_warning_decay_rate",
     "mds_request_load_average_decay_rate",
     "mds_session_cache_liveness_decay_rate",
-    "mds_asio_thread_count",
     NULL
   };
   return KEYS;
@@ -3688,11 +3702,6 @@ void MDSRankDispatcher::handle_conf_change(const ConfigProxy& conf, const std::s
     mdcache->handle_conf_change(changed, *mdsmap);
     purge_queue.handle_conf_change(changed, *mdsmap);
   }));
-
-  if (changed.count("mds_asio_thread_count")) {
-    ctxpool.stop();
-    ctxpool.start(conf.get_val<std::uint64_t>("mds_asio_thread_count"));
-  }
 }
 
 void MDSRank::get_task_status(std::map<std::string, std::string> *status) {
diff --git a/src/mds/MDSRank.h b/src/mds/MDSRank.h
index fac6b08e47e..4187ee7c5ec 100644
--- a/src/mds/MDSRank.h
+++ b/src/mds/MDSRank.h
@@ -17,7 +17,6 @@
 
 #include <string_view>
 
-#include "common/async/context_pool.h"
 #include "common/DecayCounter.h"
 #include "common/LogClient.h"
 #include "common/Timer.h"
@@ -152,8 +151,7 @@ class MDSRank {
         MonClient *monc_,
         MgrClient *mgrc,
         Context *respawn_hook_,
-        Context *suicide_hook_,
-        boost::asio::io_context& ictx);
+        Context *suicide_hook_);
 
     mds_rank_t get_nodeid() const { return whoami; }
     int64_t get_metadata_pool();
@@ -356,7 +354,6 @@ class MDSRank {
 
     std::unique_ptr<MDSMap> &mdsmap; /* MDSDaemon::mdsmap */
 
-    ceph::async::io_context_pool ctxpool;
     Objecter *objecter;
 
     // sub systems
@@ -630,8 +627,7 @@ public:
       MonClient *monc_,
       MgrClient *mgrc,
       Context *respawn_hook_,
-      Context *suicide_hook_,
-      boost::asio::io_context& ictx);
+      Context *suicide_hook_);
 
   void init();
   void tick();
diff --git a/src/mds/PurgeQueue.h b/src/mds/PurgeQueue.h
index 7e3f093c3d8..7f42ac05be9 100644
--- a/src/mds/PurgeQueue.h
+++ b/src/mds/PurgeQueue.h
@@ -16,7 +16,6 @@
 #define PURGE_QUEUE_H_
 
 #include "include/compact_set.h"
-#include "common/Finisher.h"
 #include "mds/MDSMap.h"
 #include "osdc/Journaler.h"
 
diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index c1017e96d31..2e50ccbdb7d 100644
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -5347,18 +5347,29 @@ int Server::check_layout_vxattr(MDRequestRef& mdr,
     if (req_epoch > epoch) {
 
       // well, our map is older. consult mds.
-      auto fin = new C_IO_Wrapper(mds, new C_MDS_RetryRequest(mdcache, mdr));
+      Context *fin = new C_IO_Wrapper(mds, new C_MDS_RetryRequest(mdcache, mdr));
+
+      if (!mds->objecter->wait_for_map(req_epoch, fin))
+        return r; // wait, fin will retry this request later
+
+      delete fin;
+
+      // now we have at least as new a map as the client, try again.
+      mds->objecter->with_osdmap([&](const OSDMap& osdmap) {
+          r = parse_layout_vxattr(name, value, osdmap, layout);
+          epoch = osdmap.get_epoch();
+        });
+
+      ceph_assert(epoch >= req_epoch); // otherwise wait_for_map() told a lie
 
-      mds->objecter->wait_for_map(req_epoch, lambdafy(fin));
-      return r;
     } else if (req_epoch == 0 && !mdr->waited_for_osdmap) {
 
       // For compatibility with client w/ old code, we still need get the
       // latest map. One day if COMPACT_VERSION of MClientRequest >=3,
       // we can remove those code.
       mdr->waited_for_osdmap = true;
-      mds->objecter->wait_for_latest_osdmap(std::ref(*new C_IO_Wrapper(
-        mds, new C_MDS_RetryRequest(mdcache, mdr))));
+      mds->objecter->wait_for_latest_osdmap(new C_IO_Wrapper(
+			      mds, new C_MDS_RetryRequest(mdcache, mdr)));
       return r;
     }
   }
diff --git a/src/messages/MGetPoolStats.h b/src/messages/MGetPoolStats.h
index 6b64e4feee2..51bc134a21f 100644
--- a/src/messages/MGetPoolStats.h
+++ b/src/messages/MGetPoolStats.h
@@ -21,10 +21,10 @@
 class MGetPoolStats : public PaxosServiceMessage {
 public:
   uuid_d fsid;
-  std::vector<std::string> pools;
+  std::list<std::string> pools;
 
   MGetPoolStats() : PaxosServiceMessage{MSG_GETPOOLSTATS, 0} {}
-  MGetPoolStats(const uuid_d& f, ceph_tid_t t, std::vector<std::string>& ls, version_t l) :
+  MGetPoolStats(const uuid_d& f, ceph_tid_t t, std::list<std::string>& ls, version_t l) :
     PaxosServiceMessage{MSG_GETPOOLSTATS, l},
     fsid(f), pools(ls) {
     set_tid(t);
diff --git a/src/messages/MGetPoolStatsReply.h b/src/messages/MGetPoolStatsReply.h
index 063b6f7cb28..ff474d3d5db 100644
--- a/src/messages/MGetPoolStatsReply.h
+++ b/src/messages/MGetPoolStatsReply.h
@@ -22,7 +22,7 @@ class MGetPoolStatsReply : public PaxosServiceMessage {
 
 public:
   uuid_d fsid;
-  boost::container::flat_map<std::string, pool_stat_t> pool_stats;
+  std::map<std::string,pool_stat_t> pool_stats;
   bool per_pool = false;
 
   MGetPoolStatsReply() : PaxosServiceMessage{MSG_GETPOOLSTATSREPLY, 0,
diff --git a/src/messages/MOSDOp.h b/src/messages/MOSDOp.h
index 5301f0a43b2..43bd4005469 100644
--- a/src/messages/MOSDOp.h
+++ b/src/messages/MOSDOp.h
@@ -30,10 +30,8 @@
  *
  */
 
-class MOSDOpReply;
+class OSD;
 
-namespace _mosdop {
-template<typename V>
 class MOSDOp : public MOSDFastDispatchOp {
 private:
   static constexpr int HEAD_VERSION = 8;
@@ -56,7 +54,7 @@ private:
   std::atomic<bool> final_decode_needed;
   //
 public:
-  V ops;
+  std::vector<OSDOp> ops;
 private:
   snapid_t snap_seq;
   std::vector<snapid_t> snaps;
@@ -66,7 +64,7 @@ private:
   osd_reqid_t reqid; // reqid explicitly set by sender
 
 public:
-  friend MOSDOpReply;
+  friend class MOSDOpReply;
 
   ceph_tid_t get_client_tid() { return header.tid; }
   void set_snapid(const snapid_t& s) {
@@ -600,9 +598,6 @@ private:
   template<class T, typename... Args>
   friend boost::intrusive_ptr<T> ceph::make_message(Args&&... args);
 };
-}
-
-using MOSDOp = _mosdop::MOSDOp<std::vector<OSDOp>>;
 
 
 #endif
diff --git a/src/mgr/BaseMgrModule.cc b/src/mgr/BaseMgrModule.cc
index d18b35e947e..b340f74f1a1 100644
--- a/src/mgr/BaseMgrModule.cc
+++ b/src/mgr/BaseMgrModule.cc
@@ -154,9 +154,10 @@ ceph_send_command(BaseMgrModule *self, PyObject *args)
     // can wait for those.
     auto c = new LambdaContext([command_c, self](int command_r){
       self->py_modules->get_objecter().wait_for_latest_osdmap(
-	[command_c, command_r](boost::system::error_code) {
-	  command_c->complete(command_r);
-	});
+          new LambdaContext([command_c, command_r](int wait_r){
+            command_c->complete(command_r);
+          })
+      );
     });
 
     self->py_modules->get_monc().start_mon_command(
@@ -184,12 +185,9 @@ ceph_send_command(BaseMgrModule *self, PyObject *args)
         {cmd_json},
         {},
         &tid,
-	[command_c, f = &self->py_modules->cmd_finisher]
-	(boost::system::error_code ec, std::string s, ceph::buffer::list bl) {
-	  command_c->outs = std::move(s);
-	  command_c->outbl = std::move(bl);
-	  f->queue(command_c);
-	});
+        &command_c->outbl,
+        &command_c->outs,
+        new C_OnFinisher(command_c, &self->py_modules->cmd_finisher));
   } else if (std::string(type) == "mds") {
     int r = self->py_modules->get_client().mds_command(
         name,
@@ -222,12 +220,9 @@ ceph_send_command(BaseMgrModule *self, PyObject *args)
         {cmd_json},
         {},
         &tid,
-	[command_c, f = &self->py_modules->cmd_finisher]
-	(boost::system::error_code ec, std::string s, ceph::buffer::list bl) {
-	  command_c->outs = std::move(s);
-	  command_c->outbl = std::move(bl);
-	  f->queue(command_c);
-	});
+        &command_c->outbl,
+        &command_c->outs,
+        new C_OnFinisher(command_c, &self->py_modules->cmd_finisher));
     PyEval_RestoreThread(tstate);
     return nullptr;
   } else {
diff --git a/src/mgr/MgrContext.h b/src/mgr/MgrContext.h
index c6e647365b5..bec51a36871 100644
--- a/src/mgr/MgrContext.h
+++ b/src/mgr/MgrContext.h
@@ -17,7 +17,6 @@
 #include <memory>
 
 #include "common/ceph_json.h"
-#include "common/Cond.h"
 #include "mon/MonClient.h"
 
 class Command
diff --git a/src/mgr/MgrStandby.cc b/src/mgr/MgrStandby.cc
index 913009e7e10..b021a182971 100644
--- a/src/mgr/MgrStandby.cc
+++ b/src/mgr/MgrStandby.cc
@@ -38,7 +38,7 @@
 
 MgrStandby::MgrStandby(int argc, const char **argv) :
   Dispatcher(g_ceph_context),
-  monc{g_ceph_context, poolctx},
+  monc{g_ceph_context},
   client_messenger(Messenger::create(
 		     g_ceph_context,
 		     cct->_conf.get_val<std::string>("ms_type"),
@@ -46,7 +46,7 @@ MgrStandby::MgrStandby(int argc, const char **argv) :
 		     "mgr",
 		     getpid(),
 		     0)),
-  objecter{g_ceph_context, client_messenger.get(), &monc, poolctx, 0, 0},
+  objecter{g_ceph_context, client_messenger.get(), &monc, NULL, 0, 0},
   client{client_messenger.get(), &monc, &objecter},
   mgrc(g_ceph_context, client_messenger.get(), &monc.monmap),
   log_client(g_ceph_context, client_messenger.get(), &monc.monmap, LogClient::NO_FLAGS),
@@ -115,8 +115,6 @@ int MgrStandby::init()
   client_messenger->add_dispatcher_tail(&client);
   client_messenger->start();
 
-  poolctx.start(2);
-
   // Initialize MonClient
   if (monc.build_initial_monmap() < 0) {
     client_messenger->shutdown();
@@ -171,7 +169,6 @@ int MgrStandby::init()
   // this method.
   monc.set_passthrough_monmap();
 
-  poolctx.start(1);
   client_t whoami = monc.get_global_id();
   client_messenger->set_myname(entity_name_t::MGR(whoami.v));
   monc.set_log_client(&log_client);
@@ -282,31 +279,17 @@ void MgrStandby::shutdown()
     }
 
     py_module_registry.shutdown();
-    // stop sending beacon first, i use monc to talk with monitors
-    timer.shutdown();
-    // client uses monc and objecter
-    client.shutdown();
-    mgrc.shutdown();
-    // Stop asio threads, so leftover events won't call into shut down
-    // monclient/objecter.
-    poolctx.finish();
-    // stop monc, so mon won't be able to instruct me to shutdown/activate after
-    // the active_mgr is stopped
-    monc.shutdown();
-    if (active_mgr) {
-      active_mgr->shutdown();
-    }
+
     // objecter is used by monc and active_mgr
     objecter.shutdown();
     // client_messenger is used by all of them, so stop it in the end
     client_messenger->shutdown();
   }));
 
-  // objecter is used by monc and active_mgr
-  objecter.shutdown();
-  // client_messenger is used by all of them, so stop it in the end
-  client_messenger->shutdown();
-  poolctx.finish();
+  // Then stop the finisher to ensure its enqueued contexts aren't going
+  // to touch references to the things we're about to tear down
+  finisher.wait_for_empty();
+  finisher.stop();
 }
 
 void MgrStandby::respawn()
diff --git a/src/mgr/MgrStandby.h b/src/mgr/MgrStandby.h
index cac31a57640..d79fef2d5e5 100644
--- a/src/mgr/MgrStandby.h
+++ b/src/mgr/MgrStandby.h
@@ -16,7 +16,6 @@
 #define MGR_STANDBY_H_
 
 #include "auth/Auth.h"
-#include "common/async/context_pool.h"
 #include "common/Finisher.h"
 #include "common/Timer.h"
 #include "common/LogClient.h"
@@ -40,7 +39,6 @@ public:
 			  const std::set <std::string> &changed) override;
 
 protected:
-  ceph::async::io_context_pool poolctx;
   MonClient monc;
   std::unique_ptr<Messenger> client_messenger;
   Objecter objecter;
diff --git a/src/mon/AuthMonitor.cc b/src/mon/AuthMonitor.cc
index aee0e33a4ea..3cb261b412a 100644
--- a/src/mon/AuthMonitor.cc
+++ b/src/mon/AuthMonitor.cc
@@ -1773,7 +1773,7 @@ bool AuthMonitor::_upgrade_format_to_dumpling()
     // set daemon profiles
     if ((p->first.is_osd() || p->first.is_mds()) &&
         mon_caps == "allow rwx") {
-      new_caps = string("allow profile ") + std::string(p->first.get_type_name());
+      new_caps = string("allow profile ") + p->first.get_type_name();
     }
 
     // update bootstrap keys
diff --git a/src/mon/ConfigMap.cc b/src/mon/ConfigMap.cc
index 696b18edbc4..b7343ce4264 100644
--- a/src/mon/ConfigMap.cc
+++ b/src/mon/ConfigMap.cc
@@ -115,7 +115,7 @@ ConfigMap::generate_entity_map(
   vector<pair<string,Section*>> sections = { make_pair("global", &global) };
   auto p = by_type.find(name.get_type_name());
   if (p != by_type.end()) {
-    sections.emplace_back(name.get_type_name(), &p->second);
+    sections.push_back(make_pair(name.get_type_name(), &p->second));
   }
   vector<std::string> name_bits;
   boost::split(name_bits, name.to_str(), [](char c){ return c == '.'; });
diff --git a/src/mon/ConfigMap.h b/src/mon/ConfigMap.h
index 2b16c43eef3..17b52af1d1d 100644
--- a/src/mon/ConfigMap.h
+++ b/src/mon/ConfigMap.h
@@ -97,8 +97,8 @@ struct Section {
 
 struct ConfigMap {
   Section global;
-  std::map<std::string,Section, std::less<>> by_type;
-  std::map<std::string,Section, std::less<>> by_id;
+  std::map<std::string,Section> by_type;
+  std::map<std::string,Section> by_id;
 
   Section *find_section(const std::string& name) {
     if (name == "global") {
diff --git a/src/mon/MonClient.cc b/src/mon/MonClient.cc
index 63e3d5cfa3c..dbb4287a2a2 100644
--- a/src/mon/MonClient.cc
+++ b/src/mon/MonClient.cc
@@ -26,7 +26,6 @@
 
 #include "messages/MMonGetMap.h"
 #include "messages/MMonGetVersion.h"
-#include "messages/MMonGetMap.h"
 #include "messages/MMonGetVersionReply.h"
 #include "messages/MMonMap.h"
 #include "messages/MConfig.h"
@@ -47,7 +46,6 @@
 #include "common/LogClient.h"
 
 #include "MonClient.h"
-#include "error_code.h"
 #include "MonMap.h"
 
 #include "auth/Auth.h"
@@ -60,16 +58,14 @@
 #undef dout_prefix
 #define dout_prefix *_dout << "monclient" << (_hunting() ? "(hunting)":"") << ": "
 
-namespace bs = boost::system;
 using std::string;
-using namespace std::literals;
 
-MonClient::MonClient(CephContext *cct_, boost::asio::io_context& service) :
+MonClient::MonClient(CephContext *cct_) :
   Dispatcher(cct_),
   AuthServer(cct_),
   messenger(NULL),
   timer(cct_, monc_lock),
-  service(service),
+  finisher(cct_),
   initialized(false),
   log_client(NULL),
   more_log_pending(false),
@@ -99,7 +95,7 @@ int MonClient::get_monmap()
 {
   ldout(cct, 10) << __func__ << dendl;
   std::unique_lock l(monc_lock);
-
+  
   sub.want("monmap", 0, 0);
   if (!_opened())
     _reopen_session();
@@ -436,18 +432,13 @@ void MonClient::handle_monmap(MMonMap *m)
 void MonClient::handle_config(MConfig *m)
 {
   ldout(cct,10) << __func__ << " " << *m << dendl;
-  // Take the sledgehammer approach to ensuring we don't depend on
-  // anything in MonClient.
-  boost::asio::defer(finish_strand,
-		    [m, cct = boost::intrusive_ptr<CephContext>(cct),
-		     config_notify_cb = config_notify_cb,
-		     config_cb = config_cb]() {
-		      cct->_conf.set_mon_vals(cct.get(), m->config, config_cb);
-		      if (config_notify_cb) {
-			config_notify_cb();
-		      }
-		      m->put();
-		    });
+  finisher.queue(new LambdaContext([this, m](int r) {
+	cct->_conf.set_mon_vals(cct, m->config, config_cb);
+	if (config_notify_cb) {
+	  config_notify_cb();
+	}
+	m->put();
+      }));
   got_config = true;
   map_cond.notify_all();
 }
@@ -487,6 +478,7 @@ int MonClient::init()
   messenger->add_dispatcher_head(this);
 
   timer.init();
+  finisher.start();
   schedule_tick();
 
   return 0;
@@ -498,10 +490,10 @@ void MonClient::shutdown()
   monc_lock.lock();
   stopping = true;
   while (!version_requests.empty()) {
-    ceph::async::defer(std::move(version_requests.begin()->second),
-		       monc_errc::shutting_down, 0, 0);
+    version_requests.begin()->second->context->complete(-ECANCELED);
     ldout(cct, 20) << __func__ << " canceling and discarding version request "
-		   << version_requests.begin()->first << dendl;
+		   << version_requests.begin()->second << dendl;
+    delete version_requests.begin()->second;
     version_requests.erase(version_requests.begin());
   }
   while (!mon_commands.empty()) {
@@ -519,6 +511,8 @@ void MonClient::shutdown()
   monc_lock.unlock();
 
   if (initialized) {
+    finisher.wait_for_empty();
+    finisher.stop();
     initialized = false;
   }
   monc_lock.lock();
@@ -702,8 +696,8 @@ void MonClient::_reopen_session(int rank)
 
   // throw out version check requests
   while (!version_requests.empty()) {
-    ceph::async::defer(std::move(version_requests.begin()->second),
-		       monc_errc::session_reset, 0, 0);
+    finisher.queue(version_requests.begin()->second->context, -EAGAIN);
+    delete version_requests.begin()->second;
     version_requests.erase(version_requests.begin());
   }
 
@@ -1087,9 +1081,10 @@ void MonClient::_send_command(MonCommand *r)
   if (r->is_tell()) {
     ++r->send_attempts;
     if (r->send_attempts > cct->_conf->mon_client_directed_command_retry) {
-      _finish_command(r, monc_errc::mon_unavailable, "mon unavailable", {});
+      _finish_command(r, -ENXIO, "mon unavailable");
       return;
     }
+
     // tell-style command
     if (monmap.min_mon_release >= ceph_release_t::octopus) {
       if (r->target_con) {
@@ -1099,7 +1094,7 @@ void MonClient::_send_command(MonCommand *r)
 	if (r->target_rank >= (int)monmap.size()) {
 	  ldout(cct, 10) << " target " << r->target_rank
 			 << " >= max mon " << monmap.size() << dendl;
-	  _finish_command(r, monc_errc::rank_dne, "mon rank dne"sv, {});
+	  _finish_command(r, -ENOENT, "mon rank dne");
 	  return;
 	}
 	r->target_con = messenger->connect_to_mon(
@@ -1108,7 +1103,7 @@ void MonClient::_send_command(MonCommand *r)
 	if (!monmap.contains(r->target_name)) {
 	  ldout(cct, 10) << " target " << r->target_name
 			 << " not present in monmap" << dendl;
-	  _finish_command(r, monc_errc::mon_dne, "mon dne"sv, {});
+	  _finish_command(r, -ENOENT, "mon dne");
 	  return;
 	}
 	r->target_con = messenger->connect_to_mon(
@@ -1143,7 +1138,7 @@ void MonClient::_send_command(MonCommand *r)
       if (r->target_rank >= (int)monmap.size()) {
 	ldout(cct, 10) << " target " << r->target_rank
 		       << " >= max mon " << monmap.size() << dendl;
-	_finish_command(r, monc_errc::rank_dne, "mon rank dne"sv, {});
+	_finish_command(r, -ENOENT, "mon rank dne");
 	return;
       }
       _reopen_session(r->target_rank);
@@ -1158,7 +1153,7 @@ void MonClient::_send_command(MonCommand *r)
       if (!monmap.contains(r->target_name)) {
 	ldout(cct, 10) << " target " << r->target_name
 		       << " not present in monmap" << dendl;
-	_finish_command(r, monc_errc::mon_dne, "mon dne"sv, {});
+	_finish_command(r, -ENOENT, "mon dne");
 	return;
       }
       _reopen_session(monmap.get_rank(r->target_name));
@@ -1229,10 +1224,9 @@ void MonClient::handle_mon_command_ack(MMonCommandAck *ack)
   }
 
   ldout(cct, 10) << __func__ << " " << r->tid << " " << r->cmd << dendl;
-  auto ec = ack->r < 0 ? bs::error_code(-ack->r, mon_category())
-    : bs::error_code();
-  _finish_command(r, ec, ack->rs,
-		  std::move(ack->get_data()));
+  if (r->poutbl)
+    r->poutbl->claim(ack->get_data());
+  _finish_command(r, ack->r, ack->rs);
   ack->put();
 }
 
@@ -1257,9 +1251,9 @@ void MonClient::handle_command_reply(MCommandReply *reply)
   }
 
   ldout(cct, 10) << __func__ << " " << r->tid << " " << r->cmd << dendl;
-  auto ec = reply->r < 0 ? bs::error_code(-reply->r, mon_category())
-    : bs::error_code();
-  _finish_command(r, ec, reply->rs, std::move(reply->get_data()));
+  if (r->poutbl)
+    r->poutbl->claim(reply->get_data());
+  _finish_command(r, reply->r, reply->rs);
   reply->put();
 }
 
@@ -1276,17 +1270,19 @@ int MonClient::_cancel_mon_command(uint64_t tid)
   ldout(cct, 10) << __func__ << " tid " << tid << dendl;
 
   MonCommand *cmd = it->second;
-  _finish_command(cmd, monc_errc::timed_out, "timed out"sv, {});
+  _finish_command(cmd, -ETIMEDOUT, "");
   return 0;
 }
 
-void MonClient::_finish_command(MonCommand *r, bs::error_code ret,
-				std::string_view rs, ceph::buffer::list&& bl)
+void MonClient::_finish_command(MonCommand *r, int ret, string rs)
 {
-  ldout(cct, 10) << __func__ << " " << r->tid << " = " << ret << " " << rs
-		 << dendl;
-  ceph::async::defer(std::move(r->onfinish), ret, std::string(rs),
-		     std::move(bl));
+  ldout(cct, 10) << __func__ << " " << r->tid << " = " << ret << " " << rs << dendl;
+  if (r->prval)
+    *(r->prval) = ret;
+  if (r->prs)
+    *(r->prs) = rs;
+  if (r->onfinish)
+    finisher.queue(r->onfinish, ret);
   if (r->target_con) {
     r->target_con->mark_down();
   }
@@ -1294,8 +1290,117 @@ void MonClient::_finish_command(MonCommand *r, bs::error_code ret,
   delete r;
 }
 
+void MonClient::start_mon_command(const std::vector<string>& cmd,
+                                  const ceph::buffer::list& inbl,
+                                  ceph::buffer::list *outbl, string *outs,
+                                  Context *onfinish)
+{
+  ldout(cct,10) << __func__ << " cmd=" << cmd << dendl;
+  std::lock_guard l(monc_lock);
+  if (!initialized || stopping) {
+    if (onfinish) {
+      onfinish->complete(-ECANCELED);
+    }
+    return;
+  }
+  MonCommand *r = new MonCommand(++last_mon_command_tid);
+  r->cmd = cmd;
+  r->inbl = inbl;
+  r->poutbl = outbl;
+  r->prs = outs;
+  r->onfinish = onfinish;
+  if (cct->_conf->rados_mon_op_timeout > 0) {
+    class C_CancelMonCommand : public Context
+    {
+      uint64_t tid;
+      MonClient *monc;
+      public:
+      C_CancelMonCommand(uint64_t tid, MonClient *monc) : tid(tid), monc(monc) {}
+      void finish(int r) override {
+	monc->_cancel_mon_command(tid);
+      }
+    };
+    r->ontimeout = new C_CancelMonCommand(r->tid, this);
+    timer.add_event_after(cct->_conf->rados_mon_op_timeout, r->ontimeout);
+  }
+  mon_commands[r->tid] = r;
+  _send_command(r);
+}
+
+void MonClient::start_mon_command(const string &mon_name,
+                                  const std::vector<string>& cmd,
+                                  const ceph::buffer::list& inbl,
+                                  ceph::buffer::list *outbl, string *outs,
+                                  Context *onfinish)
+{
+  ldout(cct,10) << __func__ << " mon." << mon_name << " cmd=" << cmd << dendl;
+  std::lock_guard l(monc_lock);
+  if (!initialized || stopping) {
+    if (onfinish) {
+      onfinish->complete(-ECANCELED);
+    }
+    return;
+  }
+  MonCommand *r = new MonCommand(++last_mon_command_tid);
+
+  // detect/tolerate mon *rank* passed as a string
+  string err;
+  int rank = strict_strtoll(mon_name.c_str(), 10, &err);
+  if (err.size() == 0 && rank >= 0) {
+    ldout(cct,10) << __func__ << " interpreting name '" << mon_name
+		  << "' as rank " << rank << dendl;
+    r->target_rank = rank;
+  } else {
+    r->target_name = mon_name;
+  }
+  r->cmd = cmd;
+  r->inbl = inbl;
+  r->poutbl = outbl;
+  r->prs = outs;
+  r->onfinish = onfinish;
+  mon_commands[r->tid] = r;
+  _send_command(r);
+}
+
+void MonClient::start_mon_command(int rank,
+                                  const std::vector<string>& cmd,
+                                  const ceph::buffer::list& inbl,
+                                  ceph::buffer::list *outbl, string *outs,
+                                  Context *onfinish)
+{
+  ldout(cct,10) << __func__ << " rank " << rank << " cmd=" << cmd << dendl;
+  std::lock_guard l(monc_lock);
+  if (!initialized || stopping) {
+    if (onfinish) {
+      onfinish->complete(-ECANCELED);
+    }
+    return;
+  }
+  MonCommand *r = new MonCommand(++last_mon_command_tid);
+  r->target_rank = rank;
+  r->cmd = cmd;
+  r->inbl = inbl;
+  r->poutbl = outbl;
+  r->prs = outs;
+  r->onfinish = onfinish;
+  mon_commands[r->tid] = r;
+  _send_command(r);
+}
+
 // ---------
 
+void MonClient::get_version(string map, version_t *newest, version_t *oldest, Context *onfinish)
+{
+  version_req_d *req = new version_req_d(onfinish, newest, oldest);
+  ldout(cct, 10) << "get_version " << map << " req " << req << dendl;
+  std::lock_guard l(monc_lock);
+  auto m = ceph::make_message<MMonGetVersion>();
+  m->what = map;
+  m->handle = ++version_req_id;
+  version_requests[m->handle] = req;
+  _send_mon_message(std::move(m));
+}
+
 void MonClient::handle_get_version_reply(MMonGetVersionReply* m)
 {
   ceph_assert(ceph_mutex_is_locked(monc_lock));
@@ -1304,12 +1409,15 @@ void MonClient::handle_get_version_reply(MMonGetVersionReply* m)
     ldout(cct, 0) << __func__ << " version request with handle " << m->handle
 		  << " not found" << dendl;
   } else {
-    auto req = std::move(iter->second);
-    ldout(cct, 10) << __func__ << " finishing " << iter->first << " version "
-		   << m->version << dendl;
+    version_req_d *req = iter->second;
+    ldout(cct, 10) << __func__ << " finishing " << req << " version " << m->version << dendl;
     version_requests.erase(iter);
-    ceph::async::defer(std::move(req), bs::error_code(),
-		       m->version, m->oldest_version);
+    if (req->newest)
+      *req->newest = m->version;
+    if (req->oldest)
+      *req->oldest = m->oldest_version;
+    finisher.queue(req->context, 0);
+    delete req;
   }
   m->put();
 }
@@ -1479,8 +1587,7 @@ int MonClient::handle_auth_bad_method(
 	    allowed_methods,
 	    allowed_modes);
 	  if (r < 0) {
-	    auto ec = bs::error_code(-r, mon_category());
-	    _finish_command(i.second, ec, "auth failed"sv, {});
+	    _finish_command(i.second, r, "auth failed");
 	  }
 	  return r;
 	}
@@ -1889,97 +1996,3 @@ void MonClient::register_config_callback(md_config_t::config_callback fn) {
 md_config_t::config_callback MonClient::get_config_callback() {
   return config_cb;
 }
-
-class monc_error_category : public ceph::converting_category {
-public:
-  monc_error_category(){}
-  const char* name() const noexcept override;
-  std::string message(int ev) const override;
-  bs::error_condition default_error_condition(int ev) const noexcept
-    override;
-  bool equivalent(int ev, const bs::error_condition& c) const
-    noexcept override;
-  using ceph::converting_category::equivalent;
-  int from_code(int ev) const noexcept override;
-};
-
-const char* monc_error_category::name() const noexcept {
-  return "monc";
-}
-
-std::string monc_error_category::message(int ev) const {
-  if (ev == 0)
-    return "No error";
-
-  switch (static_cast<monc_errc>(ev)) {
-  case monc_errc::shutting_down: // Command failed due to MonClient shutting down
-    return "Command failed due to MonClient shutting down";
-  case monc_errc::session_reset:
-    return "Monitor session was reset";
-  case monc_errc::rank_dne:
-    return "Requested monitor rank does not exist";
-  case monc_errc::mon_dne:
-    return "Requested monitor does not exist";
-  case monc_errc::timed_out:
-    return "Monitor operation timed out";
-  case monc_errc::mon_unavailable:
-    return "Monitor unavailable";
-  }
-
-  return "Unknown error";
-}
-
-bs::error_condition monc_error_category::default_error_condition(int ev) const noexcept {
-  switch (static_cast<monc_errc>(ev)) {
-  case monc_errc::shutting_down:
-    return bs::errc::operation_canceled;
-  case monc_errc::session_reset:
-    return bs::errc::resource_unavailable_try_again;
-  case monc_errc::rank_dne:
-    [[fallthrough]];
-  case monc_errc::mon_dne:
-    return ceph::errc::not_in_map;
-  case monc_errc::timed_out:
-    return bs::errc::timed_out;
-  case monc_errc::mon_unavailable:
-    return bs::errc::no_such_device;
-  }
-  return { ev, *this };
-}
-
-bool monc_error_category::equivalent(int ev, const bs::error_condition& c) const noexcept {
-  switch (static_cast<monc_errc>(ev)) {
-  case monc_errc::rank_dne:
-    [[fallthrough]];
-  case monc_errc::mon_dne:
-      return c == bs::errc::no_such_file_or_directory;
-  default:
-    return default_error_condition(ev) == c;
-  }
-}
-
-int monc_error_category::from_code(int ev) const noexcept {
-  if (ev == 0)
-    return 0;
-
-  switch (static_cast<monc_errc>(ev)) {
-  case monc_errc::shutting_down:
-    return -ECANCELED;
-  case monc_errc::session_reset:
-    return -EAGAIN;
-  case monc_errc::rank_dne:
-    [[fallthrough]];
-  case monc_errc::mon_dne:
-    return -ENOENT;
-  case monc_errc::timed_out:
-    return -ETIMEDOUT;
-  case monc_errc::mon_unavailable:
-    return -ENXIO;
-  }
-  return -EDOM;
-}
-
-const bs::error_category& monc_category() noexcept {
-  static const monc_error_category c;
-  return c;
-}
diff --git a/src/mon/MonClient.h b/src/mon/MonClient.h
index 2d97440df8f..52c55fb18f3 100644
--- a/src/mon/MonClient.h
+++ b/src/mon/MonClient.h
@@ -27,10 +27,9 @@
 #include "MonMap.h"
 #include "MonSub.h"
 
-#include "common/async/completion.h"
 #include "common/Timer.h"
+#include "common/Finisher.h"
 #include "common/config.h"
-#include "messages/MMonGetVersion.h"
 
 #include "auth/AuthClient.h"
 #include "auth/AuthServer.h"
@@ -38,8 +37,11 @@
 class MMonMap;
 class MConfig;
 class MMonGetVersionReply;
+struct MMonSubscribeAck;
 class MMonCommandAck;
+struct MAuthReply;
 class LogClient;
+class AuthAuthorizer;
 class AuthClientHandler;
 class AuthRegistry;
 class KeyRing;
@@ -139,6 +141,7 @@ private:
 
 struct MonClientPinger : public Dispatcher,
 			 public AuthClient {
+
   ceph::mutex lock = ceph::make_mutex("MonClientPinger::lock");
   ceph::condition_variable ping_recvd_cond;
   std::string *result;
@@ -237,52 +240,11 @@ struct MonClientPinger : public Dispatcher,
   }
 };
 
-const boost::system::error_category& monc_category() noexcept;
-
-enum class monc_errc {
-  shutting_down = 1, // Command failed due to MonClient shutting down
-  session_reset, // Monitor session was reset
-  rank_dne, // Requested monitor rank does not exist
-  mon_dne, // Requested monitor does not exist
-  timed_out, // Monitor operation timed out
-  mon_unavailable // Monitor unavailable
-};
-
-namespace boost {
-namespace system {
-template<>
-struct is_error_code_enum<::monc_errc> {
-  static const bool value = true;
-};
-}
-}
-
-//  explicit conversion:
-inline boost::system::error_code make_error_code(monc_errc e) noexcept {
-  return { static_cast<int>(e), monc_category() };
-}
-
-// implicit conversion:
-inline boost::system::error_condition make_error_condition(monc_errc e)
-  noexcept {
-  return { static_cast<int>(e), monc_category() };
-}
-
-const boost::system::error_category& monc_category() noexcept;
 
 class MonClient : public Dispatcher,
 		  public AuthClient,
 		  public AuthServer /* for mgr, osd, mds */ {
-  static constexpr auto dout_subsys = ceph_subsys_monc;
 public:
-  // Error, Newest, Oldest
-  using VersionSig = void(boost::system::error_code, version_t, version_t);
-  using VersionCompletion = ceph::async::Completion<VersionSig>;
-
-  using CommandSig = void(boost::system::error_code, std::string,
-			  ceph::buffer::list);
-  using CommandCompletion = ceph::async::Completion<CommandSig>;
-
   MonMap monmap;
   std::map<std::string,std::string> config_mgr;
 
@@ -297,8 +259,7 @@ private:
 
   mutable ceph::mutex monc_lock = ceph::make_mutex("MonClient::monc_lock");
   SafeTimer timer;
-  boost::asio::io_context& service;
-  boost::asio::io_context::strand finish_strand{service};
+  Finisher finisher;
 
   bool initialized;
   bool stopping = false;
@@ -343,6 +304,7 @@ private:
   double reopen_interval_multiplier;
 
   Dispatcher *handle_authentication_dispatcher = nullptr;
+  
   bool _opened() const;
   bool _hunting() const;
   void _start_hunting();
@@ -399,7 +361,7 @@ public:
     bool more,
     uint32_t auth_method,
     const ceph::buffer::list& bl,
-    ceph::buffer::list *reply) override;
+    ceph::buffer::list *reply);
 
   void set_entity_name(EntityName name) { entity_name = name; }
   void set_handle_authentication_dispatcher(Dispatcher *d) {
@@ -448,12 +410,12 @@ public:
     std::lock_guard l(monc_lock);
     return sub.inc_want(what, start, flags);
   }
-
+  
   std::unique_ptr<KeyRing> keyring;
   std::unique_ptr<RotatingKeyRing> rotating_secrets;
 
  public:
-  MonClient(CephContext *cct_, boost::asio::io_context& service);
+  explicit MonClient(CephContext *cct_);
   MonClient(const MonClient &) = delete;
   MonClient& operator=(const MonClient &) = delete;
   ~MonClient() override;
@@ -549,206 +511,66 @@ private:
   struct MonCommand {
     // for tell only
     std::string target_name;
-    int target_rank = -1;
+    int target_rank;
     ConnectionRef target_con;
     std::unique_ptr<MonConnection> target_session;
     unsigned send_attempts = 0;  ///< attempt count for legacy mons
     utime_t last_send_attempt;
+
     uint64_t tid;
     std::vector<std::string> cmd;
     ceph::buffer::list inbl;
-    std::unique_ptr<CommandCompletion> onfinish;
-    std::optional<boost::asio::steady_timer> cancel_timer;
-
-    MonCommand(MonClient& monc, uint64_t t, std::unique_ptr<CommandCompletion> onfinish)
-      : tid(t), onfinish(std::move(onfinish)) {
-      auto timeout = ceph::maybe_timespan(monc.cct->_conf->rados_mon_op_timeout);
-      if (timeout) {
-	cancel_timer.emplace(monc.service, *timeout);
-	cancel_timer->async_wait(
-          [this, &monc](boost::system::error_code ec) {
-	    if (ec)
-	      return;
-	    std::scoped_lock l(monc.monc_lock);
-	    monc._cancel_mon_command(tid);
-	  });
-      }
-    }
+    ceph::buffer::list *poutbl;
+    std::string *prs;
+    int *prval;
+    Context *onfinish, *ontimeout;
+
+    explicit MonCommand(uint64_t t)
+      : target_rank(-1),
+	tid(t),
+	poutbl(NULL), prs(NULL), prval(NULL), onfinish(NULL), ontimeout(NULL)
+    {}
 
     bool is_tell() const {
       return target_name.size() || target_rank >= 0;
     }
   };
-  friend MonCommand;
   std::map<uint64_t,MonCommand*> mon_commands;
 
   void _send_command(MonCommand *r);
   void _check_tell_commands();
   void _resend_mon_commands();
   int _cancel_mon_command(uint64_t tid);
-  void _finish_command(MonCommand *r, boost::system::error_code ret, std::string_view rs,
-		       bufferlist&& bl);
+  void _finish_command(MonCommand *r, int ret, std::string rs);
   void _finish_auth();
   void handle_mon_command_ack(MMonCommandAck *ack);
   void handle_command_reply(MCommandReply *reply);
 
 public:
-  template<typename CompletionToken>
-  auto start_mon_command(const std::vector<std::string>& cmd,
-                         const ceph::buffer::list& inbl,
-			 CompletionToken&& token) {
-    ldout(cct,10) << __func__ << " cmd=" << cmd << dendl;
-    boost::asio::async_completion<CompletionToken, CommandSig> init(token);
-    {
-      std::scoped_lock l(monc_lock);
-      auto h = CommandCompletion::create(service.get_executor(),
-					 std::move(init.completion_handler));
-      if (!initialized || stopping) {
-	ceph::async::post(std::move(h), monc_errc::shutting_down, std::string{},
-			  bufferlist{});
-      } else {
-	auto r = new MonCommand(*this, ++last_mon_command_tid, std::move(h));
-	r->cmd = cmd;
-	r->inbl = inbl;
-	mon_commands.emplace(r->tid, r);
-	_send_command(r);
-      }
-    }
-    return init.result.get();
-  }
-
-  template<typename CompletionToken>
-  auto start_mon_command(int mon_rank, const std::vector<std::string>& cmd,
-			 const ceph::buffer::list& inbl, CompletionToken&& token) {
-    ldout(cct,10) << __func__ << " cmd=" << cmd << dendl;
-    boost::asio::async_completion<CompletionToken, CommandSig> init(token);
-    {
-      std::scoped_lock l(monc_lock);
-      auto h = CommandCompletion::create(service.get_executor(),
-					 std::move(init.completion_handler));
-      if (!initialized || stopping) {
-	ceph::async::post(std::move(h), monc_errc::shutting_down, std::string{},
-			  bufferlist{});
-      } else {
-	auto r = new MonCommand(*this, ++last_mon_command_tid, std::move(h));
-	r->target_rank = mon_rank;
-	r->cmd = cmd;
-	r->inbl = inbl;
-	mon_commands.emplace(r->tid, r);
-	_send_command(r);
-      }
-    }
-    return init.result.get();
-  }
-
-  template<typename CompletionToken>
-  auto start_mon_command(const std::string& mon_name,
-                         const std::vector<std::string>& cmd,
-			 const ceph::buffer::list& inbl,
-			 CompletionToken&& token) {
-    ldout(cct,10) << __func__ << " cmd=" << cmd << dendl;
-    boost::asio::async_completion<CompletionToken, CommandSig> init(token);
-    {
-      std::scoped_lock l(monc_lock);
-      auto h = CommandCompletion::create(service.get_executor(),
-					 std::move(init.completion_handler));
-      if (!initialized || stopping) {
-	ceph::async::post(std::move(h), monc_errc::shutting_down, std::string{},
-			  bufferlist{});
-      } else {
-	auto r = new MonCommand(*this, ++last_mon_command_tid, std::move(h));
-	// detect/tolerate mon *rank* passed as a string
-	std::string err;
-	int rank = strict_strtoll(mon_name.c_str(), 10, &err);
-	if (err.size() == 0 && rank >= 0) {
-	  ldout(cct,10) << __func__ << " interpreting name '" << mon_name
-			<< "' as rank " << rank << dendl;
-	  r->target_rank = rank;
-	} else {
-	  r->target_name = mon_name;
-	}
-	r->cmd = cmd;
-	r->inbl = inbl;
-	mon_commands.emplace(r->tid, r);
-	_send_command(r);
-      }
-    }
-    return init.result.get();
-  }
-
-  class ContextVerter {
-    std::string* outs;
-    ceph::bufferlist* outbl;
-    Context* onfinish;
-
-  public:
-    ContextVerter(std::string* outs, ceph::bufferlist* outbl, Context* onfinish)
-      : outs(outs), outbl(outbl), onfinish(onfinish) {}
-    ~ContextVerter() = default;
-    ContextVerter(const ContextVerter&) = default;
-    ContextVerter& operator =(const ContextVerter&) = default;
-    ContextVerter(ContextVerter&&) = default;
-    ContextVerter& operator =(ContextVerter&&) = default;
-
-    void operator()(boost::system::error_code e,
-		    std::string s,
-		    ceph::bufferlist bl) {
-      if (outs)
-	*outs = std::move(s);
-      if (outbl)
-	*outbl = std::move(bl);
-      if (onfinish)
-	onfinish->complete(ceph::from_error_code(e));
-    }
-  };
-
-  void start_mon_command(const vector<string>& cmd, const bufferlist& inbl,
-			 bufferlist *outbl, string *outs,
-			 Context *onfinish) {
-    start_mon_command(cmd, inbl, ContextVerter(outs, outbl, onfinish));
-  }
+  void start_mon_command(const std::vector<std::string>& cmd, const ceph::buffer::list& inbl,
+			ceph::buffer::list *outbl, std::string *outs,
+			Context *onfinish);
   void start_mon_command(int mon_rank,
-			 const vector<string>& cmd, const bufferlist& inbl,
-			 bufferlist *outbl, string *outs,
-			 Context *onfinish) {
-    start_mon_command(mon_rank, cmd, inbl, ContextVerter(outs, outbl, onfinish));
-  }
-  void start_mon_command(const string &mon_name,  ///< mon name, with mon. prefix
-			 const vector<string>& cmd, const bufferlist& inbl,
-			 bufferlist *outbl, string *outs,
-			 Context *onfinish) {
-    start_mon_command(mon_name, cmd, inbl, ContextVerter(outs, outbl, onfinish));
-  }
-
+                         const std::vector<std::string>& cmd, const ceph::buffer::list& inbl,
+                         ceph::buffer::list *outbl, std::string *outs,
+                         Context *onfinish);
+  void start_mon_command(const std::string &mon_name,  ///< mon name, with mon. prefix
+                         const std::vector<std::string>& cmd, const ceph::buffer::list& inbl,
+                         ceph::buffer::list *outbl, std::string *outs,
+                         Context *onfinish);
 
   // version requests
 public:
   /**
    * get latest known version(s) of cluster map
    *
-   * @param map string name of map (e.g., 'osdmap')
-   * @param token context that will be triggered on completion
-   * @return (via Completion) {} on success,
-   *         boost::system::errc::resource_unavailable_try_again if we need to
-   *         resubmit our request
+   * @param map std::string name of map (e.g., 'osdmap')
+   * @param newest pointer where newest map version will be stored
+   * @param oldest pointer where oldest map version will be stored
+   * @param onfinish context that will be triggered on completion
+   * @return (via context) 0 on success, -EAGAIN if we need to resubmit our request
    */
-  template<typename CompletionToken>
-  auto get_version(std::string&& map, CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, VersionSig> init(token);
-    {
-      std::scoped_lock l(monc_lock);
-      auto m = ceph::make_message<MMonGetVersion>();
-      m->what = std::move(map);
-      m->handle = ++version_req_id;
-      version_requests.emplace(m->handle,
-			       VersionCompletion::create(
-				 service.get_executor(),
-				 std::move(init.completion_handler)));
-      _send_mon_message(m);
-    }
-    return init.result.get();
-  }
-
+  void get_version(std::string map, version_t *newest, version_t *oldest, Context *onfinish);
   /**
    * Run a callback within our lock, with a reference
    * to the MonMap
@@ -767,10 +589,16 @@ public:
   md_config_t::config_callback get_config_callback();
 
 private:
+  struct version_req_d {
+    Context *context;
+    version_t *newest, *oldest;
+    version_req_d(Context *con, version_t *n, version_t *o) : context(con),newest(n), oldest(o) {}
+  };
 
-  std::map<ceph_tid_t, std::unique_ptr<VersionCompletion>> version_requests;
+  std::map<ceph_tid_t, version_req_d*> version_requests;
   ceph_tid_t version_req_id;
   void handle_get_version_reply(MMonGetVersionReply* m);
+
   md_config_t::config_callback config_cb;
   std::function<void(void)> config_notify_cb;
 };
diff --git a/src/mon/MonMap.cc b/src/mon/MonMap.cc
index 95b92ec8f4a..d63bada0fd6 100644
--- a/src/mon/MonMap.cc
+++ b/src/mon/MonMap.cc
@@ -658,7 +658,7 @@ int MonMap::init_with_config_file(const ConfigProxy& conf,
 
 using namespace seastar;
 
-seastar::future<> MonMap::read_monmap(const std::string& monmap)
+future<> MonMap::read_monmap(const std::string& monmap)
 {
   return open_file_dma(monmap, open_flags::ro).then([this] (file f) {
     return f.size().then([this, f = std::move(f)](size_t s) {
@@ -673,7 +673,7 @@ seastar::future<> MonMap::read_monmap(const std::string& monmap)
   });
 }
 
-seastar::future<> MonMap::init_with_dns_srv(bool for_mkfs, const std::string& name)
+future<> MonMap::init_with_dns_srv(bool for_mkfs, const std::string& name)
 {
   string domain;
   string service = name;
@@ -750,7 +750,7 @@ seastar::future<> MonMap::build_monmap(const crimson::common::ConfigProxy& conf,
   });
 }
 
-seastar::future<> MonMap::build_initial(const crimson::common::ConfigProxy& conf, bool for_mkfs)
+future<> MonMap::build_initial(const crimson::common::ConfigProxy& conf, bool for_mkfs)
 {
   // file?
   if (const auto monmap = conf.get_val<std::string>("monmap");
diff --git a/src/mon/error_code.cc b/src/mon/error_code.cc
deleted file mode 100644
index 67ff736da78..00000000000
--- a/src/mon/error_code.cc
+++ /dev/null
@@ -1,64 +0,0 @@
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2019 Red Hat <contact@redhat.com>
- * Author: Adam C. Emerson <aemerson@redhat.com>
- *
- * This is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License version 2.1, as published by the Free Software
- * Foundation.  See file COPYING.
- *
- */
-
-#include <string>
-
-#include "common/error_code.h"
-#include "common/errno.h"
-#include "error_code.h"
-
-namespace bs = boost::system;
-
-class mon_error_category : public ceph::converting_category {
-public:
-  mon_error_category(){}
-  const char* name() const noexcept override;
-  std::string message(int ev) const override;
-  bs::error_condition default_error_condition(int ev) const noexcept
-    override;
-  bool equivalent(int ev, const bs::error_condition& c) const
-    noexcept override;
-  using ceph::converting_category::equivalent;
-  int from_code(int ev) const noexcept override;
-};
-
-const char* mon_error_category::name() const noexcept {
-  return "mon";
-}
-
-std::string mon_error_category::message(int ev) const {
-  if (ev == 0)
-    return "No error";
-
-  return cpp_strerror(ev);
-}
-
-bs::error_condition
-mon_error_category::default_error_condition(int ev) const noexcept {
-  return { ev, bs::generic_category() };
-}
-
-bool mon_error_category::equivalent(int ev,const bs::error_condition& c) const noexcept {
-  return default_error_condition(ev) == c;
-}
-
-int mon_error_category::from_code(int ev) const noexcept {
-  return -ev;
-}
-
-const bs::error_category& mon_category() noexcept {
-  static const mon_error_category c;
-  return c;
-}
diff --git a/src/mon/error_code.h b/src/mon/error_code.h
deleted file mode 100644
index c247c68f98d..00000000000
--- a/src/mon/error_code.h
+++ /dev/null
@@ -1,47 +0,0 @@
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2019 Red Hat <contact@redhat.com>
- * Author: Adam C. Emerson <aemerson@redhat.com>
- *
- * This is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License version 2.1, as published by the Free Software
- * Foundation.  See file COPYING.
- *
- */
-
-#pragma once
-
-#include <boost/system/error_code.hpp>
-
-#include "include/rados.h"
-
-const boost::system::error_category& mon_category() noexcept;
-
-// The Monitor, like the OSD, mostly replies with POSIX error codes.
-
-enum mon_errc {
-};
-
-namespace boost {
-namespace system {
-template<>
-struct is_error_code_enum<::mon_errc> {
-  static const bool value = true;
-};
-}
-}
-
-//  explicit conversion:
-inline boost::system::error_code make_error_code(mon_errc e) noexcept {
-  return { e, mon_category() };
-}
-
-// implicit conversion:
-inline boost::system::error_condition make_error_condition(mon_errc e)
-  noexcept {
-  return { e, mon_category() };
-}
diff --git a/src/mount/conf.cc b/src/mount/conf.cc
index 0ec86b29fcb..228f53e7218 100644
--- a/src/mount/conf.cc
+++ b/src/mount/conf.cc
@@ -6,17 +6,13 @@
 #include <cstring>
 #include <map>
 
-#include "common/async/context_pool.h"
 #include "common/ceph_context.h"
 #include "common/ceph_argparse.h"
-#include "common/config.h"
 #include "global/global_init.h"
-
+#include "common/config.h"
 #include "auth/KeyRing.h"
-#include "mon/MonClient.h"
-
 #include "mount.ceph.h"
-
+#include "mon/MonClient.h"
 
 extern "C" void mount_ceph_get_config_info(const char *config_file,
 					   const char *name,
@@ -44,8 +40,7 @@ extern "C" void mount_ceph_get_config_info(const char *config_file,
   conf.parse_env(cct->get_module_type()); // environment variables override
   conf.apply_changes(nullptr);
 
-  ceph::async::io_context_pool ioc(1);
-  MonClient monc = MonClient(cct.get(), ioc);
+  MonClient monc = MonClient(cct.get());
   err = monc.build_initial_monmap();
   if (err)
     goto scrape_keyring;
diff --git a/src/msg/MessageRef.h b/src/msg/MessageRef.h
index 4f30f0b352a..5eb3655cb84 100644
--- a/src/msg/MessageRef.h
+++ b/src/msg/MessageRef.h
@@ -133,6 +133,8 @@ class MOSDForceRecovery;
 class MOSDFull;
 class MOSDMap;
 class MOSDMarkMeDown;
+class MOSDOp;
+class MOSDOpReply;
 class MOSDPeeringOp;
 class MOSDPGBackfill;
 class MOSDPGBackfillRemove;
diff --git a/src/msg/async/ProtocolV2.cc b/src/msg/async/ProtocolV2.cc
index aa864cf3311..fe2158289a5 100644
--- a/src/msg/async/ProtocolV2.cc
+++ b/src/msg/async/ProtocolV2.cc
@@ -44,7 +44,7 @@ void ProtocolV2::run_continuation(CtRef continuation) {
   try {
     CONTINUATION_RUN(continuation)
   } catch (const buffer::error &e) {
-    lderr(cct) << __func__ << " failed decoding of frame header: " << e.what()
+    lderr(cct) << __func__ << " failed decoding of frame header: " << e
                << dendl;
     _fault();
   } catch (const ceph::crypto::onwire::MsgAuthError &e) {
diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 446ddfbad87..4c85cd82f98 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -51,7 +51,6 @@
 #include "common/ceph_releases.h"
 #include "common/ceph_time.h"
 #include "common/version.h"
-#include "common/async/waiter.h"
 #include "common/pick_address.h"
 #include "common/blkdev.h"
 #include "common/numa.h"
@@ -221,7 +220,7 @@ CompatSet OSD::get_osd_compat_set() {
   return compat;
 }
 
-OSDService::OSDService(OSD *osd, ceph::async::io_context_pool& poolctx) :
+OSDService::OSDService(OSD *osd) :
   osd(osd),
   cct(osd->cct),
   whoami(osd->whoami), store(osd->store),
@@ -249,8 +248,7 @@ OSDService::OSDService(OSD *osd, ceph::async::io_context_pool& poolctx) :
   last_recalibrate(ceph_clock_now()),
   promote_max_objects(0),
   promote_max_bytes(0),
-  poolctx(poolctx),
-  objecter(new Objecter(osd->client_messenger->cct, osd->objecter_messenger, osd->monc, poolctx, 0, 0)),
+  objecter(new Objecter(osd->client_messenger->cct, osd->objecter_messenger, osd->monc, NULL, 0, 0)),
   m_objecter_finishers(cct->_conf->osd_objecter_finishers),
   watch_timer(osd->client_messenger->cct, watch_lock),
   next_notif_id(0),
@@ -2148,8 +2146,7 @@ OSD::OSD(CephContext *cct_, ObjectStore *store_,
 	 Messenger *hb_back_serverm,
 	 Messenger *osdc_messenger,
 	 MonClient *mc,
-	 const std::string &dev, const std::string &jdev,
-	 ceph::async::io_context_pool& poolctx) :
+	 const std::string &dev, const std::string &jdev) :
   Dispatcher(cct_),
   tick_timer(cct, osd_lock),
   tick_timer_without_osd_lock(cct, tick_timer_lock),
@@ -2196,7 +2193,7 @@ OSD::OSD(CephContext *cct_, ObjectStore *store_,
   up_thru_wanted(0),
   requested_full_first(0),
   requested_full_last(0),
-  service(this, poolctx)
+  service(this)
 {
 
   if (!gss_ktfile_client.empty()) {
@@ -6223,12 +6220,12 @@ bool OSD::ms_handle_refused(Connection *con)
   return true;
 }
 
-struct CB_OSD_GetVersion {
+struct C_OSD_GetVersion : public Context {
   OSD *osd;
-  explicit CB_OSD_GetVersion(OSD *o) : osd(o) {}
-  void operator ()(boost::system::error_code ec, version_t newest,
-		   version_t oldest) {
-    if (!ec)
+  uint64_t oldest, newest;
+  explicit C_OSD_GetVersion(OSD *o) : osd(o), oldest(0), newest(0) {}
+  void finish(int r) override {
+    if (r >= 0)
       osd->_got_mon_epochs(oldest, newest);
   }
 };
@@ -6248,7 +6245,8 @@ void OSD::start_boot()
   set_state(STATE_PREBOOT);
   dout(10) << "start_boot - have maps " << superblock.oldest_map
 	   << ".." << superblock.newest_map << dendl;
-  monc->get_version("osdmap", CB_OSD_GetVersion(this));
+  C_OSD_GetVersion *c = new C_OSD_GetVersion(this);
+  monc->get_version("osdmap", &c->newest, &c->oldest, c);
 }
 
 void OSD::_got_mon_epochs(epoch_t oldest, epoch_t newest)
@@ -9828,10 +9826,6 @@ void OSD::handle_conf_change(const ConfigProxy& conf,
     dout(0) << __func__ << ": scrub interval change" << dendl;
   }
   check_config();
-  if (changed.count("osd_asio_thread_count")) {
-    service.poolctx.stop();
-    service.poolctx.start(conf.get_val<std::uint64_t>("osd_asio_thread_count"));
-  }
 }
 
 void OSD::update_log_config()
@@ -9878,9 +9872,9 @@ void OSD::get_latest_osdmap()
 {
   dout(10) << __func__ << " -- start" << dendl;
 
-  ceph::async::waiter<boost::system::error_code> w;
-  service.objecter->wait_for_latest_osdmap(w);
-  w.wait();
+  C_SaferCond cond;
+  service.objecter->wait_for_latest_osdmap(&cond);
+  cond.wait();
 
   dout(10) << __func__ << " -- finish" << dendl;
 }
diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index 174c9dac645..56f6149180a 100644
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -19,7 +19,6 @@
 
 #include "msg/Dispatcher.h"
 
-#include "common/async/context_pool.h"
 #include "common/Timer.h"
 #include "common/WorkQueue.h"
 #include "common/AsyncReserver.h"
@@ -91,6 +90,7 @@ struct C_FinishSplits;
 struct C_OpenPGs;
 class LogChannel;
 class CephContext;
+class MOSDOp;
 
 class MOSDPGCreate2;
 class MOSDPGQuery;
@@ -517,7 +517,6 @@ public:
   void promote_throttle_recalibrate();
 
   // -- Objecter, for tiering reads/writes from/to other OSDs --
-  ceph::async::io_context_pool& poolctx;
   Objecter *objecter;
   int m_objecter_finishers;
   vector<Finisher*> objecter_finishers;
@@ -905,7 +904,7 @@ public:
   void dump_live_pgids();
 #endif
 
-  OSDService(OSD *osd, ceph::async::io_context_pool& poolctx);
+  explicit OSDService(OSD *osd);
   ~OSDService();
 };
 
@@ -1834,7 +1833,7 @@ protected:
 
   void send_full_update();
   
-  friend struct CB_OSD_GetVersion;
+  friend struct C_OSD_GetVersion;
 
   // -- alive --
   epoch_t up_thru_wanted;
@@ -2008,8 +2007,7 @@ private:
       Messenger *hb_front_server,
       Messenger *hb_back_server,
       Messenger *osdc_messenger,
-      MonClient *mc, const std::string &dev, const std::string &jdev,
-      ceph::async::io_context_pool& poolctx);
+      MonClient *mc, const std::string &dev, const std::string &jdev);
   ~OSD() override;
 
   // static bits
diff --git a/src/osd/OSDMap.h b/src/osd/OSDMap.h
index f87e38f45f7..f0055611b27 100644
--- a/src/osd/OSDMap.h
+++ b/src/osd/OSDMap.h
@@ -571,7 +571,7 @@ private:
   mempool::osdmap::map<int64_t,pg_pool_t> pools;
   mempool::osdmap::map<int64_t,std::string> pool_name;
   mempool::osdmap::map<std::string, std::map<std::string,std::string>> erasure_code_profiles;
-  mempool::osdmap::map<std::string,int64_t, std::less<>> name_pool;
+  mempool::osdmap::map<std::string,int64_t> name_pool;
 
   std::shared_ptr< mempool::osdmap::vector<uuid_d> > osd_uuid;
   mempool::osdmap::vector<osd_xinfo_t> osd_xinfo;
@@ -1299,7 +1299,7 @@ public:
     return new_purged_snaps;
   }
 
-  int64_t lookup_pg_pool_name(std::string_view name) const {
+  int64_t lookup_pg_pool_name(const std::string& name) const {
     auto p = name_pool.find(name);
     if (p == name_pool.end())
       return -ENOENT;
diff --git a/src/osd/PG.h b/src/osd/PG.h
index e372f8896d7..1ed490a0a9a 100644
--- a/src/osd/PG.h
+++ b/src/osd/PG.h
@@ -61,6 +61,7 @@ class OSD;
 class OSDService;
 class OSDShard;
 class OSDShardPGSlot;
+class MOSDOp;
 class MOSDPGScan;
 class MOSDPGBackfill;
 class MOSDPGInfo;
diff --git a/src/osd/PrimaryLogPG.cc b/src/osd/PrimaryLogPG.cc
index da3f52be83f..e06fa6c8c55 100644
--- a/src/osd/PrimaryLogPG.cc
+++ b/src/osd/PrimaryLogPG.cc
@@ -11406,7 +11406,7 @@ SnapSetContext *PrimaryLogPG::get_snapset_context(
       try {
 	ssc->snapset.decode(bvp);
       } catch (buffer::error& e) {
-        dout(0) << __func__ << " Can't decode snapset: " << e.what() << dendl;
+        dout(0) << __func__ << " Can't decode snapset: " << e << dendl;
 	return NULL;
       }
       ssc->exists = true;
diff --git a/src/osd/PrimaryLogPG.h b/src/osd/PrimaryLogPG.h
index 59ff68bfac9..a350714bbfc 100644
--- a/src/osd/PrimaryLogPG.h
+++ b/src/osd/PrimaryLogPG.h
@@ -40,6 +40,8 @@ class PrimaryLogPG;
 class PGLSFilter;
 class HitSet;
 struct TierAgentState;
+class MOSDOp;
+class MOSDOpReply;
 class OSDService;
 
 void intrusive_ptr_add_ref(PrimaryLogPG *pg);
diff --git a/src/osd/error_code.cc b/src/osd/error_code.cc
deleted file mode 100644
index 074064dd074..00000000000
--- a/src/osd/error_code.cc
+++ /dev/null
@@ -1,78 +0,0 @@
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2019 Red Hat <contact@redhat.com>
- * Author: Adam C. Emerson <aemerson@redhat.com>
- *
- * This is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License version 2.1, as published by the Free Software
- * Foundation.  See file COPYING.
- *
- */
-
-#include <string>
-
-#include "common/error_code.h"
-#include "common/errno.h"
-#include "error_code.h"
-
-class osd_error_category : public ceph::converting_category {
-public:
-  osd_error_category(){}
-  const char* name() const noexcept override;
-  std::string message(int ev) const override;
-  boost::system::error_condition default_error_condition(int ev) const noexcept
-    override;
-  bool equivalent(int ev, const boost::system::error_condition& c) const
-    noexcept override;
-  using ceph::converting_category::equivalent;
-  int from_code(int ev) const noexcept override;
-};
-
-const char* osd_error_category::name() const noexcept {
-  return "osd";
-}
-
-std::string osd_error_category::message(int ev) const {
-  if (ev == 0)
-    return "No error";
-
-  switch (static_cast<osd_errc>(ev)) {
-  case osd_errc::old_snapc:
-    return "ORDERSNAP flag set; writer has old snapc";
-  case osd_errc::blacklisted:
-    return "Blacklisted";
-  }
-
-  return cpp_strerror(ev);
-}
-
-boost::system::error_condition osd_error_category::default_error_condition(int ev) const noexcept {
-  if (ev == static_cast<int>(osd_errc::old_snapc) ||
-      ev == static_cast<int>(osd_errc::blacklisted))
-    return { ev, *this };
-  else
-    return { ev, boost::system::generic_category() };
-}
-
-bool osd_error_category::equivalent(int ev, const boost::system::error_condition& c) const noexcept {
-  switch (static_cast<osd_errc>(ev)) {
-  case osd_errc::old_snapc:
-      return c == boost::system::errc::invalid_argument;
-  case osd_errc::blacklisted:
-      return c == boost::system::errc::operation_not_permitted;
-  }
-  return default_error_condition(ev) == c;
-}
-
-int osd_error_category::from_code(int ev) const noexcept {
-  return -ev;
-}
-
-const boost::system::error_category& osd_category() noexcept {
-  static const osd_error_category c;
-  return c;
-}
diff --git a/src/osd/error_code.h b/src/osd/error_code.h
deleted file mode 100644
index f7385692190..00000000000
--- a/src/osd/error_code.h
+++ /dev/null
@@ -1,51 +0,0 @@
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2019 Red Hat <contact@redhat.com>
- * Author: Adam C. Emerson <aemerson@redhat.com>
- *
- * This is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License version 2.1, as published by the Free Software
- * Foundation.  See file COPYING.
- *
- */
-
-#pragma once
-
-#include <boost/system/error_code.hpp>
-
-#include "include/rados.h"
-
-const boost::system::error_category& osd_category() noexcept;
-
-// Since the OSD mostly uses POSIX error codes plus a couple
-// additions, this will be a degenerate error category for now that
-// mostly forwards to POSIX.
-
-enum class osd_errc {
-  old_snapc = 85,  /* ORDERSNAP flag set; writer has old snapc*/
-  blacklisted = 108 /* blacklisted */
-};
-
-namespace boost {
-namespace system {
-template<>
-struct is_error_code_enum<::osd_errc> {
-  static const bool value = true;
-};
-}
-}
-
-//  explicit conversion:
-inline boost::system::error_code make_error_code(osd_errc e) noexcept {
-  return { static_cast<int>(e), osd_category() };
-}
-
-// implicit conversion:
-inline boost::system::error_condition make_error_condition(osd_errc e)
-  noexcept {
-  return { static_cast<int>(e), osd_category() };
-}
diff --git a/src/osd/osd_types.cc b/src/osd/osd_types.cc
index 70250c12443..7cf74108ae7 100644
--- a/src/osd/osd_types.cc
+++ b/src/osd/osd_types.cc
@@ -1679,7 +1679,7 @@ bool pg_pool_t::is_removed_snap(snapid_t s) const
     return removed_snaps.contains(s);
 }
 
-snapid_t pg_pool_t::snap_exists(std::string_view s) const
+snapid_t pg_pool_t::snap_exists(const char *s) const
 {
   for (auto p = snaps.cbegin(); p != snaps.cend(); ++p)
     if (p->second.name == s)
@@ -6766,6 +6766,26 @@ ostream& operator<<(ostream& out, const OSDOp& op)
 }
 
 
+void OSDOp::split_osd_op_vector_in_data(vector<OSDOp>& ops, ceph::buffer::list& in)
+{
+  ceph::buffer::list::iterator datap = in.begin();
+  for (unsigned i = 0; i < ops.size(); i++) {
+    if (ops[i].op.payload_len) {
+      datap.copy(ops[i].op.payload_len, ops[i].indata);
+    }
+  }
+}
+
+void OSDOp::merge_osd_op_vector_in_data(vector<OSDOp>& ops, ceph::buffer::list& out)
+{
+  for (unsigned i = 0; i < ops.size(); i++) {
+    if (ops[i].indata.length()) {
+      ops[i].op.payload_len = ops[i].indata.length();
+      out.append(ops[i].indata);
+    }
+  }
+}
+
 void OSDOp::split_osd_op_vector_out_data(vector<OSDOp>& ops, ceph::buffer::list& in)
 {
   auto datap = in.begin();
@@ -6786,6 +6806,35 @@ void OSDOp::merge_osd_op_vector_out_data(vector<OSDOp>& ops, ceph::buffer::list&
   }
 }
 
+void OSDOp::clear_data(vector<OSDOp>& ops)
+{
+  for (unsigned i = 0; i < ops.size(); i++) {
+    OSDOp& op = ops[i];
+    op.outdata.clear();
+    if (ceph_osd_op_type_attr(op.op.op) &&
+        op.op.xattr.name_len &&
+	op.indata.length() >= op.op.xattr.name_len) {
+      ceph::buffer::ptr bp(op.op.xattr.name_len);
+      ceph::buffer::list bl;
+      bl.append(bp);
+      bl.copy_in(0, op.op.xattr.name_len, op.indata);
+      op.indata.claim(bl);
+    } else if (ceph_osd_op_type_exec(op.op.op) &&
+               op.op.cls.class_len &&
+	       op.indata.length() >
+	         (op.op.cls.class_len + op.op.cls.method_len)) {
+      __u8 len = op.op.cls.class_len + op.op.cls.method_len;
+      ceph::buffer::ptr bp(len);
+      ceph::buffer::list bl;
+      bl.append(bp);
+      bl.copy_in(0, len, op.indata);
+      op.indata.claim(bl);
+    } else {
+      op.indata.clear();
+    }
+  }
+}
+
 int prepare_info_keymap(
   CephContext* cct,
   map<string,bufferlist> *km,
diff --git a/src/osd/osd_types.h b/src/osd/osd_types.h
index 6e482ee8dbf..1976456c6ca 100644
--- a/src/osd/osd_types.h
+++ b/src/osd/osd_types.h
@@ -256,10 +256,10 @@ namespace std {
 // does it go in.
 struct object_locator_t {
   // You specify either the hash or the key -- not both
-  std::int64_t pool;     ///< pool id
-  std::string key;       ///< key string (if non-empty)
+  int64_t pool;     ///< pool id
+  std::string key;       ///< key std::string (if non-empty)
   std::string nspace;    ///< namespace
-  std::int64_t hash;     ///< hash position (if >= 0)
+  int64_t hash;     ///< hash position (if >= 0)
 
   explicit object_locator_t()
     : pool(-1), hash(-1) {}
@@ -267,11 +267,11 @@ struct object_locator_t {
     : pool(po), hash(-1)  {}
   explicit object_locator_t(int64_t po, int64_t ps)
     : pool(po), hash(ps)  {}
-  explicit object_locator_t(int64_t po, std::string_view ns)
+  explicit object_locator_t(int64_t po, std::string ns)
     : pool(po), nspace(ns), hash(-1) {}
-  explicit object_locator_t(int64_t po, std::string_view ns, int64_t ps)
+  explicit object_locator_t(int64_t po, std::string ns, int64_t ps)
     : pool(po), nspace(ns), hash(ps) {}
-  explicit object_locator_t(int64_t po, std::string_view ns, std::string_view s)
+  explicit object_locator_t(int64_t po, std::string ns, std::string s)
     : pool(po), key(s), nspace(ns), hash(-1) {}
   explicit object_locator_t(const hobject_t& soid)
     : pool(soid.pool), key(soid.get_key()), nspace(soid.nspace), hash(-1) {}
@@ -1692,7 +1692,7 @@ public:
   bool is_unmanaged_snaps_mode() const;
   bool is_removed_snap(snapid_t s) const;
 
-  snapid_t snap_exists(std::string_view s) const;
+  snapid_t snap_exists(const char *s) const;
   void add_snap(const char *n, utime_t stamp);
   uint64_t add_unmanaged_snap(bool preoctopus_compat);
   void remove_snap(snapid_t s);
@@ -3949,16 +3949,7 @@ struct OSDOp {
    * @param ops [out] vector of OSDOps
    * @param in  [in] combined data buffer
    */
-  template<typename V>
-  static void split_osd_op_vector_in_data(V& ops,
-					  ceph::buffer::list& in) {
-    ceph::buffer::list::iterator datap = in.begin();
-    for (unsigned i = 0; i < ops.size(); i++) {
-      if (ops[i].op.payload_len) {
-	datap.copy(ops[i].op.payload_len, ops[i].indata);
-      }
-    }
-  }
+  static void split_osd_op_vector_in_data(std::vector<OSDOp>& ops, ceph::buffer::list& in);
 
   /**
    * merge indata members of a vector of OSDOp into a single ceph::buffer::list
@@ -3969,15 +3960,7 @@ struct OSDOp {
    * @param ops [in] vector of OSDOps
    * @param out [out] combined data buffer
    */
-  template<typename V>
-  static void merge_osd_op_vector_in_data(V& ops, ceph::buffer::list& out) {
-    for (unsigned i = 0; i < ops.size(); i++) {
-      if (ops[i].indata.length()) {
-	ops[i].op.payload_len = ops[i].indata.length();
-	out.append(ops[i].indata);
-      }
-    }
-  }
+  static void merge_osd_op_vector_in_data(std::vector<OSDOp>& ops, ceph::buffer::list& out);
 
   /**
    * split a ceph::buffer::list into constituent outdata members of a vector of OSDOps
@@ -4000,37 +3983,11 @@ struct OSDOp {
    *
    * @param ops [in] vector of OSDOps
    */
-  template<typename V>
-  static void clear_data(V& ops) {
-    for (unsigned i = 0; i < ops.size(); i++) {
-      OSDOp& op = ops[i];
-      op.outdata.clear();
-      if (ceph_osd_op_type_attr(op.op.op) &&
-	  op.op.xattr.name_len &&
-	  op.indata.length() >= op.op.xattr.name_len) {
-	ceph::buffer::ptr bp(op.op.xattr.name_len);
-	ceph::buffer::list bl;
-	bl.append(bp);
-	bl.copy_in(0, op.op.xattr.name_len, op.indata);
-	op.indata.claim(bl);
-      } else if (ceph_osd_op_type_exec(op.op.op) &&
-		 op.op.cls.class_len &&
-		 op.indata.length() >
-	         (op.op.cls.class_len + op.op.cls.method_len)) {
-	__u8 len = op.op.cls.class_len + op.op.cls.method_len;
-	ceph::buffer::ptr bp(len);
-	ceph::buffer::list bl;
-	bl.append(bp);
-	bl.copy_in(0, len, op.indata);
-	op.indata.claim(bl);
-      } else {
-	op.indata.clear();
-      }
-    }
-  }
+  static void clear_data(std::vector<OSDOp>& ops);
 };
 std::ostream& operator<<(std::ostream& out, const OSDOp& op);
 
+
 struct pg_log_op_return_item_t {
   int32_t rval;
   bufferlist bl;
@@ -5002,14 +4959,14 @@ using pg_missing_tracker_t = pg_missing_set<true>;
  */
 struct pg_nls_response_t {
   collection_list_handle_t handle;
-  std::vector<librados::ListObjectImpl> entries;
+  std::list<librados::ListObjectImpl> entries;
 
   void encode(ceph::buffer::list& bl) const {
     ENCODE_START(1, 1, bl);
     encode(handle, bl);
     __u32 n = (__u32)entries.size();
     encode(n, bl);
-    for (auto i = entries.begin(); i != entries.end(); ++i) {
+    for (std::list<librados::ListObjectImpl>::const_iterator i = entries.begin(); i != entries.end(); ++i) {
       encode(i->nspace, bl);
       encode(i->oid, bl);
       encode(i->locator, bl);
@@ -5034,7 +4991,7 @@ struct pg_nls_response_t {
   void dump(ceph::Formatter *f) const {
     f->dump_stream("handle") << handle;
     f->open_array_section("entries");
-    for (auto p = entries.begin(); p != entries.end(); ++p) {
+    for (std::list<librados::ListObjectImpl>::const_iterator p = entries.begin(); p != entries.end(); ++p) {
       f->open_object_section("object");
       f->dump_string("namespace", p->nspace);
       f->dump_string("object", p->oid);
diff --git a/src/osdc/CMakeLists.txt b/src/osdc/CMakeLists.txt
index 551b105f893..6dd97b39362 100644
--- a/src/osdc/CMakeLists.txt
+++ b/src/osdc/CMakeLists.txt
@@ -2,7 +2,6 @@ set(osdc_files
   Filer.cc
   ObjectCacher.cc
   Objecter.cc
-  error_code.cc
   Striper.cc)
 add_library(osdc STATIC ${osdc_files})
 if(WITH_EVENTTRACE)
diff --git a/src/osdc/Filer.cc b/src/osdc/Filer.cc
index 4a2aeb88077..83370d8c08b 100644
--- a/src/osdc/Filer.cc
+++ b/src/osdc/Filer.cc
@@ -415,7 +415,7 @@ void Filer::truncate(inodeno_t ino,
   if (num_objs == 1) {
     vector<ObjectExtent> extents;
     Striper::file_to_extents(cct, ino, layout, offset, len, 0, extents);
-    osdc_opvec ops(1);
+    vector<OSDOp> ops(1);
     ops[0].op.op = CEPH_OSD_OP_TRIMTRUNC;
     ops[0].op.extent.truncate_seq = truncate_seq;
     ops[0].op.extent.truncate_size = extents[0].offset;
@@ -474,7 +474,7 @@ void Filer::_do_truncate_range(TruncRange *tr, int fin)
 
   // Issue objecter ops outside tr->lock to avoid lock dependency loop
   for (const auto& p : extents) {
-    osdc_opvec ops(1);
+    vector<OSDOp> ops(1);
     ops[0].op.op = CEPH_OSD_OP_TRIMTRUNC;
     ops[0].op.extent.truncate_size = p.offset;
     ops[0].op.extent.truncate_seq = tr->truncate_seq;
diff --git a/src/osdc/Objecter.cc b/src/osdc/Objecter.cc
index f1c9552fb57..37dc01d0144 100644
--- a/src/osdc/Objecter.cc
+++ b/src/osdc/Objecter.cc
@@ -12,16 +12,13 @@
  *
  */
 
-#include <algorithm>
 #include <cerrno>
 
 #include "Objecter.h"
 #include "osd/OSDMap.h"
-#include "osd/error_code.h"
 #include "Filer.h"
 
 #include "mon/MonClient.h"
-#include "mon/error_code.h"
 
 #include "msg/Messenger.h"
 #include "msg/Message.h"
@@ -48,16 +45,12 @@
 #include "messages/MWatchNotify.h"
 
 
-#include "common/Cond.h"
 #include "common/config.h"
 #include "common/perf_counters.h"
 #include "common/scrub_types.h"
 #include "include/str_list.h"
 #include "common/errno.h"
 #include "common/EventTrace.h"
-#include "common/async/waiter.h"
-#include "error_code.h"
-
 
 using std::list;
 using std::make_pair;
@@ -88,11 +81,6 @@ using ceph::shunique_lock;
 using ceph::acquire_shared;
 using ceph::acquire_unique;
 
-namespace bc = boost::container;
-namespace bs = boost::system;
-namespace ca = ceph::async;
-namespace cb = ceph::buffer;
-
 #define dout_subsys ceph_subsys_objecter
 #undef dout_prefix
 #define dout_prefix *_dout << messenger->get_myname() << ".objecter "
@@ -106,7 +94,6 @@ enum {
   l_osdc_op_send_bytes,
   l_osdc_op_resend,
   l_osdc_op_reply,
-  l_osdc_oplen_avg,
 
   l_osdc_op,
   l_osdc_op_r,
@@ -177,11 +164,6 @@ enum {
   l_osdc_last,
 };
 
-namespace {
-inline bs::error_code osdcode(int r) {
-  return (r < 0) ? bs::error_code(-r, osd_category()) : bs::error_code();
-}
-}
 
 // config obs ----------------------------
 
@@ -197,19 +179,55 @@ public:
   int call(std::string_view command, const cmdmap_t& cmdmap,
 	   Formatter *f,
 	   std::ostream& ss,
-	   cb::list& out) override;
+	   ceph::buffer::list& out) override;
+};
+
+/**
+ * This is a more limited form of C_Contexts, but that requires
+ * a ceph_context which we don't have here.
+ */
+class ObjectOperation::C_TwoContexts : public Context {
+  Context *first;
+  Context *second;
+public:
+  C_TwoContexts(Context *first, Context *second) :
+    first(first), second(second) {}
+  void finish(int r) override {
+    first->complete(r);
+    second->complete(r);
+    first = NULL;
+    second = NULL;
+  }
+
+  ~C_TwoContexts() override {
+    delete first;
+    delete second;
+  }
 };
 
-std::unique_lock<std::mutex> Objecter::OSDSession::get_lock(object_t& oid)
+void ObjectOperation::add_handler(Context *extra) {
+  size_t last = out_handler.size() - 1;
+  Context *orig = out_handler[last];
+  if (orig) {
+    Context *wrapper = new C_TwoContexts(orig, extra);
+    out_handler[last] = wrapper;
+  } else {
+    out_handler[last] = extra;
+  }
+}
+
+Objecter::OSDSession::unique_completion_lock Objecter::OSDSession::get_lock(
+  object_t& oid)
 {
   if (oid.name.empty())
-    return {};
+    return unique_completion_lock();
 
   static constexpr uint32_t HASH_PRIME = 1021;
   uint32_t h = ceph_str_hash_linux(oid.name.c_str(), oid.name.size())
     % HASH_PRIME;
 
-  return {completion_locks[h % num_locks], std::defer_lock};
+  return unique_completion_lock(completion_locks[h % num_locks],
+				std::defer_lock);
 }
 
 const char** Objecter::get_tracked_conf_keys() const
@@ -251,7 +269,6 @@ void Objecter::init()
     pcb.add_u64_counter(l_osdc_op_send_bytes, "op_send_bytes", "Sent data", NULL, 0, unit_t(UNIT_BYTES));
     pcb.add_u64_counter(l_osdc_op_resend, "op_resend", "Resent operations");
     pcb.add_u64_counter(l_osdc_op_reply, "op_reply", "Operation reply");
-    pcb.add_u64_avg(l_osdc_oplen_avg, "oplen_avg", "Average length of operation vector");
 
     pcb.add_u64_counter(l_osdc_op, "op", "Operations");
     pcb.add_u64_counter(l_osdc_op_r, "op_r", "Read operations", "rd",
@@ -367,7 +384,7 @@ void Objecter::init()
   }
 
   m_request_state_hook = new RequestStateHook(this);
-  auto admin_socket = cct->get_admin_socket();
+  AdminSocket* admin_socket = cct->get_admin_socket();
   int ret = admin_socket->register_command("objecter_requests",
 					   m_request_state_hook,
 					   "show in-progress osd requests");
@@ -414,54 +431,57 @@ void Objecter::shutdown()
   cct->_conf.remove_observer(this);
   wl.lock();
 
+  map<int,OSDSession*>::iterator p;
   while (!osd_sessions.empty()) {
-    auto p = osd_sessions.begin();
+    p = osd_sessions.begin();
     close_session(p->second);
   }
 
   while(!check_latest_map_lingers.empty()) {
-    auto i = check_latest_map_lingers.begin();
+    map<uint64_t, LingerOp*>::iterator i = check_latest_map_lingers.begin();
     i->second->put();
     check_latest_map_lingers.erase(i->first);
   }
 
   while(!check_latest_map_ops.empty()) {
-    auto i = check_latest_map_ops.begin();
+    map<ceph_tid_t, Op*>::iterator i = check_latest_map_ops.begin();
     i->second->put();
     check_latest_map_ops.erase(i->first);
   }
 
   while(!check_latest_map_commands.empty()) {
-    auto i = check_latest_map_commands.begin();
+    map<ceph_tid_t, CommandOp*>::iterator i
+      = check_latest_map_commands.begin();
     i->second->put();
     check_latest_map_commands.erase(i->first);
   }
 
   while(!poolstat_ops.empty()) {
-    auto i = poolstat_ops.begin();
+    map<ceph_tid_t,PoolStatOp*>::iterator i = poolstat_ops.begin();
     delete i->second;
     poolstat_ops.erase(i->first);
   }
 
   while(!statfs_ops.empty()) {
-    auto i = statfs_ops.begin();
+    map<ceph_tid_t, StatfsOp*>::iterator i = statfs_ops.begin();
     delete i->second;
     statfs_ops.erase(i->first);
   }
 
   while(!pool_ops.empty()) {
-    auto i = pool_ops.begin();
+    map<ceph_tid_t, PoolOp*>::iterator i = pool_ops.begin();
     delete i->second;
     pool_ops.erase(i->first);
   }
 
   ldout(cct, 20) << __func__ << " clearing up homeless session..." << dendl;
   while(!homeless_session->linger_ops.empty()) {
-    auto i = homeless_session->linger_ops.begin();
+    std::map<uint64_t, LingerOp*>::iterator i
+      = homeless_session->linger_ops.begin();
     ldout(cct, 10) << " linger_op " << i->first << dendl;
     LingerOp *lop = i->second;
     {
-      std::unique_lock swl(homeless_session->lock);
+      OSDSession::unique_lock swl(homeless_session->lock);
       _session_linger_op_remove(homeless_session, lop);
     }
     linger_ops.erase(lop->linger_id);
@@ -470,22 +490,23 @@ void Objecter::shutdown()
   }
 
   while(!homeless_session->ops.empty()) {
-    auto i = homeless_session->ops.begin();
+    std::map<ceph_tid_t, Op*>::iterator i = homeless_session->ops.begin();
     ldout(cct, 10) << " op " << i->first << dendl;
-    auto op = i->second;
+    Op *op = i->second;
     {
-      std::unique_lock swl(homeless_session->lock);
+      OSDSession::unique_lock swl(homeless_session->lock);
       _session_op_remove(homeless_session, op);
     }
     op->put();
   }
 
   while(!homeless_session->command_ops.empty()) {
-    auto i = homeless_session->command_ops.begin();
+    std::map<ceph_tid_t, CommandOp*>::iterator i
+      = homeless_session->command_ops.begin();
     ldout(cct, 10) << " command_op " << i->first << dendl;
-    auto cop = i->second;
+    CommandOp *cop = i->second;
     {
-      std::unique_lock swl(homeless_session->lock);
+      OSDSession::unique_lock swl(homeless_session->lock);
       _session_command_op_remove(homeless_session, cop);
     }
     cop->put();
@@ -511,7 +532,7 @@ void Objecter::shutdown()
   // This is safe because we guarantee no concurrent calls to
   // shutdown() with the ::initialized check at start.
   if (m_request_state_hook) {
-    auto admin_socket = cct->get_admin_socket();
+    AdminSocket* admin_socket = cct->get_admin_socket();
     admin_socket->unregister_commands(m_request_state_hook);
     delete m_request_state_hook;
     m_request_state_hook = NULL;
@@ -519,13 +540,14 @@ void Objecter::shutdown()
 }
 
 void Objecter::_send_linger(LingerOp *info,
-			    ceph::shunique_lock<ceph::shared_mutex>& sul)
+			    shunique_lock& sul)
 {
   ceph_assert(sul.owns_lock() && sul.mutex() == &rwlock);
-  std::unique_ptr<Op::OpComp> oncommit;
-  osdc_opvec opv;
-  std::shared_lock watchl(info->watch_lock);
-  cb::list *poutbl = nullptr;
+
+  vector<OSDOp> opv;
+  Context *oncommit = NULL;
+  LingerOp::shared_lock watchl(info->watch_lock);
+  ceph::buffer::list *poutbl = NULL;
   if (info->registered && info->is_watch) {
     ldout(cct, 15) << "send_linger " << info->linger_id << " reconnect"
 		   << dendl;
@@ -534,28 +556,22 @@ void Objecter::_send_linger(LingerOp *info,
     opv.back().op.watch.cookie = info->get_cookie();
     opv.back().op.watch.op = CEPH_OSD_WATCH_OP_RECONNECT;
     opv.back().op.watch.gen = ++info->register_gen;
-    oncommit = Op::OpComp::create(service.get_executor(),
-				  CB_Linger_Reconnect(this, info));
+    oncommit = new C_Linger_Reconnect(this, info);
   } else {
     ldout(cct, 15) << "send_linger " << info->linger_id << " register"
 		   << dendl;
     opv = info->ops;
-    // TODO Augment ca::Completion with an equivalent of
-    // target so we can handle these cases better.
-    auto c = std::make_unique<CB_Linger_Commit>(this, info);
+    C_Linger_Commit *c = new C_Linger_Commit(this, info);
     if (!info->is_watch) {
       info->notify_id = 0;
       poutbl = &c->outbl;
     }
-    oncommit = Op::OpComp::create(service.get_executor(),
-				  [c = std::move(c)](bs::error_code ec) mutable {
-				    std::move(*c)(ec);
-				  });
+    oncommit = c;
   }
   watchl.unlock();
-  auto o = new Op(info->target.base_oid, info->target.base_oloc,
-		  std::move(opv), info->target.flags | CEPH_OSD_FLAG_READ,
-		  std::move(oncommit), info->pobjver);
+  Op *o = new Op(info->target.base_oid, info->target.base_oloc,
+		 opv, info->target.flags | CEPH_OSD_FLAG_READ,
+		 oncommit, info->pobjver);
   o->outbl = poutbl;
   o->snapid = info->snap;
   o->snapc = info->snapc;
@@ -570,9 +586,9 @@ void Objecter::_send_linger(LingerOp *info,
 
   if (info->register_tid) {
     // repeat send.  cancel old registration op, if any.
-    std::unique_lock sl(info->session->lock);
+    OSDSession::unique_lock sl(info->session->lock);
     if (info->session->ops.count(info->register_tid)) {
-      auto o = info->session->ops[info->register_tid];
+      Op *o = info->session->ops[info->register_tid];
       _op_cancel_map_check(o);
       _cancel_linger_op(o);
     }
@@ -584,20 +600,17 @@ void Objecter::_send_linger(LingerOp *info,
   logger->inc(l_osdc_linger_send);
 }
 
-void Objecter::_linger_commit(LingerOp *info, bs::error_code ec,
-			      cb::list& outbl)
+void Objecter::_linger_commit(LingerOp *info, int r, ceph::buffer::list& outbl)
 {
-  std::unique_lock wl(info->watch_lock);
+  LingerOp::unique_lock wl(info->watch_lock);
   ldout(cct, 10) << "_linger_commit " << info->linger_id << dendl;
   if (info->on_reg_commit) {
-    info->on_reg_commit->defer(std::move(info->on_reg_commit),
-			       ec, cb::list{});
-    info->on_reg_commit.reset();
+    info->on_reg_commit->complete(r);
+    info->on_reg_commit = NULL;
   }
-  if (ec && info->on_notify_finish) {
-    info->on_notify_finish->defer(std::move(info->on_notify_finish),
-				  ec, cb::list{});
-    info->on_notify_finish.reset();
+  if (r < 0 && info->on_notify_finish) {
+    info->on_notify_finish->complete(r);
+    info->on_notify_finish = nullptr;
   }
 
   // only tell the user the first time we do this
@@ -612,29 +625,27 @@ void Objecter::_linger_commit(LingerOp *info, bs::error_code ec,
       ldout(cct, 10) << "_linger_commit  notify_id=" << info->notify_id
 		     << dendl;
     }
-    catch (cb::error& e) {
+    catch (ceph::buffer::error& e) {
     }
   }
 }
 
-class CB_DoWatchError {
+struct C_DoWatchError : public Context {
   Objecter *objecter;
   Objecter::LingerOp *info;
-  bs::error_code ec;
-public:
-  CB_DoWatchError(Objecter *o, Objecter::LingerOp *i,
-		  bs::error_code ec)
-    : objecter(o), info(i), ec(ec) {
+  int err;
+  C_DoWatchError(Objecter *o, Objecter::LingerOp *i, int r)
+    : objecter(o), info(i), err(r) {
     info->get();
     info->_queued_async();
   }
-  void operator()() {
-    std::unique_lock wl(objecter->rwlock);
+  void finish(int r) override {
+    Objecter::unique_lock wl(objecter->rwlock);
     bool canceled = info->canceled;
     wl.unlock();
 
     if (!canceled) {
-      info->handle(ec, 0, info->get_cookie(), 0, {});
+      info->watch_context->handle_error(info->get_cookie(), err);
     }
 
     info->finished_async();
@@ -642,27 +653,27 @@ public:
   }
 };
 
-bs::error_code Objecter::_normalize_watch_error(bs::error_code ec)
+int Objecter::_normalize_watch_error(int r)
 {
   // translate ENOENT -> ENOTCONN so that a delete->disconnection
   // notification and a failure to reconnect because we raced with
   // the delete appear the same to the user.
-  if (ec == bs::errc::no_such_file_or_directory)
-    ec = bs::error_code(ENOTCONN, osd_category());
-  return ec;
+  if (r == -ENOENT)
+    r = -ENOTCONN;
+  return r;
 }
 
-void Objecter::_linger_reconnect(LingerOp *info, bs::error_code ec)
+void Objecter::_linger_reconnect(LingerOp *info, int r)
 {
-  ldout(cct, 10) << __func__ << " " << info->linger_id << " = " << ec 
+  ldout(cct, 10) << __func__ << " " << info->linger_id << " = " << r
 		 << " (last_error " << info->last_error << ")" << dendl;
-  if (ec) {
-    std::unique_lock wl(info->watch_lock);
+  if (r < 0) {
+    LingerOp::unique_lock wl(info->watch_lock);
     if (!info->last_error) {
-      ec = _normalize_watch_error(ec);
-      info->last_error = ec;
-      if (info->handle) {
-	boost::asio::defer(finish_strand, CB_DoWatchError(this, info, ec));
+      r = _normalize_watch_error(r);
+      info->last_error = r;
+      if (info->watch_context) {
+	finisher->queue(new C_DoWatchError(this, info, r));
       }
     }
     wl.unlock();
@@ -688,48 +699,43 @@ void Objecter::_send_linger_ping(LingerOp *info)
   ldout(cct, 10) << __func__ << " " << info->linger_id << " now " << now
 		 << dendl;
 
-  osdc_opvec opv(1);
+  vector<OSDOp> opv(1);
   opv[0].op.op = CEPH_OSD_OP_WATCH;
   opv[0].op.watch.cookie = info->get_cookie();
   opv[0].op.watch.op = CEPH_OSD_WATCH_OP_PING;
   opv[0].op.watch.gen = info->register_gen;
-  auto onack = new CB_Linger_Ping(this, info);
+  C_Linger_Ping *onack = new C_Linger_Ping(this, info);
   Op *o = new Op(info->target.base_oid, info->target.base_oloc,
-		 std::move(opv), info->target.flags | CEPH_OSD_FLAG_READ,
-		 Op::OpComp::create(service.get_executor(),
-				    [c = std::unique_ptr<CB_Linger_Ping>(onack)]
-				    (bs::error_code ec) mutable {
-				      std::move(*c)(ec);
-				    }),
-		 nullptr, nullptr);
+		 opv, info->target.flags | CEPH_OSD_FLAG_READ,
+		 onack, NULL, NULL);
   o->target = info->target;
   o->should_resend = false;
   _send_op_account(o);
   o->tid = ++last_tid;
   _session_op_assign(info->session, o);
-  onack->sent = now;
   _send_op(o);
   info->ping_tid = o->tid;
 
+  onack->sent = now;
   logger->inc(l_osdc_linger_ping);
 }
 
-void Objecter::_linger_ping(LingerOp *info, bs::error_code ec, ceph::coarse_mono_time sent,
+void Objecter::_linger_ping(LingerOp *info, int r, ceph::coarse_mono_time sent,
 			    uint32_t register_gen)
 {
-  std::unique_lock l(info->watch_lock);
+  LingerOp::unique_lock l(info->watch_lock);
   ldout(cct, 10) << __func__ << " " << info->linger_id
-		 << " sent " << sent << " gen " << register_gen << " = " << ec
+		 << " sent " << sent << " gen " << register_gen << " = " << r
 		 << " (last_error " << info->last_error
 		 << " register_gen " << info->register_gen << ")" << dendl;
   if (info->register_gen == register_gen) {
-    if (!ec) {
+    if (r == 0) {
       info->watch_valid_thru = sent;
-    } else if (ec && !info->last_error) {
-      ec = _normalize_watch_error(ec);
-      info->last_error = ec;
-      if (info->handle) {
-	boost::asio::defer(finish_strand, CB_DoWatchError(this, info, ec));
+    } else if (r < 0 && !info->last_error) {
+      r = _normalize_watch_error(r);
+      info->last_error = r;
+      if (info->watch_context) {
+	finisher->queue(new C_DoWatchError(this, info, r));
       }
     }
   } else {
@@ -737,10 +743,9 @@ void Objecter::_linger_ping(LingerOp *info, bs::error_code ec, ceph::coarse_mono
   }
 }
 
-tl::expected<ceph::timespan,
-	     bs::error_code> Objecter::linger_check(LingerOp *info)
+int Objecter::linger_check(LingerOp *info)
 {
-  std::shared_lock l(info->watch_lock);
+  LingerOp::shared_lock l(info->watch_lock);
 
   ceph::coarse_mono_time stamp = info->watch_valid_thru;
   if (!info->watch_pending_async.empty())
@@ -751,9 +756,10 @@ tl::expected<ceph::timespan,
 		 << " err " << info->last_error
 		 << " age " << age << dendl;
   if (info->last_error)
-    return tl::unexpected(info->last_error);
+    return info->last_error;
   // return a safe upper bound (we are truncating to ms)
-  return age;
+  return
+    1 + std::chrono::duration_cast<std::chrono::milliseconds>(age).count();
 }
 
 void Objecter::linger_cancel(LingerOp *info)
@@ -769,7 +775,7 @@ void Objecter::_linger_cancel(LingerOp *info)
   ldout(cct, 20) << __func__ << " linger_id=" << info->linger_id << dendl;
   if (!info->canceled) {
     OSDSession *s = info->session;
-    std::unique_lock sl(s->lock);
+    OSDSession::unique_lock sl(s->lock);
     _session_linger_op_remove(s, info);
     sl.unlock();
 
@@ -790,7 +796,7 @@ Objecter::LingerOp *Objecter::linger_register(const object_t& oid,
 					      const object_locator_t& oloc,
 					      int flags)
 {
-  auto info = new LingerOp(this);
+  LingerOp *info = new LingerOp(this);
   info->target.base_oid = oid;
   info->target.base_oloc = oloc;
   if (info->target.base_oloc.key == oid)
@@ -818,8 +824,8 @@ ceph_tid_t Objecter::linger_watch(LingerOp *info,
 				  ObjectOperation& op,
 				  const SnapContext& snapc,
 				  real_time mtime,
-				  cb::list& inbl,
-				  decltype(info->on_reg_commit)&& oncommit,
+				  ceph::buffer::list& inbl,
+				  Context *oncommit,
 				  version_t *objver)
 {
   info->is_watch = true;
@@ -828,8 +834,9 @@ ceph_tid_t Objecter::linger_watch(LingerOp *info,
   info->target.flags |= CEPH_OSD_FLAG_WRITE;
   info->ops = op.ops;
   info->inbl = inbl;
+  info->poutbl = NULL;
   info->pobjver = objver;
-  info->on_reg_commit = std::move(oncommit);
+  info->on_reg_commit = oncommit;
 
   info->ctx_budget = take_linger_budget(info);
 
@@ -837,34 +844,34 @@ ceph_tid_t Objecter::linger_watch(LingerOp *info,
   _linger_submit(info, sul);
   logger->inc(l_osdc_linger_active);
 
-  op.clear();
   return info->linger_id;
 }
 
 ceph_tid_t Objecter::linger_notify(LingerOp *info,
 				   ObjectOperation& op,
-				   snapid_t snap, cb::list& inbl,
-				   decltype(LingerOp::on_reg_commit)&& onfinish,
+				   snapid_t snap, ceph::buffer::list& inbl,
+				   ceph::buffer::list *poutbl,
+				   Context *onfinish,
 				   version_t *objver)
 {
   info->snap = snap;
   info->target.flags |= CEPH_OSD_FLAG_READ;
   info->ops = op.ops;
   info->inbl = inbl;
+  info->poutbl = poutbl;
   info->pobjver = objver;
-  info->on_reg_commit = std::move(onfinish);
-  info->ctx_budget = take_linger_budget(info);
+  info->on_reg_commit = onfinish;
 
+  info->ctx_budget = take_linger_budget(info);
+  
   shunique_lock sul(rwlock, ceph::acquire_unique);
   _linger_submit(info, sul);
   logger->inc(l_osdc_linger_active);
 
-  op.clear();
   return info->linger_id;
 }
 
-void Objecter::_linger_submit(LingerOp *info,
-			      ceph::shunique_lock<ceph::shared_mutex>& sul)
+void Objecter::_linger_submit(LingerOp *info, shunique_lock& sul)
 {
   ceph_assert(sul.owns_lock() && sul.mutex() == &rwlock);
   ceph_assert(info->linger_id);
@@ -877,7 +884,7 @@ void Objecter::_linger_submit(LingerOp *info,
   // Create LingerOp<->OSDSession relation
   int r = _get_session(info->target.osd, &s, sul);
   ceph_assert(r == 0);
-  unique_lock sl(s->lock);
+  OSDSession::unique_lock sl(s->lock);
   _session_linger_op_assign(s, info);
   sl.unlock();
   put_session(s);
@@ -885,17 +892,17 @@ void Objecter::_linger_submit(LingerOp *info,
   _send_linger(info, sul);
 }
 
-struct CB_DoWatchNotify {
+struct C_DoWatchNotify : public Context {
   Objecter *objecter;
   Objecter::LingerOp *info;
   MWatchNotify *msg;
-  CB_DoWatchNotify(Objecter *o, Objecter::LingerOp *i, MWatchNotify *m)
+  C_DoWatchNotify(Objecter *o, Objecter::LingerOp *i, MWatchNotify *m)
     : objecter(o), info(i), msg(m) {
     info->get();
     info->_queued_async();
     msg->get();
   }
-  void operator()() {
+  void finish(int r) override {
     objecter->_do_watch_notify(info, msg);
   }
 };
@@ -912,13 +919,12 @@ void Objecter::handle_watch_notify(MWatchNotify *m)
     ldout(cct, 7) << __func__ << " cookie " << m->cookie << " dne" << dendl;
     return;
   }
-  std::unique_lock wl(info->watch_lock);
+  LingerOp::unique_lock wl(info->watch_lock);
   if (m->opcode == CEPH_WATCH_EVENT_DISCONNECT) {
     if (!info->last_error) {
-      info->last_error = bs::error_code(ENOTCONN, osd_category());
-      if (info->handle) {
-	boost::asio::defer(finish_strand, CB_DoWatchError(this, info,
-							  info->last_error));
+      info->last_error = -ENOTCONN;
+      if (info->watch_context) {
+	finisher->queue(new C_DoWatchError(this, info, -ENOTCONN));
       }
     }
   } else if (!info->is_watch) {
@@ -930,16 +936,15 @@ void Objecter::handle_watch_notify(MWatchNotify *m)
       ldout(cct, 10) << __func__ << " reply notify " << m->notify_id
 		     << " != " << info->notify_id << ", ignoring" << dendl;
     } else if (info->on_notify_finish) {
-      info->on_notify_finish->defer(
-	std::move(info->on_notify_finish),
-	osdcode(m->return_code), std::move(m->get_data()));
+      info->notify_result_bl->claim(m->get_data());
+      info->on_notify_finish->complete(m->return_code);
 
       // if we race with reconnect we might get a second notify; only
       // notify the caller once!
-      info->on_notify_finish = nullptr;
+      info->on_notify_finish = NULL;
     }
   } else {
-    boost::asio::defer(finish_strand, CB_DoWatchNotify(this, info, m));
+    finisher->queue(new C_DoWatchNotify(this, info, m));
   }
 }
 
@@ -957,14 +962,15 @@ void Objecter::_do_watch_notify(LingerOp *info, MWatchNotify *m)
 
   // notify completion?
   ceph_assert(info->is_watch);
-  ceph_assert(info->handle);
+  ceph_assert(info->watch_context);
   ceph_assert(m->opcode != CEPH_WATCH_EVENT_DISCONNECT);
 
   l.unlock();
 
   switch (m->opcode) {
   case CEPH_WATCH_EVENT_NOTIFY:
-    info->handle({}, m->notify_id, m->cookie, m->notifier_gid, std::move(m->bl));
+    info->watch_context->handle_notify(m->notify_id, m->cookie,
+				       m->notifier_gid, m->bl);
     break;
   }
 
@@ -1030,18 +1036,18 @@ void Objecter::_scan_requests(
   map<ceph_tid_t, Op*>& need_resend,
   list<LingerOp*>& need_resend_linger,
   map<ceph_tid_t, CommandOp*>& need_resend_command,
-  ceph::shunique_lock<ceph::shared_mutex>& sul)
+  shunique_lock& sul)
 {
   ceph_assert(sul.owns_lock() && sul.mutex() == &rwlock);
 
   list<LingerOp*> unregister_lingers;
 
-  std::unique_lock sl(s->lock);
+  OSDSession::unique_lock sl(s->lock);
 
   // check for changed linger mappings (_before_ regular ops)
-  auto lp = s->linger_ops.begin();
+  map<ceph_tid_t,LingerOp*>::iterator lp = s->linger_ops.begin();
   while (lp != s->linger_ops.end()) {
-    auto op = lp->second;
+    LingerOp *op = lp->second;
     ceph_assert(op->session == s);
     // check_linger_pool_dne() may touch linger_ops; prevent iterator
     // invalidation
@@ -1074,7 +1080,7 @@ void Objecter::_scan_requests(
   }
 
   // check for changed request mappings
-  auto p = s->ops.begin();
+  map<ceph_tid_t,Op*>::iterator p = s->ops.begin();
   while (p != s->ops.end()) {
     Op *op = p->second;
     ++p;   // check_op_pool_dne() may touch ops; prevent iterator invalidation
@@ -1103,9 +1109,9 @@ void Objecter::_scan_requests(
   }
 
   // commands
-  auto cp = s->command_ops.begin();
+  map<ceph_tid_t,CommandOp*>::iterator cp = s->command_ops.begin();
   while (cp != s->command_ops.end()) {
-    auto c = cp->second;
+    CommandOp *c = cp->second;
     ++cp;
     ldout(cct, 10) << " checking command " << c->tid << dendl;
     bool force_resend_writes = cluster_full;
@@ -1134,7 +1140,7 @@ void Objecter::_scan_requests(
 
   sl.unlock();
 
-  for (auto iter = unregister_lingers.begin();
+  for (list<LingerOp*>::iterator iter = unregister_lingers.begin();
        iter != unregister_lingers.end();
        ++iter) {
     _linger_cancel(*iter);
@@ -1144,7 +1150,7 @@ void Objecter::_scan_requests(
 
 void Objecter::handle_osd_map(MOSDMap *m)
 {
-  ceph::shunique_lock sul(rwlock, acquire_unique);
+  shunique_lock sul(rwlock, acquire_unique);
   if (!initialized)
     return;
 
@@ -1161,7 +1167,8 @@ void Objecter::handle_osd_map(MOSDMap *m)
   bool was_pausewr = osdmap->test_flag(CEPH_OSDMAP_PAUSEWR) || cluster_full ||
     _osdmap_has_pool_full();
   map<int64_t, bool> pool_full_map;
-  for (auto it = osdmap->get_pools().begin();
+  for (map<int64_t, pg_pool_t>::const_iterator it
+	 = osdmap->get_pools().begin();
        it != osdmap->get_pools().end(); ++it)
     pool_full_map[it->first] = _osdmap_pool_full(it->second);
 
@@ -1234,9 +1241,9 @@ void Objecter::handle_osd_map(MOSDMap *m)
 	_scan_requests(homeless_session, skipped_map, cluster_full,
 		       &pool_full_map, need_resend,
 		       need_resend_linger, need_resend_command, sul);
-	for (auto p = osd_sessions.begin();
+	for (map<int,OSDSession*>::iterator p = osd_sessions.begin();
 	     p != osd_sessions.end(); ) {
-	  auto s = p->second;
+	  OSDSession *s = p->second;
 	  _scan_requests(s, skipped_map, cluster_full,
 			 &pool_full_map, need_resend,
 			 need_resend_linger, need_resend_command, sul);
@@ -1255,7 +1262,7 @@ void Objecter::handle_osd_map(MOSDMap *m)
     } else {
       // first map.  we want the full thing.
       if (m->maps.count(m->get_last())) {
-	for (auto p = osd_sessions.begin();
+	for (map<int,OSDSession*>::iterator p = osd_sessions.begin();
 	     p != osd_sessions.end(); ++p) {
 	  OSDSession *s = p->second;
 	  _scan_requests(s, false, false, NULL, need_resend,
@@ -1306,10 +1313,10 @@ void Objecter::handle_osd_map(MOSDMap *m)
   }
 
   // resend requests
-  for (auto p = need_resend.begin();
+  for (map<ceph_tid_t, Op*>::iterator p = need_resend.begin();
        p != need_resend.end(); ++p) {
-    auto op = p->second;
-    auto s = op->session;
+    Op *op = p->second;
+    OSDSession *s = op->session;
     bool mapped_session = false;
     if (!s) {
       int r = _map_session(&op->target, &s, sul);
@@ -1318,7 +1325,7 @@ void Objecter::handle_osd_map(MOSDMap *m)
     } else {
       get_session(s);
     }
-    std::unique_lock sl(s->lock);
+    OSDSession::unique_lock sl(s->lock);
     if (mapped_session) {
       _session_op_assign(s, op);
     }
@@ -1334,7 +1341,7 @@ void Objecter::handle_osd_map(MOSDMap *m)
     sl.unlock();
     put_session(s);
   }
-  for (auto p = need_resend_linger.begin();
+  for (list<LingerOp*>::iterator p = need_resend_linger.begin();
        p != need_resend_linger.end(); ++p) {
     LingerOp *op = *p;
     ceph_assert(op->session);
@@ -1343,9 +1350,9 @@ void Objecter::handle_osd_map(MOSDMap *m)
       _send_linger(op, sul);
     }
   }
-  for (auto p = need_resend_command.begin();
+  for (map<ceph_tid_t,CommandOp*>::iterator p = need_resend_command.begin();
        p != need_resend_command.end(); ++p) {
-    auto c = p->second;
+    CommandOp *c = p->second;
     if (c->target.osd >= 0) {
       _assign_command_session(c, sul);
       if (c->session && !c->session->is_homeless()) {
@@ -1357,12 +1364,14 @@ void Objecter::handle_osd_map(MOSDMap *m)
   _dump_active();
 
   // finish any Contexts that were waiting on a map update
-  auto p = waiting_for_map.begin();
+  map<epoch_t,list< pair< Context*, int > > >::iterator p =
+    waiting_for_map.begin();
   while (p != waiting_for_map.end() &&
 	 p->first <= osdmap->get_epoch()) {
     //go through the list and call the onfinish methods
-    for (auto& [c, ec] : p->second) {
-      ca::post(std::move(c), ec);
+    for (list<pair<Context*, int> >::iterator i = p->second.begin();
+	 i != p->second.end(); ++i) {
+      i->first->complete(i->second);
     }
     waiting_for_map.erase(p++);
   }
@@ -1428,20 +1437,19 @@ void Objecter::emit_blacklist_events(const OSDMap &old_osd_map,
 
 // op pool check
 
-void Objecter::CB_Op_Map_Latest::operator()(bs::error_code e,
-					    version_t latest, version_t)
+void Objecter::C_Op_Map_Latest::finish(int r)
 {
-  if (e == bs::errc::resource_unavailable_try_again ||
-      e == bs::errc::operation_canceled)
+  if (r == -EAGAIN || r == -ECANCELED)
     return;
 
   lgeneric_subdout(objecter->cct, objecter, 10)
-    << "op_map_latest r=" << e << " tid=" << tid
+    << "op_map_latest r=" << r << " tid=" << tid
     << " latest " << latest << dendl;
 
-  unique_lock wl(objecter->rwlock);
+  Objecter::unique_lock wl(objecter->rwlock);
 
-  auto iter = objecter->check_latest_map_ops.find(tid);
+  map<ceph_tid_t, Op*>::iterator iter =
+    objecter->check_latest_map_ops.find(tid);
   if (iter == objecter->check_latest_map_ops.end()) {
     lgeneric_subdout(objecter->cct, objecter, 10)
       << "op_map_latest op "<< tid << " not found" << dendl;
@@ -1457,7 +1465,7 @@ void Objecter::CB_Op_Map_Latest::operator()(bs::error_code e,
   if (op->map_dne_bound == 0)
     op->map_dne_bound = latest;
 
-  unique_lock sl(op->session->lock, defer_lock);
+  OSDSession::unique_lock sl(op->session->lock, defer_lock);
   objecter->_check_op_pool_dne(op, &sl);
 
   op->put();
@@ -1511,7 +1519,7 @@ int Objecter::pool_snap_list(int64_t poolid, vector<uint64_t> *snaps)
   const pg_pool_t *pi = osdmap->get_pg_pool(poolid);
   if (!pi)
     return -ENOENT;
-  for (auto p = pi->snaps.begin();
+  for (map<snapid_t,pool_snap_info_t>::const_iterator p = pi->snaps.begin();
        p != pi->snaps.end();
        ++p) {
     snaps->push_back(p->first);
@@ -1520,7 +1528,7 @@ int Objecter::pool_snap_list(int64_t poolid, vector<uint64_t> *snaps)
 }
 
 // sl may be unlocked.
-void Objecter::_check_op_pool_dne(Op *op, std::unique_lock<ceph::shared_mutex> *sl)
+void Objecter::_check_op_pool_dne(Op *op, unique_lock *sl)
 {
   // rwlock is locked unique
 
@@ -1543,9 +1551,9 @@ void Objecter::_check_op_pool_dne(Op *op, std::unique_lock<ceph::shared_mutex> *
       ldout(cct, 10) << "check_op_pool_dne tid " << op->tid
 		     << " concluding pool " << op->target.base_pgid.pool()
 		     << " dne" << dendl;
-      if (op->has_completion()) {
+      if (op->onfinish) {
 	num_in_flight--;
-	op->complete(osdc_errc::pool_dne, -ENOENT);
+	op->onfinish->complete(-ENOENT);
       }
 
       OSDSession *s = op->session;
@@ -1576,14 +1584,16 @@ void Objecter::_send_op_map_check(Op *op)
   if (check_latest_map_ops.count(op->tid) == 0) {
     op->get();
     check_latest_map_ops[op->tid] = op;
-    monc->get_version("osdmap", CB_Op_Map_Latest(this, op->tid));
+    C_Op_Map_Latest *c = new C_Op_Map_Latest(this, op->tid);
+    monc->get_version("osdmap", &c->latest, NULL, c);
   }
 }
 
 void Objecter::_op_cancel_map_check(Op *op)
 {
   // rwlock is locked unique
-  auto iter = check_latest_map_ops.find(op->tid);
+  map<ceph_tid_t, Op*>::iterator iter =
+    check_latest_map_ops.find(op->tid);
   if (iter != check_latest_map_ops.end()) {
     Op *op = iter->second;
     op->put();
@@ -1593,24 +1603,22 @@ void Objecter::_op_cancel_map_check(Op *op)
 
 // linger pool check
 
-void Objecter::CB_Linger_Map_Latest::operator()(bs::error_code e,
-						version_t latest,
-						version_t)
+void Objecter::C_Linger_Map_Latest::finish(int r)
 {
-  if (e == bs::errc::resource_unavailable_try_again ||
-      e == bs::errc::operation_canceled) {
+  if (r == -EAGAIN || r == -ECANCELED) {
     // ignore callback; we will retry in resend_mon_ops()
     return;
   }
 
   unique_lock wl(objecter->rwlock);
 
-  auto iter = objecter->check_latest_map_lingers.find(linger_id);
+  map<uint64_t, LingerOp*>::iterator iter =
+    objecter->check_latest_map_lingers.find(linger_id);
   if (iter == objecter->check_latest_map_lingers.end()) {
     return;
   }
 
-  auto op = iter->second;
+  LingerOp *op = iter->second;
   objecter->check_latest_map_lingers.erase(iter);
 
   if (op->map_dne_bound == 0)
@@ -1645,15 +1653,13 @@ void Objecter::_check_linger_pool_dne(LingerOp *op, bool *need_unregister)
   }
   if (op->map_dne_bound > 0) {
     if (osdmap->get_epoch() >= op->map_dne_bound) {
-      std::unique_lock wl{op->watch_lock};
+      LingerOp::unique_lock wl{op->watch_lock};
       if (op->on_reg_commit) {
-	op->on_reg_commit->defer(std::move(op->on_reg_commit),
-				 osdc_errc::pool_dne, cb::list{});
+	op->on_reg_commit->complete(-ENOENT);
 	op->on_reg_commit = nullptr;
       }
       if (op->on_notify_finish) {
-	op->on_notify_finish->defer(std::move(op->on_notify_finish),
-				    osdc_errc::pool_dne, cb::list{});
+        op->on_notify_finish->complete(-ENOENT);
         op->on_notify_finish = nullptr;
       }
       *need_unregister = true;
@@ -1669,7 +1675,8 @@ void Objecter::_send_linger_map_check(LingerOp *op)
   if (check_latest_map_lingers.count(op->linger_id) == 0) {
     op->get();
     check_latest_map_lingers[op->linger_id] = op;
-    monc->get_version("osdmap", CB_Linger_Map_Latest(this, op->linger_id));
+    C_Linger_Map_Latest *c = new C_Linger_Map_Latest(this, op->linger_id);
+    monc->get_version("osdmap", &c->latest, NULL, c);
   }
 }
 
@@ -1677,7 +1684,8 @@ void Objecter::_linger_cancel_map_check(LingerOp *op)
 {
   // rwlock is locked unique
 
-  auto iter = check_latest_map_lingers.find(op->linger_id);
+  map<uint64_t, LingerOp*>::iterator iter =
+    check_latest_map_lingers.find(op->linger_id);
   if (iter != check_latest_map_lingers.end()) {
     LingerOp *op = iter->second;
     op->put();
@@ -1687,29 +1695,28 @@ void Objecter::_linger_cancel_map_check(LingerOp *op)
 
 // command pool check
 
-void Objecter::CB_Command_Map_Latest::operator()(bs::error_code e,
-						 version_t latest, version_t)
+void Objecter::C_Command_Map_Latest::finish(int r)
 {
-  if (e == bs::errc::resource_unavailable_try_again ||
-      e == bs::errc::operation_canceled) {
+  if (r == -EAGAIN || r == -ECANCELED) {
     // ignore callback; we will retry in resend_mon_ops()
     return;
   }
 
   unique_lock wl(objecter->rwlock);
 
-  auto iter = objecter->check_latest_map_commands.find(tid);
+  map<uint64_t, CommandOp*>::iterator iter =
+    objecter->check_latest_map_commands.find(tid);
   if (iter == objecter->check_latest_map_commands.end()) {
     return;
   }
 
-  auto c = iter->second;
+  CommandOp *c = iter->second;
   objecter->check_latest_map_commands.erase(iter);
 
   if (c->map_dne_bound == 0)
     c->map_dne_bound = latest;
 
-  unique_lock sul(c->session->lock);
+  OSDSession::unique_lock sul(c->session->lock);
   objecter->_check_command_map_dne(c);
   sul.unlock();
 
@@ -1727,8 +1734,7 @@ void Objecter::_check_command_map_dne(CommandOp *c)
 		 << dendl;
   if (c->map_dne_bound > 0) {
     if (osdmap->get_epoch() >= c->map_dne_bound) {
-      _finish_command(c, osdcode(c->map_check_error),
-		      std::move(c->map_check_error_str), {});
+      _finish_command(c, c->map_check_error, c->map_check_error_str);
     }
   } else {
     _send_command_map_check(c);
@@ -1744,7 +1750,8 @@ void Objecter::_send_command_map_check(CommandOp *c)
   if (check_latest_map_commands.count(c->tid) == 0) {
     c->get();
     check_latest_map_commands[c->tid] = c;
-    monc->get_version("osdmap", CB_Command_Map_Latest(this, c->tid));
+    C_Command_Map_Latest *f = new C_Command_Map_Latest(this, c->tid);
+    monc->get_version("osdmap", &f->latest, NULL, f);
   }
 }
 
@@ -1752,9 +1759,10 @@ void Objecter::_command_cancel_map_check(CommandOp *c)
 {
   // rwlock is locked uniqe
 
-  auto iter = check_latest_map_commands.find(c->tid);
+  map<uint64_t, CommandOp*>::iterator iter =
+    check_latest_map_commands.find(c->tid);
   if (iter != check_latest_map_commands.end()) {
-    auto c = iter->second;
+    CommandOp *c = iter->second;
     c->put();
     check_latest_map_commands.erase(iter);
   }
@@ -1767,8 +1775,7 @@ void Objecter::_command_cancel_map_check(CommandOp *c)
  * @returns 0 on success, or -EAGAIN if the lock context requires
  * promotion to write.
  */
-int Objecter::_get_session(int osd, OSDSession **session,
-			   shunique_lock<ceph::shared_mutex>& sul)
+int Objecter::_get_session(int osd, OSDSession **session, shunique_lock& sul)
 {
   ceph_assert(sul && sul.mutex() == &rwlock);
 
@@ -1779,9 +1786,9 @@ int Objecter::_get_session(int osd, OSDSession **session,
     return 0;
   }
 
-  auto p = osd_sessions.find(osd);
+  map<int,OSDSession*>::iterator p = osd_sessions.find(osd);
   if (p != osd_sessions.end()) {
-    auto s = p->second;
+    OSDSession *s = p->second;
     s->get();
     *session = s;
     ldout(cct, 20) << __func__ << " s=" << s << " osd=" << osd << " "
@@ -1791,7 +1798,7 @@ int Objecter::_get_session(int osd, OSDSession **session,
   if (!sul.owns_lock()) {
     return -EAGAIN;
   }
-  auto s = new OSDSession(cct, osd);
+  OSDSession *s = new OSDSession(cct, osd);
   osd_sessions[osd] = s;
   s->con = messenger->connect_to_osd(osdmap->get_addrs(osd));
   s->con->set_priv(RefCountedPtr{s});
@@ -1853,28 +1860,28 @@ void Objecter::close_session(OSDSession *s)
     s->con->mark_down();
     logger->inc(l_osdc_osd_session_close);
   }
-  unique_lock sl(s->lock);
+  OSDSession::unique_lock sl(s->lock);
 
   std::list<LingerOp*> homeless_lingers;
   std::list<CommandOp*> homeless_commands;
   std::list<Op*> homeless_ops;
 
   while (!s->linger_ops.empty()) {
-    auto i = s->linger_ops.begin();
+    std::map<uint64_t, LingerOp*>::iterator i = s->linger_ops.begin();
     ldout(cct, 10) << " linger_op " << i->first << dendl;
     homeless_lingers.push_back(i->second);
     _session_linger_op_remove(s, i->second);
   }
 
   while (!s->ops.empty()) {
-    auto i = s->ops.begin();
+    std::map<ceph_tid_t, Op*>::iterator i = s->ops.begin();
     ldout(cct, 10) << " op " << i->first << dendl;
     homeless_ops.push_back(i->second);
     _session_op_remove(s, i->second);
   }
 
   while (!s->command_ops.empty()) {
-    auto i = s->command_ops.begin();
+    std::map<ceph_tid_t, CommandOp*>::iterator i = s->command_ops.begin();
     ldout(cct, 10) << " command_op " << i->first << dendl;
     homeless_commands.push_back(i->second);
     _session_command_op_remove(s, i->second);
@@ -1886,16 +1893,16 @@ void Objecter::close_session(OSDSession *s)
 
   // Assign any leftover ops to the homeless session
   {
-    unique_lock hsl(homeless_session->lock);
-    for (auto i = homeless_lingers.begin();
+    OSDSession::unique_lock hsl(homeless_session->lock);
+    for (std::list<LingerOp*>::iterator i = homeless_lingers.begin();
 	 i != homeless_lingers.end(); ++i) {
       _session_linger_op_assign(homeless_session, *i);
     }
-    for (auto i = homeless_ops.begin();
+    for (std::list<Op*>::iterator i = homeless_ops.begin();
 	 i != homeless_ops.end(); ++i) {
       _session_op_assign(homeless_session, *i);
     }
-    for (auto i = homeless_commands.begin();
+    for (std::list<CommandOp*>::iterator i = homeless_commands.begin();
 	 i != homeless_commands.end(); ++i) {
       _session_command_op_assign(homeless_session, *i);
     }
@@ -1912,29 +1919,55 @@ void Objecter::wait_for_osd_map()
     return;
   }
 
-  ca::waiter<bs::error_code> w;
-  waiting_for_map[0].emplace_back(OpCompletion::create(
-				    service.get_executor(),
-				    w.ref()),
-				  bs::error_code{});
+  // Leave this since it goes with C_SafeCond
+  ceph::mutex lock = ceph::make_mutex("");
+  ceph::condition_variable cond;
+  bool done;
+  std::unique_lock mlock{lock};
+  C_SafeCond *context = new C_SafeCond(lock, cond, &done, NULL);
+  waiting_for_map[0].push_back(pair<Context*, int>(context, 0));
   l.unlock();
-  w.wait();
+  cond.wait(mlock, [&done] { return done; });
+}
+
+struct C_Objecter_GetVersion : public Context {
+  Objecter *objecter;
+  uint64_t oldest, newest;
+  Context *fin;
+  C_Objecter_GetVersion(Objecter *o, Context *c)
+    : objecter(o), oldest(0), newest(0), fin(c) {}
+  void finish(int r) override {
+    if (r >= 0) {
+      objecter->get_latest_version(oldest, newest, fin);
+    } else if (r == -EAGAIN) { // try again as instructed
+      objecter->wait_for_latest_osdmap(fin);
+    } else {
+      // it doesn't return any other error codes!
+      ceph_abort();
+    }
+  }
+};
+
+void Objecter::wait_for_latest_osdmap(Context *fin)
+{
+  ldout(cct, 10) << __func__ << dendl;
+  C_Objecter_GetVersion *c = new C_Objecter_GetVersion(this, fin);
+  monc->get_version("osdmap", &c->newest, &c->oldest, c);
 }
 
-void Objecter::_get_latest_version(epoch_t oldest, epoch_t newest,
-				   std::unique_ptr<OpCompletion> fin,
-				   std::unique_lock<ceph::shared_mutex>&& l)
+void Objecter::get_latest_version(epoch_t oldest, epoch_t newest, Context *fin)
 {
-  ceph_assert(fin);
+  unique_lock wl(rwlock);
   if (osdmap->get_epoch() >= newest) {
     ldout(cct, 10) << __func__ << " latest " << newest << ", have it" << dendl;
-    l.unlock();
-    ca::defer(std::move(fin), bs::error_code{});
-  } else {
-    ldout(cct, 10) << __func__ << " latest " << newest << ", waiting" << dendl;
-    _wait_for_new_map(std::move(fin), newest, bs::error_code{});
-    l.unlock();
+    wl.unlock();
+    if (fin)
+      fin->complete(0);
+    return;
   }
+
+  ldout(cct, 10) << __func__ << " latest " << newest << ", waiting" << dendl;
+  _wait_for_new_map(fin, newest, 0);
 }
 
 void Objecter::maybe_request_map()
@@ -1963,11 +1996,10 @@ void Objecter::_maybe_request_map()
   }
 }
 
-void Objecter::_wait_for_new_map(std::unique_ptr<OpCompletion> c, epoch_t epoch,
-				 bs::error_code ec)
+void Objecter::_wait_for_new_map(Context *c, epoch_t epoch, int err)
 {
   // rwlock is locked unique
-  waiting_for_map[epoch].emplace_back(std::move(c), ec);
+  waiting_for_map[epoch].push_back(pair<Context *, int>(c, err));
   _maybe_request_map();
 }
 
@@ -1992,6 +2024,16 @@ bool Objecter::have_map(const epoch_t epoch)
   }
 }
 
+bool Objecter::wait_for_map(epoch_t epoch, Context *c, int err)
+{
+  unique_lock wl(rwlock);
+  if (osdmap->get_epoch() >= epoch) {
+    return true;
+  }
+  _wait_for_new_map(c, epoch, err);
+  return false;
+}
+
 void Objecter::_kick_requests(OSDSession *session,
 			      map<uint64_t, LingerOp *>& lresend)
 {
@@ -2003,7 +2045,8 @@ void Objecter::_kick_requests(OSDSession *session,
 
   // resend ops
   map<ceph_tid_t,Op*> resend;  // resend in tid order
-  for (auto p = session->ops.begin(); p != session->ops.end();) {
+  for (map<ceph_tid_t, Op*>::iterator p = session->ops.begin();
+       p != session->ops.end();) {
     Op *op = p->second;
     ++p;
     if (op->should_resend) {
@@ -2023,7 +2066,7 @@ void Objecter::_kick_requests(OSDSession *session,
 
   // resend lingers
   logger->inc(l_osdc_linger_resend, session->linger_ops.size());
-  for (auto j = session->linger_ops.begin();
+  for (map<ceph_tid_t, LingerOp*>::iterator j = session->linger_ops.begin();
        j != session->linger_ops.end(); ++j) {
     LingerOp *op = j->second;
     op->get();
@@ -2034,7 +2077,7 @@ void Objecter::_kick_requests(OSDSession *session,
   // resend commands
   logger->inc(l_osdc_command_resend, session->command_ops.size());
   map<uint64_t,CommandOp*> cresend;  // resend in order
-  for (auto k = session->command_ops.begin();
+  for (map<ceph_tid_t, CommandOp*>::iterator k = session->command_ops.begin();
        k != session->command_ops.end(); ++k) {
     cresend[k->first] = k->second;
   }
@@ -2045,7 +2088,7 @@ void Objecter::_kick_requests(OSDSession *session,
 }
 
 void Objecter::_linger_ops_resend(map<uint64_t, LingerOp *>& lresend,
-				  unique_lock<ceph::shared_mutex>& ul)
+				  unique_lock& ul)
 {
   ceph_assert(ul.owns_lock());
   shunique_lock sul(std::move(ul));
@@ -2092,13 +2135,15 @@ void Objecter::tick()
 
   unsigned laggy_ops = 0;
 
-  for (auto siter = osd_sessions.begin();
+  for (map<int,OSDSession*>::iterator siter = osd_sessions.begin();
        siter != osd_sessions.end(); ++siter) {
-    auto s = siter->second;
-    scoped_lock l(s->lock);
+    OSDSession *s = siter->second;
+    OSDSession::lock_guard l(s->lock);
     bool found = false;
-    for (auto p = s->ops.begin(); p != s->ops.end(); ++p) {
-      auto op = p->second;
+    for (map<ceph_tid_t,Op*>::iterator p = s->ops.begin();
+	p != s->ops.end();
+	++p) {
+      Op *op = p->second;
       ceph_assert(op->session);
       if (op->stamp < cutoff) {
 	ldout(cct, 2) << " tid " << p->first << " on osd." << op->session->osd
@@ -2107,11 +2152,11 @@ void Objecter::tick()
 	++laggy_ops;
       }
     }
-    for (auto p = s->linger_ops.begin();
+    for (map<uint64_t,LingerOp*>::iterator p = s->linger_ops.begin();
 	p != s->linger_ops.end();
 	++p) {
-      auto op = p->second;
-      std::unique_lock wl(op->watch_lock);
+      LingerOp *op = p->second;
+      LingerOp::unique_lock wl(op->watch_lock);
       ceph_assert(op->session);
       ldout(cct, 10) << " pinging osd that serves lingering tid " << p->first
 		     << " (osd." << op->session->osd << ")" << dendl;
@@ -2119,10 +2164,10 @@ void Objecter::tick()
       if (op->is_watch && op->registered && !op->last_error)
 	_send_linger_ping(op);
     }
-    for (auto p = s->command_ops.begin();
+    for (map<uint64_t,CommandOp*>::iterator p = s->command_ops.begin();
 	p != s->command_ops.end();
 	++p) {
-      auto op = p->second;
+      CommandOp *op = p->second;
       ceph_assert(op->session);
       ldout(cct, 10) << " pinging osd that serves command tid " << p->first
 		     << " (osd." << op->session->osd << ")" << dendl;
@@ -2141,7 +2186,9 @@ void Objecter::tick()
   if (!toping.empty()) {
     // send a ping to these osds, to ensure we detect any session resets
     // (osd reply message policy is lossy)
-    for (auto i = toping.begin(); i != toping.end(); ++i) {
+    for (set<OSDSession*>::const_iterator i = toping.begin();
+	 i != toping.end();
+	 ++i) {
       (*i)->con->send_message(new MPing);
     }
   }
@@ -2159,37 +2206,48 @@ void Objecter::resend_mon_ops()
 
   ldout(cct, 10) << "resend_mon_ops" << dendl;
 
-  for (auto p = poolstat_ops.begin(); p != poolstat_ops.end(); ++p) {
+  for (map<ceph_tid_t,PoolStatOp*>::iterator p = poolstat_ops.begin();
+       p != poolstat_ops.end();
+       ++p) {
     _poolstat_submit(p->second);
     logger->inc(l_osdc_poolstat_resend);
   }
 
-  for (auto p = statfs_ops.begin(); p != statfs_ops.end(); ++p) {
+  for (map<ceph_tid_t,StatfsOp*>::iterator p = statfs_ops.begin();
+       p != statfs_ops.end();
+       ++p) {
     _fs_stats_submit(p->second);
     logger->inc(l_osdc_statfs_resend);
   }
 
-  for (auto p = pool_ops.begin(); p != pool_ops.end(); ++p) {
+  for (map<ceph_tid_t,PoolOp*>::iterator p = pool_ops.begin();
+       p != pool_ops.end();
+       ++p) {
     _pool_op_submit(p->second);
     logger->inc(l_osdc_poolop_resend);
   }
 
-  for (auto p = check_latest_map_ops.begin();
+  for (map<ceph_tid_t, Op*>::iterator p = check_latest_map_ops.begin();
        p != check_latest_map_ops.end();
        ++p) {
-    monc->get_version("osdmap", CB_Op_Map_Latest(this, p->second->tid));
+    C_Op_Map_Latest *c = new C_Op_Map_Latest(this, p->second->tid);
+    monc->get_version("osdmap", &c->latest, NULL, c);
   }
 
-  for (auto p = check_latest_map_lingers.begin();
+  for (map<uint64_t, LingerOp*>::iterator p = check_latest_map_lingers.begin();
        p != check_latest_map_lingers.end();
        ++p) {
-    monc->get_version("osdmap", CB_Linger_Map_Latest(this, p->second->linger_id));
+    C_Linger_Map_Latest *c
+      = new C_Linger_Map_Latest(this, p->second->linger_id);
+    monc->get_version("osdmap", &c->latest, NULL, c);
   }
 
-  for (auto p = check_latest_map_commands.begin();
+  for (map<uint64_t, CommandOp*>::iterator p
+	 = check_latest_map_commands.begin();
        p != check_latest_map_commands.end();
        ++p) {
-    monc->get_version("osdmap", CB_Command_Map_Latest(this, p->second->tid));
+    C_Command_Map_Latest *c = new C_Command_Map_Latest(this, p->second->tid);
+    monc->get_version("osdmap", &c->latest, NULL, c);
   }
 }
 
@@ -2205,8 +2263,7 @@ void Objecter::op_submit(Op *op, ceph_tid_t *ptid, int *ctx_budget)
   _op_submit_with_budget(op, rl, ptid, ctx_budget);
 }
 
-void Objecter::_op_submit_with_budget(Op *op,
-				      shunique_lock<ceph::shared_mutex>& sul,
+void Objecter::_op_submit_with_budget(Op *op, shunique_lock& sul,
 				      ceph_tid_t *ptid,
 				      int *ctx_budget)
 {
@@ -2244,7 +2301,7 @@ void Objecter::_send_op_account(Op *op)
   inflight_ops++;
 
   // add to gather set(s)
-  if (op->has_completion()) {
+  if (op->onfinish) {
     num_in_flight++;
   } else {
     ldout(cct, 20) << " note: not requesting reply" << dendl;
@@ -2252,7 +2309,6 @@ void Objecter::_send_op_account(Op *op)
 
   logger->inc(l_osdc_op_active);
   logger->inc(l_osdc_op);
-  logger->inc(l_osdc_oplen_avg, op->ops.size());
 
   if ((op->target.flags & (CEPH_OSD_FLAG_READ | CEPH_OSD_FLAG_WRITE)) ==
       (CEPH_OSD_FLAG_READ|CEPH_OSD_FLAG_WRITE))
@@ -2265,7 +2321,7 @@ void Objecter::_send_op_account(Op *op)
   if (op->target.flags & CEPH_OSD_FLAG_PGOP)
     logger->inc(l_osdc_op_pg);
 
-  for (auto p = op->ops.begin(); p != op->ops.end(); ++p) {
+  for (vector<OSDOp>::iterator p = op->ops.begin(); p != op->ops.end(); ++p) {
     int code = l_osdc_osdop_other;
     switch (p->op.op) {
     case CEPH_OSD_OP_STAT: code = l_osdc_osdop_stat; break;
@@ -2310,7 +2366,7 @@ void Objecter::_send_op_account(Op *op)
   }
 }
 
-void Objecter::_op_submit(Op *op, shunique_lock<ceph::shared_mutex>& sul, ceph_tid_t *ptid)
+void Objecter::_op_submit(Op *op, shunique_lock& sul, ceph_tid_t *ptid)
 {
   // rwlock is locked
 
@@ -2396,7 +2452,7 @@ void Objecter::_op_submit(Op *op, shunique_lock<ceph::shared_mutex>& sul, ceph_t
     _maybe_request_map();
   }
 
-  unique_lock sl(s->lock);
+  OSDSession::unique_lock sl(s->lock);
   if (op->tid == 0)
     op->tid = ++last_tid;
 
@@ -2432,9 +2488,9 @@ int Objecter::op_cancel(OSDSession *s, ceph_tid_t tid, int r)
 {
   ceph_assert(initialized);
 
-  unique_lock sl(s->lock);
+  OSDSession::unique_lock sl(s->lock);
 
-  auto p = s->ops.find(tid);
+  map<ceph_tid_t, Op*>::iterator p = s->ops.find(tid);
   if (p == s->ops.end()) {
     ldout(cct, 10) << __func__ << " tid " << tid << " dne in session "
 		   << s->osd << dendl;
@@ -2452,9 +2508,10 @@ int Objecter::op_cancel(OSDSession *s, ceph_tid_t tid, int r)
   ldout(cct, 10) << __func__ << " tid " << tid << " in session " << s->osd
 		 << dendl;
   Op *op = p->second;
-  if (op->has_completion()) {
+  if (op->onfinish) {
     num_in_flight--;
-    op->complete(osdcode(r), r);
+    op->onfinish->complete(r);
+    op->onfinish = NULL;
   }
   _op_cancel_map_check(op);
   _finish_op(op, r);
@@ -2492,10 +2549,10 @@ int Objecter::_op_cancel(ceph_tid_t tid, int r)
 
 start:
 
-  for (auto siter = osd_sessions.begin();
+  for (map<int, OSDSession *>::iterator siter = osd_sessions.begin();
        siter != osd_sessions.end(); ++siter) {
     OSDSession *s = siter->second;
-    shared_lock sl(s->lock);
+    OSDSession::shared_lock sl(s->lock);
     if (s->ops.find(tid) != s->ops.end()) {
       sl.unlock();
       ret = op_cancel(s, tid, r);
@@ -2511,7 +2568,7 @@ start:
 		<< " not found in live sessions" << dendl;
 
   // Handle case where the op is in homeless session
-  shared_lock sl(homeless_session->lock);
+  OSDSession::shared_lock sl(homeless_session->lock);
   if (homeless_session->ops.find(tid) != homeless_session->ops.end()) {
     sl.unlock();
     ret = op_cancel(homeless_session, tid, r);
@@ -2539,11 +2596,11 @@ epoch_t Objecter::op_cancel_writes(int r, int64_t pool)
   std::vector<ceph_tid_t> to_cancel;
   bool found = false;
 
-  for (auto siter = osd_sessions.begin();
+  for (map<int, OSDSession *>::iterator siter = osd_sessions.begin();
        siter != osd_sessions.end(); ++siter) {
     OSDSession *s = siter->second;
-    shared_lock sl(s->lock);
-    for (auto op_i = s->ops.begin();
+    OSDSession::shared_lock sl(s->lock);
+    for (map<ceph_tid_t, Op*>::iterator op_i = s->ops.begin();
 	 op_i != s->ops.end(); ++op_i) {
       if (op_i->second->target.flags & CEPH_OSD_FLAG_WRITE
 	  && (pool == -1 || op_i->second->target.target_oloc.pool == pool)) {
@@ -2552,7 +2609,9 @@ epoch_t Objecter::op_cancel_writes(int r, int64_t pool)
     }
     sl.unlock();
 
-    for (auto titer = to_cancel.begin(); titer != to_cancel.end(); ++titer) {
+    for (std::vector<ceph_tid_t>::iterator titer = to_cancel.begin();
+	 titer != to_cancel.end();
+	 ++titer) {
       int cancel_result = op_cancel(s, *titer, r);
       // We hold rwlock across search and cancellation, so cancels
       // should always succeed
@@ -2638,7 +2697,8 @@ bool Objecter::_osdmap_pool_full(const int64_t pool_id) const
 
 bool Objecter::_osdmap_has_pool_full() const
 {
-  for (auto it = osdmap->get_pools().begin();
+  for (map<int64_t, pg_pool_t>::const_iterator it
+	 = osdmap->get_pools().begin();
        it != osdmap->get_pools().end(); ++it) {
     if (_osdmap_pool_full(it->second))
       return true;
@@ -2944,7 +3004,7 @@ int Objecter::_calc_target(op_target_t *t, Connection *con, bool any_change)
 }
 
 int Objecter::_map_session(op_target_t *target, OSDSession **s,
-			   shunique_lock<ceph::shared_mutex>& sul)
+			   shunique_lock& sul)
 {
   _calc_target(target, nullptr);
   return _get_session(target->osd, s, sul);
@@ -3051,7 +3111,7 @@ void Objecter::_session_command_op_assign(OSDSession *to, CommandOp *op)
 }
 
 int Objecter::_recalc_linger_op_target(LingerOp *linger_op,
-				       shunique_lock<ceph::shared_mutex>& sul)
+				       shunique_lock& sul)
 {
   // rwlock is locked unique
 
@@ -3070,7 +3130,7 @@ int Objecter::_recalc_linger_op_target(LingerOp *linger_op,
       // same time here is only safe because we are the only one that
       // takes two, and we are holding rwlock for write.  Disable
       // lockdep because it doesn't know that.
-      unique_lock sl(s->lock);
+      OSDSession::unique_lock sl(s->lock);
       _session_linger_op_remove(linger_op->session, linger_op);
       _session_linger_op_assign(s, linger_op);
     }
@@ -3086,8 +3146,8 @@ void Objecter::_cancel_linger_op(Op *op)
   ldout(cct, 15) << "cancel_op " << op->tid << dendl;
 
   ceph_assert(!op->should_resend);
-  if (op->has_completion()) {
-    op->onfinish = nullptr;
+  if (op->onfinish) {
+    delete op->onfinish;
     num_in_flight--;
   }
 
@@ -3121,7 +3181,7 @@ void Objecter::_finish_op(Op *op, int r)
   op->put();
 }
 
-Objecter::MOSDOp *Objecter::_prepare_osd_op(Op *op)
+MOSDOp *Objecter::_prepare_osd_op(Op *op)
 {
   // rwlock is locked
 
@@ -3139,10 +3199,10 @@ Objecter::MOSDOp *Objecter::_prepare_osd_op(Op *op)
   op->stamp = ceph::coarse_mono_clock::now();
 
   hobject_t hobj = op->target.get_hobj();
-  auto m = new MOSDOp(client_inc, op->tid,
-		      hobj, op->target.actual_pgid,
-		      osdmap->get_epoch(),
-		      flags, op->features);
+  MOSDOp *m = new MOSDOp(client_inc, op->tid,
+			 hobj, op->target.actual_pgid,
+			 osdmap->get_epoch(),
+			 flags, op->features);
 
   m->set_snapid(op->snapid);
   m->set_snap_seq(op->snapc.seq);
@@ -3248,10 +3308,12 @@ void Objecter::_send_op(Op *op)
   op->session->con->send_message(m);
 }
 
-int Objecter::calc_op_budget(const bc::small_vector_base<OSDOp>& ops)
+int Objecter::calc_op_budget(const vector<OSDOp>& ops)
 {
   int op_budget = 0;
-  for (auto i = ops.begin(); i != ops.end(); ++i) {
+  for (vector<OSDOp>::const_iterator i = ops.begin();
+       i != ops.end();
+       ++i) {
     if (i->op.op & CEPH_OSD_OP_MODE_WR) {
       op_budget += i->indata.length();
     } else if (ceph_osd_op_mode_read(i->op.op)) {
@@ -3267,7 +3329,7 @@ int Objecter::calc_op_budget(const bc::small_vector_base<OSDOp>& ops)
 }
 
 void Objecter::_throttle_op(Op *op,
-			    shunique_lock<ceph::shared_mutex>& sul,
+			    shunique_lock& sul,
 			    int op_budget)
 {
   ceph_assert(sul && sul.mutex() == &rwlock);
@@ -3321,7 +3383,7 @@ void Objecter::handle_osd_op_reply(MOSDOpReply *m)
     return;
   }
 
-  unique_lock sl(s->lock);
+  OSDSession::unique_lock sl(s->lock);
 
   map<ceph_tid_t, Op *>::iterator iter = s->ops.find(tid);
   if (iter == s->ops.end()) {
@@ -3347,7 +3409,7 @@ void Objecter::handle_osd_op_reply(MOSDOpReply *m)
   if (retry_writes_after_first_reply && op->attempts == 1 &&
       (op->target.flags & CEPH_OSD_FLAG_WRITE)) {
     ldout(cct, 7) << "retrying write after first reply: " << tid << dendl;
-    if (op->has_completion()) {
+    if (op->onfinish) {
       num_in_flight--;
     }
     _session_op_remove(s, op);
@@ -3375,13 +3437,13 @@ void Objecter::handle_osd_op_reply(MOSDOpReply *m)
     // have, but that is better than doing callbacks out of order.
   }
 
-  decltype(op->onfinish) onfinish;
+  Context *onfinish = 0;
 
   int rc = m->get_result();
 
   if (m->is_redirect_reply()) {
     ldout(cct, 5) << " got redirect reply; redirecting" << dendl;
-    if (op->has_completion())
+    if (op->onfinish)
       num_in_flight--;
     _session_op_remove(s, op);
     sl.unlock();
@@ -3401,7 +3463,7 @@ void Objecter::handle_osd_op_reply(MOSDOpReply *m)
 
   if (rc == -EAGAIN) {
     ldout(cct, 7) << " got -EAGAIN, resubmitting" << dendl;
-    if (op->has_completion())
+    if (op->onfinish)
       num_in_flight--;
     _session_op_remove(s, op);
     sl.unlock();
@@ -3439,7 +3501,7 @@ void Objecter::handle_osd_op_reply(MOSDOpReply *m)
       ldout(cct,10) << __func__ << " copying resulting " << bl.length()
 		    << " into existing ceph::buffer of length " << op->outbl->length()
 		    << dendl;
-      cb::list t;
+      ceph::buffer::list t;
       t.claim(*op->outbl);
       t.invalidate_crc();  // we're overwriting the raw buffers via c_str()
       bl.copy(0, bl.length(), t.c_str());
@@ -3459,20 +3521,15 @@ void Objecter::handle_osd_op_reply(MOSDOpReply *m)
 		  << " != request ops " << op->ops
 		  << " from " << m->get_source_inst() << dendl;
 
-  ceph_assert(op->ops.size() == op->out_bl.size());
-  ceph_assert(op->ops.size() == op->out_rval.size());
-  ceph_assert(op->ops.size() == op->out_ec.size());
-  ceph_assert(op->ops.size() == op->out_handler.size());
-  auto pb = op->out_bl.begin();
-  auto pr = op->out_rval.begin();
-  auto pe = op->out_ec.begin();
-  auto ph = op->out_handler.begin();
+  vector<ceph::buffer::list*>::iterator pb = op->out_bl.begin();
+  vector<int*>::iterator pr = op->out_rval.begin();
+  vector<Context*>::iterator ph = op->out_handler.begin();
   ceph_assert(op->out_bl.size() == op->out_rval.size());
   ceph_assert(op->out_bl.size() == op->out_handler.size());
-  auto p = out_ops.begin();
+  vector<OSDOp>::iterator p = out_ops.begin();
   for (unsigned i = 0;
        p != out_ops.end() && pb != op->out_bl.end();
-       ++i, ++p, ++pb, ++pr, ++pe, ++ph) {
+       ++i, ++p, ++pb, ++pr, ++ph) {
     ldout(cct, 10) << " op " << i << " rval " << p->rval
 		   << " len " << p->outdata.length() << dendl;
     if (*pb)
@@ -3481,22 +3538,20 @@ void Objecter::handle_osd_op_reply(MOSDOpReply *m)
     // can change it if e.g. decoding fails
     if (*pr)
       **pr = ceph_to_hostos_errno(p->rval);
-    if (*pe)
-      **pe = p->rval < 0 ? bs::error_code(-p->rval, osd_category()) :
-	bs::error_code();
     if (*ph) {
-      std::move((*ph))(bs::error_code(-p->rval, osd_category()),
-		       p->rval, p->outdata);
+      ldout(cct, 10) << " op " << i << " handler " << *ph << dendl;
+      (*ph)->complete(ceph_to_hostos_errno(p->rval));
+      *ph = NULL;
     }
   }
 
   // NOTE: we assume that since we only request ONDISK ever we will
   // only ever get back one (type of) ack ever.
 
-  if (op->has_completion()) {
+  if (op->onfinish) {
     num_in_flight--;
-    onfinish = std::move(op->onfinish);
-    op->onfinish = nullptr;
+    onfinish = op->onfinish;
+    op->onfinish = NULL;
   }
   logger->inc(l_osdc_op_reply);
 
@@ -3515,8 +3570,8 @@ void Objecter::handle_osd_op_reply(MOSDOpReply *m)
   sl.unlock();
 
   // do callbacks
-  if (Op::has_completion(onfinish)) {
-    Op::complete(std::move(onfinish), osdcode(rc), rc);
+  if (onfinish) {
+    onfinish->complete(rc);
   }
   if (completion_lock.mutex()) {
     completion_lock.unlock();
@@ -3545,7 +3600,7 @@ void Objecter::handle_osd_backoff(MOSDBackoff *m)
 
   get_session(s);
 
-  unique_lock sl(s->lock);
+  OSDSession::unique_lock sl(s->lock);
 
   switch (m->op) {
   case CEPH_OSD_BACKOFF_OP_BLOCK:
@@ -3560,9 +3615,10 @@ void Objecter::handle_osd_backoff(MOSDBackoff *m)
 
       // ack with original backoff's epoch so that the osd can discard this if
       // there was a pg split.
-      auto r = new MOSDBackoff(m->pgid, m->map_epoch,
-			       CEPH_OSD_BACKOFF_OP_ACK_BLOCK,
-			       m->id, m->begin, m->end);
+      Message *r = new MOSDBackoff(m->pgid,
+				   m->map_epoch,
+				   CEPH_OSD_BACKOFF_OP_ACK_BLOCK,
+				   m->id, m->begin, m->end);
       // this priority must match the MOSDOps from _prepare_osd_op
       r->set_priority(cct->_conf->osd_client_op_priority);
       con->send_message(r);
@@ -3725,7 +3781,7 @@ void Objecter::list_nobjects(NListContext *list_context, Context *onfinish)
   op.pg_nls(list_context->max_entries, list_context->filter,
 	    list_context->pos, osdmap->get_epoch());
   list_context->bl.clear();
-  auto onack = new C_NList(list_context, onfinish, this);
+  C_NList *onack = new C_NList(list_context, onfinish, this);
   object_locator_t oloc(list_context->pool_id, list_context->nspace);
 
   // note current_pg in case we don't have (or lose) SORTBITWISE
@@ -3747,7 +3803,7 @@ void Objecter::_nlist_reply(NListContext *list_context, int r,
   decode(response, iter);
   if (!iter.end()) {
     // we do this as legacy.
-    cb::list legacy_extra_info;
+    ceph::buffer::list legacy_extra_info;
     decode(legacy_extra_info, iter);
   }
 
@@ -3776,9 +3832,7 @@ void Objecter::_nlist_reply(NListContext *list_context, int r,
 		 << ", handle " << response.handle
 		 << ", tentative new pos " << list_context->pos << dendl;
   if (response_size) {
-    std::move(response.entries.begin(), response.entries.end(),
-	      std::back_inserter(list_context->list));
-    response.entries.clear();
+    list_context->list.splice(list_context->list.end(), response.entries);
   }
 
   if (list_context->list.size() >= list_context->max_entries) {
@@ -3807,177 +3861,176 @@ void Objecter::put_nlist_context_budget(NListContext *list_context)
 
 // snapshots
 
-void Objecter::create_pool_snap(int64_t pool, std::string_view snap_name,
-				decltype(PoolOp::onfinish)&& onfinish)
+int Objecter::create_pool_snap(int64_t pool, string& snap_name,
+			       Context *onfinish)
 {
   unique_lock wl(rwlock);
   ldout(cct, 10) << "create_pool_snap; pool: " << pool << "; snap: "
 		 << snap_name << dendl;
 
   const pg_pool_t *p = osdmap->get_pg_pool(pool);
-  if (!p) {
-    onfinish->defer(std::move(onfinish), osdc_errc::pool_dne, cb::list{});
-    return;
-  }
-  if (p->snap_exists(snap_name)) {
-    onfinish->defer(std::move(onfinish), osdc_errc::snapshot_exists,
-		    cb::list{});
-    return;
-  }
+  if (!p)
+    return -EINVAL;
+  if (p->snap_exists(snap_name.c_str()))
+    return -EEXIST;
 
-  auto op = new PoolOp;
+  PoolOp *op = new PoolOp;
+  if (!op)
+    return -ENOMEM;
   op->tid = ++last_tid;
   op->pool = pool;
   op->name = snap_name;
-  op->onfinish = std::move(onfinish);
+  op->onfinish = onfinish;
   op->pool_op = POOL_OP_CREATE_SNAP;
   pool_ops[op->tid] = op;
 
   pool_op_submit(op);
+
+  return 0;
 }
 
-struct CB_SelfmanagedSnap {
-  std::unique_ptr<ca::Completion<void(bs::error_code, snapid_t)>> fin;
-  CB_SelfmanagedSnap(decltype(fin)&& fin)
-    : fin(std::move(fin)) {}
-  void operator()(bs::error_code ec, const cb::list& bl) {
-    snapid_t snapid = 0;
-    if (!ec) {
+struct C_SelfmanagedSnap : public Context {
+  ceph::buffer::list bl;
+  snapid_t *psnapid;
+  Context *fin;
+  C_SelfmanagedSnap(snapid_t *ps, Context *f) : psnapid(ps), fin(f) {}
+  void finish(int r) override {
+    if (r == 0) {
       try {
-	auto p = bl.cbegin();
-	decode(snapid, p);
-      } catch (const cb::error& e) {
-        ec = e.code();
+        auto p = bl.cbegin();
+        decode(*psnapid, p);
+      } catch (ceph::buffer::error&) {
+        r = -EIO;
       }
     }
-    fin->defer(std::move(fin), ec, snapid);
+    fin->complete(r);
   }
 };
 
-void Objecter::allocate_selfmanaged_snap(
-  int64_t pool,
-  std::unique_ptr<ca::Completion<void(bs::error_code, snapid_t)>> onfinish)
+int Objecter::allocate_selfmanaged_snap(int64_t pool, snapid_t *psnapid,
+					Context *onfinish)
 {
   unique_lock wl(rwlock);
   ldout(cct, 10) << "allocate_selfmanaged_snap; pool: " << pool << dendl;
-  auto op = new PoolOp;
+  PoolOp *op = new PoolOp;
+  if (!op) return -ENOMEM;
   op->tid = ++last_tid;
   op->pool = pool;
-  op->onfinish = PoolOp::OpComp::create(
-    service.get_executor(),
-    CB_SelfmanagedSnap(std::move(onfinish)));
+  C_SelfmanagedSnap *fin = new C_SelfmanagedSnap(psnapid, onfinish);
+  op->onfinish = fin;
+  op->blp = &fin->bl;
   op->pool_op = POOL_OP_CREATE_UNMANAGED_SNAP;
   pool_ops[op->tid] = op;
 
   pool_op_submit(op);
+  return 0;
 }
 
-void Objecter::delete_pool_snap(
-  int64_t pool, std::string_view snap_name,
-  decltype(PoolOp::onfinish)&& onfinish)
+int Objecter::delete_pool_snap(int64_t pool, string& snap_name,
+			       Context *onfinish)
 {
   unique_lock wl(rwlock);
   ldout(cct, 10) << "delete_pool_snap; pool: " << pool << "; snap: "
 		 << snap_name << dendl;
 
   const pg_pool_t *p = osdmap->get_pg_pool(pool);
-  if (!p) {
-    onfinish->defer(std::move(onfinish), osdc_errc::pool_dne, cb::list{});
-    return;
-  }
-
-  if (!p->snap_exists(snap_name)) {
-    onfinish->defer(std::move(onfinish), osdc_errc::snapshot_dne, cb::list{});
-    return;
-  }
+  if (!p)
+    return -EINVAL;
+  if (!p->snap_exists(snap_name.c_str()))
+    return -ENOENT;
 
-  auto op = new PoolOp;
+  PoolOp *op = new PoolOp;
+  if (!op)
+    return -ENOMEM;
   op->tid = ++last_tid;
   op->pool = pool;
   op->name = snap_name;
-  op->onfinish = std::move(onfinish);
+  op->onfinish = onfinish;
   op->pool_op = POOL_OP_DELETE_SNAP;
   pool_ops[op->tid] = op;
 
   pool_op_submit(op);
+
+  return 0;
 }
 
-void Objecter::delete_selfmanaged_snap(int64_t pool, snapid_t snap,
-				       decltype(PoolOp::onfinish)&& onfinish)
+int Objecter::delete_selfmanaged_snap(int64_t pool, snapid_t snap,
+				      Context *onfinish)
 {
   unique_lock wl(rwlock);
   ldout(cct, 10) << "delete_selfmanaged_snap; pool: " << pool << "; snap: "
 		 << snap << dendl;
-  auto op = new PoolOp;
+  PoolOp *op = new PoolOp;
+  if (!op) return -ENOMEM;
   op->tid = ++last_tid;
   op->pool = pool;
-  op->onfinish = std::move(onfinish);
+  op->onfinish = onfinish;
   op->pool_op = POOL_OP_DELETE_UNMANAGED_SNAP;
   op->snapid = snap;
   pool_ops[op->tid] = op;
 
   pool_op_submit(op);
+
+  return 0;
 }
 
-void Objecter::create_pool(std::string_view name,
-			   decltype(PoolOp::onfinish)&& onfinish,
-			   int crush_rule)
+int Objecter::create_pool(string& name, Context *onfinish,
+			  int crush_rule)
 {
   unique_lock wl(rwlock);
   ldout(cct, 10) << "create_pool name=" << name << dendl;
 
-  if (osdmap->lookup_pg_pool_name(name) >= 0) {
-    onfinish->defer(std::move(onfinish), osdc_errc::pool_exists, cb::list{});
-    return;
-  }
+  if (osdmap->lookup_pg_pool_name(name) >= 0)
+    return -EEXIST;
 
-  auto op = new PoolOp;
+  PoolOp *op = new PoolOp;
+  if (!op)
+    return -ENOMEM;
   op->tid = ++last_tid;
   op->pool = 0;
   op->name = name;
-  op->onfinish = std::move(onfinish);
+  op->onfinish = onfinish;
   op->pool_op = POOL_OP_CREATE;
   pool_ops[op->tid] = op;
   op->crush_rule = crush_rule;
 
   pool_op_submit(op);
+
+  return 0;
 }
 
-void Objecter::delete_pool(int64_t pool,
-			   decltype(PoolOp::onfinish)&& onfinish)
+int Objecter::delete_pool(int64_t pool, Context *onfinish)
 {
   unique_lock wl(rwlock);
   ldout(cct, 10) << "delete_pool " << pool << dendl;
 
   if (!osdmap->have_pg_pool(pool))
-    onfinish->defer(std::move(onfinish), osdc_errc::pool_dne, cb::list{});
-  else
-    _do_delete_pool(pool, std::move(onfinish));
+    return -ENOENT;
+
+  _do_delete_pool(pool, onfinish);
+  return 0;
 }
 
-void Objecter::delete_pool(std::string_view pool_name,
-			   decltype(PoolOp::onfinish)&& onfinish)
+int Objecter::delete_pool(const string &pool_name, Context *onfinish)
 {
   unique_lock wl(rwlock);
   ldout(cct, 10) << "delete_pool " << pool_name << dendl;
 
   int64_t pool = osdmap->lookup_pg_pool_name(pool_name);
   if (pool < 0)
-    // This only returns one error: -ENOENT.
-    onfinish->defer(std::move(onfinish), osdc_errc::pool_dne, cb::list{});
-  else
-    _do_delete_pool(pool, std::move(onfinish));
-}
+    return pool;
 
-void Objecter::_do_delete_pool(int64_t pool,
-			       decltype(PoolOp::onfinish)&& onfinish)
+  _do_delete_pool(pool, onfinish);
+  return 0;
+}
 
+void Objecter::_do_delete_pool(int64_t pool, Context *onfinish)
 {
-  auto op = new PoolOp;
+  PoolOp *op = new PoolOp;
   op->tid = ++last_tid;
   op->pool = pool;
   op->name = "delete";
-  op->onfinish = std::move(onfinish);
+  op->onfinish = onfinish;
   op->pool_op = POOL_OP_DELETE;
   pool_ops[op->tid] = op;
   pool_op_submit(op);
@@ -3999,9 +4052,9 @@ void Objecter::_pool_op_submit(PoolOp *op)
   // rwlock is locked unique
 
   ldout(cct, 10) << "pool_op_submit " << op->tid << dendl;
-  auto m = new MPoolOp(monc->get_fsid(), op->tid, op->pool,
-		       op->name, op->pool_op,
-		       last_seen_osdmap_version);
+  MPoolOp *m = new MPoolOp(monc->get_fsid(), op->tid, op->pool,
+			   op->name, op->pool_op,
+			   last_seen_osdmap_version);
   if (op->snapid) m->snapid = op->snapid;
   if (op->crush_rule) m->crush_rule = op->crush_rule;
   monc->send_mon_message(m);
@@ -4012,15 +4065,13 @@ void Objecter::_pool_op_submit(PoolOp *op)
 
 /**
  * Handle a reply to a PoolOp message. Check that we sent the message
- * and give the caller responsibility for the returned cb::list.
+ * and give the caller responsibility for the returned ceph::buffer::list.
  * Then either call the finisher or stash the PoolOp, depending on if we
  * have a new enough map.
  * Lastly, clean up the message and PoolOp.
  */
 void Objecter::handle_pool_op_reply(MPoolOpReply *m)
 {
-  int rc = m->replyCode;
-  auto ec = rc < 0 ? bs::error_code(-rc, mon_category()) : bs::error_code();
   FUNCTRACE(cct);
   shunique_lock sul(rwlock, acquire_shared);
   if (!initialized) {
@@ -4031,13 +4082,13 @@ void Objecter::handle_pool_op_reply(MPoolOpReply *m)
 
   ldout(cct, 10) << "handle_pool_op_reply " << *m << dendl;
   ceph_tid_t tid = m->get_tid();
-  auto iter = pool_ops.find(tid);
+  map<ceph_tid_t, PoolOp *>::iterator iter = pool_ops.find(tid);
   if (iter != pool_ops.end()) {
     PoolOp *op = iter->second;
     ldout(cct, 10) << "have request " << tid << " at " << op << " Op: "
 		   << ceph_pool_op_name(op->pool_op) << dendl;
-    cb::list bl;
-    bl.claim(m->response_data);
+    if (op->blp)
+      op->blp->claim(m->response_data);
     if (m->version > last_seen_osdmap_version)
       last_seen_osdmap_version = m->version;
     if (osdmap->get_epoch() < m->epoch) {
@@ -4051,27 +4102,19 @@ void Objecter::handle_pool_op_reply(MPoolOpReply *m)
       if (osdmap->get_epoch() < m->epoch) {
 	ldout(cct, 20) << "waiting for client to reach epoch " << m->epoch
 		       << " before calling back" << dendl;
-	_wait_for_new_map(OpCompletion::create(
-			    service.get_executor(),
-			    [o = std::move(op->onfinish),
-			     bl = std::move(bl)](
-			      bs::error_code ec) mutable {
-			      o->defer(std::move(o), ec, bl);
-			    }),
-			  m->epoch,
-			  ec);
+	_wait_for_new_map(op->onfinish, m->epoch, m->replyCode);
       } else {
 	// map epoch changed, probably because a MOSDMap message
 	// sneaked in. Do caller-specified callback now or else
 	// we lose it forever.
 	ceph_assert(op->onfinish);
-	op->onfinish->defer(std::move(op->onfinish), ec, std::move(bl));
+	op->onfinish->complete(m->replyCode);
       }
     } else {
       ceph_assert(op->onfinish);
-      op->onfinish->defer(std::move(op->onfinish), ec, std::move(bl));
+      op->onfinish->complete(m->replyCode);
     }
-    op->onfinish = nullptr;
+    op->onfinish = NULL;
     if (!sul.owns_lock()) {
       sul.unlock();
       sul.lock();
@@ -4098,7 +4141,7 @@ int Objecter::pool_op_cancel(ceph_tid_t tid, int r)
 
   unique_lock wl(rwlock);
 
-  auto it = pool_ops.find(tid);
+  map<ceph_tid_t, PoolOp*>::iterator it = pool_ops.find(tid);
   if (it == pool_ops.end()) {
     ldout(cct, 10) << __func__ << " tid " << tid << " dne" << dendl;
     return -ENOENT;
@@ -4108,7 +4151,7 @@ int Objecter::pool_op_cancel(ceph_tid_t tid, int r)
 
   PoolOp *op = it->second;
   if (op->onfinish)
-    op->onfinish->defer(std::move(op->onfinish), osdcode(r), cb::list{});
+    op->onfinish->complete(r);
 
   _finish_pool_op(op, r);
   return 0;
@@ -4129,16 +4172,19 @@ void Objecter::_finish_pool_op(PoolOp *op, int r)
 
 // pool stats
 
-void Objecter::get_pool_stats(
-  const std::vector<std::string>& pools,
-  decltype(PoolStatOp::onfinish)&& onfinish)
+void Objecter::get_pool_stats(list<string>& pools,
+			      map<string,pool_stat_t> *result,
+			      bool *per_pool,
+			      Context *onfinish)
 {
   ldout(cct, 10) << "get_pool_stats " << pools << dendl;
 
-  auto op = new PoolStatOp;
+  PoolStatOp *op = new PoolStatOp;
   op->tid = ++last_tid;
   op->pools = pools;
-  op->onfinish = std::move(onfinish);
+  op->pool_stats = result;
+  op->per_pool = per_pool;
+  op->onfinish = onfinish;
   if (mon_timeout > timespan(0)) {
     op->ontimeout = timer.add_event(mon_timeout,
 				    [this, op]() {
@@ -4179,15 +4225,16 @@ void Objecter::handle_get_pool_stats_reply(MGetPoolStatsReply *m)
     return;
   }
 
-  auto iter = poolstat_ops.find(tid);
+  map<ceph_tid_t, PoolStatOp *>::iterator iter = poolstat_ops.find(tid);
   if (iter != poolstat_ops.end()) {
     PoolStatOp *op = poolstat_ops[tid];
     ldout(cct, 10) << "have request " << tid << " at " << op << dendl;
+    *op->pool_stats = m->pool_stats;
+    *op->per_pool = m->per_pool;
     if (m->version > last_seen_pgmap_version) {
       last_seen_pgmap_version = m->version;
     }
-    op->onfinish->defer(std::move(op->onfinish), bs::error_code{},
-			std::move(m->pool_stats), m->per_pool);
+    op->onfinish->complete(0);
     _finish_pool_stat_op(op, 0);
   } else {
     ldout(cct, 10) << "unknown request " << tid << dendl;
@@ -4202,7 +4249,7 @@ int Objecter::pool_stat_op_cancel(ceph_tid_t tid, int r)
 
   unique_lock wl(rwlock);
 
-  auto it = poolstat_ops.find(tid);
+  map<ceph_tid_t, PoolStatOp*>::iterator it = poolstat_ops.find(tid);
   if (it == poolstat_ops.end()) {
     ldout(cct, 10) << __func__ << " tid " << tid << " dne" << dendl;
     return -ENOENT;
@@ -4210,10 +4257,9 @@ int Objecter::pool_stat_op_cancel(ceph_tid_t tid, int r)
 
   ldout(cct, 10) << __func__ << " tid " << tid << dendl;
 
-  auto op = it->second;
+  PoolStatOp *op = it->second;
   if (op->onfinish)
-    op->onfinish->defer(std::move(op->onfinish), osdcode(r),
-			bc::flat_map<std::string, pool_stat_t>{}, false);
+    op->onfinish->complete(r);
   _finish_pool_stat_op(op, r);
   return 0;
 }
@@ -4231,16 +4277,18 @@ void Objecter::_finish_pool_stat_op(PoolStatOp *op, int r)
   delete op;
 }
 
-void Objecter::get_fs_stats(boost::optional<int64_t> poolid,
-			    decltype(StatfsOp::onfinish)&& onfinish)
+void Objecter::get_fs_stats(ceph_statfs& result,
+			    boost::optional<int64_t> data_pool,
+			    Context *onfinish)
 {
   ldout(cct, 10) << "get_fs_stats" << dendl;
   unique_lock l(rwlock);
 
-  auto op = new StatfsOp;
+  StatfsOp *op = new StatfsOp;
   op->tid = ++last_tid;
-  op->data_pool = poolid;
-  op->onfinish = std::move(onfinish);
+  op->stats = &result;
+  op->data_pool = data_pool;
+  op->onfinish = onfinish;
   if (mon_timeout > timespan(0)) {
     op->ontimeout = timer.add_event(mon_timeout,
 				    [this, op]() {
@@ -4283,9 +4331,10 @@ void Objecter::handle_fs_stats_reply(MStatfsReply *m)
   if (statfs_ops.count(tid)) {
     StatfsOp *op = statfs_ops[tid];
     ldout(cct, 10) << "have request " << tid << " at " << op << dendl;
+    *(op->stats) = m->h.st;
     if (m->h.version > last_seen_pgmap_version)
       last_seen_pgmap_version = m->h.version;
-    op->onfinish->defer(std::move(op->onfinish), bs::error_code{}, m->h.st);
+    op->onfinish->complete(0);
     _finish_statfs_op(op, 0);
   } else {
     ldout(cct, 10) << "unknown request " << tid << dendl;
@@ -4300,7 +4349,7 @@ int Objecter::statfs_op_cancel(ceph_tid_t tid, int r)
 
   unique_lock wl(rwlock);
 
-  auto it = statfs_ops.find(tid);
+  map<ceph_tid_t, StatfsOp*>::iterator it = statfs_ops.find(tid);
   if (it == statfs_ops.end()) {
     ldout(cct, 10) << __func__ << " tid " << tid << " dne" << dendl;
     return -ENOENT;
@@ -4308,9 +4357,9 @@ int Objecter::statfs_op_cancel(ceph_tid_t tid, int r)
 
   ldout(cct, 10) << __func__ << " tid " << tid << dendl;
 
-  auto op = it->second;
+  StatfsOp *op = it->second;
   if (op->onfinish)
-    op->onfinish->defer(std::move(op->onfinish), osdcode(r), ceph_statfs{});
+    op->onfinish->complete(r);
   _finish_statfs_op(op, r);
   return 0;
 }
@@ -4331,16 +4380,16 @@ void Objecter::_finish_statfs_op(StatfsOp *op, int r)
 // scatter/gather
 
 void Objecter::_sg_read_finish(vector<ObjectExtent>& extents,
-			       vector<cb::list>& resultbl,
-			       cb::list *bl, Context *onfinish)
+			       vector<ceph::buffer::list>& resultbl,
+			       ceph::buffer::list *bl, Context *onfinish)
 {
   // all done
   ldout(cct, 15) << "_sg_read_finish" << dendl;
 
   if (extents.size() > 1) {
     Striper::StripedReadResult r;
-    auto bit = resultbl.begin();
-    for (auto eit = extents.begin();
+    vector<ceph::buffer::list>::iterator bit = resultbl.begin();
+    for (vector<ObjectExtent>::iterator eit = extents.begin();
 	 eit != extents.end();
 	 ++eit, ++bit) {
       r.add_partial_result(cct, *bit, eit->buffer_extents);
@@ -4392,7 +4441,7 @@ bool Objecter::ms_handle_reset(Connection *con)
 	return false;
       }
       map<uint64_t, LingerOp *> lresend;
-      unique_lock sl(session->lock);
+      OSDSession::unique_lock sl(session->lock);
       _reopen_session(session);
       _kick_requests(session, lresend);
       sl.unlock();
@@ -4440,7 +4489,9 @@ void Objecter::op_target_t::dump(Formatter *f) const
 
 void Objecter::_dump_active(OSDSession *s)
 {
-  for (auto p = s->ops.begin(); p != s->ops.end(); ++p) {
+  for (map<ceph_tid_t,Op*>::iterator p = s->ops.begin();
+       p != s->ops.end();
+       ++p) {
     Op *op = p->second;
     ldout(cct, 20) << op->tid << "\t" << op->target.pgid
 		   << "\tosd." << (op->session ? op->session->osd : -1)
@@ -4453,10 +4504,10 @@ void Objecter::_dump_active()
 {
   ldout(cct, 20) << "dump_active .. " << num_homeless_ops << " homeless"
 		 << dendl;
-  for (auto siter = osd_sessions.begin();
+  for (map<int, OSDSession *>::iterator siter = osd_sessions.begin();
        siter != osd_sessions.end(); ++siter) {
-    auto s = siter->second;
-    shared_lock sl(s->lock);
+    OSDSession *s = siter->second;
+    OSDSession::shared_lock sl(s->lock);
     _dump_active(s);
     sl.unlock();
   }
@@ -4485,7 +4536,9 @@ void Objecter::dump_requests(Formatter *fmt)
 
 void Objecter::_dump_ops(const OSDSession *s, Formatter *fmt)
 {
-  for (auto p = s->ops.begin(); p != s->ops.end(); ++p) {
+  for (map<ceph_tid_t,Op*>::const_iterator p = s->ops.begin();
+       p != s->ops.end();
+       ++p) {
     Op *op = p->second;
     auto age = std::chrono::duration<double>(coarse_mono_clock::now() - op->stamp);
     fmt->open_object_section("op");
@@ -4499,7 +4552,9 @@ void Objecter::_dump_ops(const OSDSession *s, Formatter *fmt)
     fmt->dump_stream("mtime") << op->mtime;
 
     fmt->open_array_section("osd_ops");
-    for (auto it = op->ops.begin(); it != op->ops.end(); ++it) {
+    for (vector<OSDOp>::const_iterator it = op->ops.begin();
+	 it != op->ops.end();
+	 ++it) {
       fmt->dump_stream("osd_op") << *it;
     }
     fmt->close_section(); // osd_ops array
@@ -4512,10 +4567,10 @@ void Objecter::dump_ops(Formatter *fmt)
 {
   // Read-lock on Objecter held
   fmt->open_array_section("ops");
-  for (auto siter = osd_sessions.begin();
+  for (map<int, OSDSession *>::const_iterator siter = osd_sessions.begin();
        siter != osd_sessions.end(); ++siter) {
     OSDSession *s = siter->second;
-    shared_lock sl(s->lock);
+    OSDSession::shared_lock sl(s->lock);
     _dump_ops(s, fmt);
     sl.unlock();
   }
@@ -4525,8 +4580,10 @@ void Objecter::dump_ops(Formatter *fmt)
 
 void Objecter::_dump_linger_ops(const OSDSession *s, Formatter *fmt)
 {
-  for (auto p = s->linger_ops.begin(); p != s->linger_ops.end(); ++p) {
-    auto op = p->second;
+  for (map<uint64_t, LingerOp*>::const_iterator p = s->linger_ops.begin();
+       p != s->linger_ops.end();
+       ++p) {
+    LingerOp *op = p->second;
     fmt->open_object_section("linger_op");
     fmt->dump_unsigned("linger_id", op->linger_id);
     op->target.dump(fmt);
@@ -4540,10 +4597,10 @@ void Objecter::dump_linger_ops(Formatter *fmt)
 {
   // We have a read-lock on the objecter
   fmt->open_array_section("linger_ops");
-  for (auto siter = osd_sessions.begin();
+  for (map<int, OSDSession *>::const_iterator siter = osd_sessions.begin();
        siter != osd_sessions.end(); ++siter) {
-    auto s = siter->second;
-    shared_lock sl(s->lock);
+    OSDSession *s = siter->second;
+    OSDSession::shared_lock sl(s->lock);
     _dump_linger_ops(s, fmt);
     sl.unlock();
   }
@@ -4553,13 +4610,16 @@ void Objecter::dump_linger_ops(Formatter *fmt)
 
 void Objecter::_dump_command_ops(const OSDSession *s, Formatter *fmt)
 {
-  for (auto p = s->command_ops.begin(); p != s->command_ops.end(); ++p) {
-    auto op = p->second;
+  for (map<uint64_t, CommandOp*>::const_iterator p = s->command_ops.begin();
+       p != s->command_ops.end();
+       ++p) {
+    CommandOp *op = p->second;
     fmt->open_object_section("command_op");
     fmt->dump_unsigned("command_id", op->tid);
     fmt->dump_int("osd", op->session ? op->session->osd : -1);
     fmt->open_array_section("command");
-    for (auto q = op->cmd.begin(); q != op->cmd.end(); ++q)
+    for (vector<string>::const_iterator q = op->cmd.begin();
+	 q != op->cmd.end(); ++q)
       fmt->dump_string("word", *q);
     fmt->close_section();
     if (op->target_osd >= 0)
@@ -4574,10 +4634,10 @@ void Objecter::dump_command_ops(Formatter *fmt)
 {
   // We have a read-lock on the Objecter here
   fmt->open_array_section("command_ops");
-  for (auto siter = osd_sessions.begin();
+  for (map<int, OSDSession *>::const_iterator siter = osd_sessions.begin();
        siter != osd_sessions.end(); ++siter) {
-    auto s = siter->second;
-    shared_lock sl(s->lock);
+    OSDSession *s = siter->second;
+    OSDSession::shared_lock sl(s->lock);
     _dump_command_ops(s, fmt);
     sl.unlock();
   }
@@ -4588,8 +4648,10 @@ void Objecter::dump_command_ops(Formatter *fmt)
 void Objecter::dump_pool_ops(Formatter *fmt) const
 {
   fmt->open_array_section("pool_ops");
-  for (auto p = pool_ops.begin(); p != pool_ops.end(); ++p) {
-    auto op = p->second;
+  for (map<ceph_tid_t, PoolOp*>::const_iterator p = pool_ops.begin();
+       p != pool_ops.end();
+       ++p) {
+    PoolOp *op = p->second;
     fmt->open_object_section("pool_op");
     fmt->dump_unsigned("tid", op->tid);
     fmt->dump_int("pool", op->pool);
@@ -4606,7 +4668,7 @@ void Objecter::dump_pool_ops(Formatter *fmt) const
 void Objecter::dump_pool_stat_ops(Formatter *fmt) const
 {
   fmt->open_array_section("pool_stat_ops");
-  for (auto p = poolstat_ops.begin();
+  for (map<ceph_tid_t, PoolStatOp*>::const_iterator p = poolstat_ops.begin();
        p != poolstat_ops.end();
        ++p) {
     PoolStatOp *op = p->second;
@@ -4615,8 +4677,10 @@ void Objecter::dump_pool_stat_ops(Formatter *fmt) const
     fmt->dump_stream("last_sent") << op->last_submit;
 
     fmt->open_array_section("pools");
-    for (const auto& it : op->pools) {
-      fmt->dump_string("pool", it);
+    for (list<string>::const_iterator it = op->pools.begin();
+	 it != op->pools.end();
+	 ++it) {
+      fmt->dump_string("pool", *it);
     }
     fmt->close_section(); // pools array
 
@@ -4628,8 +4692,10 @@ void Objecter::dump_pool_stat_ops(Formatter *fmt) const
 void Objecter::dump_statfs_ops(Formatter *fmt) const
 {
   fmt->open_array_section("statfs_ops");
-  for (auto p = statfs_ops.begin(); p != statfs_ops.end(); ++p) {
-    auto op = p->second;
+  for (map<ceph_tid_t, StatfsOp*>::const_iterator p = statfs_ops.begin();
+       p != statfs_ops.end();
+       ++p) {
+    StatfsOp *op = p->second;
     fmt->open_object_section("statfs_op");
     fmt->dump_unsigned("tid", op->tid);
     fmt->dump_stream("last_sent") << op->last_submit;
@@ -4647,7 +4713,7 @@ int Objecter::RequestStateHook::call(std::string_view command,
 				     const cmdmap_t& cmdmap,
 				     Formatter *f,
 				     std::ostream& ss,
-				     cb::list& out)
+				     ceph::buffer::list& out)
 {
   shared_lock rl(m_objecter->rwlock);
   m_objecter->dump_requests(f);
@@ -4669,7 +4735,7 @@ void Objecter::blacklist_self(bool set)
   ss << messenger->get_myaddrs().front().get_legacy_str();
   cmd.push_back("\"addr\":\"" + ss.str() + "\"");
 
-  auto m = new MMonCommand(monc->get_fsid());
+  MMonCommand *m = new MMonCommand(monc->get_fsid());
   m->cmd = cmd;
 
   monc->send_mon_message(m);
@@ -4694,8 +4760,8 @@ void Objecter::handle_command_reply(MCommandReply *m)
     return;
   }
 
-  shared_lock sl(s->lock);
-  auto p = s->command_ops.find(m->get_tid());
+  OSDSession::shared_lock sl(s->lock);
+  map<ceph_tid_t,CommandOp*>::iterator p = s->command_ops.find(m->get_tid());
   if (p == s->command_ops.end()) {
     ldout(cct, 10) << "handle_command_reply tid " << m->get_tid()
 		   << " not found" << dendl;
@@ -4715,7 +4781,6 @@ void Objecter::handle_command_reply(MCommandReply *m)
     sl.unlock();
     return;
   }
-
   if (m->r == -EAGAIN) {
     ldout(cct,10) << __func__ << " tid " << m->get_tid()
 		  << " got EAGAIN, requesting map and resending" << dendl;
@@ -4728,12 +4793,14 @@ void Objecter::handle_command_reply(MCommandReply *m)
     return;
   }
 
+  if (c->poutbl) {
+    c->poutbl->claim(m->get_data());
+  }
+
   sl.unlock();
 
-  unique_lock sul(s->lock);
-  _finish_command(c, m->r < 0 ? bs::error_code(-m->r, osd_category()) :
-		  bs::error_code(), std::move(m->rs),
-		  std::move(m->get_data()));
+  OSDSession::unique_lock sul(s->lock);
+  _finish_command(c, m->r, m->rs);
   sul.unlock();
 
   m->put();
@@ -4748,7 +4815,7 @@ void Objecter::submit_command(CommandOp *c, ceph_tid_t *ptid)
   c->tid = tid;
 
   {
-    unique_lock hs_wl(homeless_session->lock);
+    OSDSession::unique_lock hs_wl(homeless_session->lock);
     _session_command_op_assign(homeless_session, c);
   }
 
@@ -4757,9 +4824,8 @@ void Objecter::submit_command(CommandOp *c, ceph_tid_t *ptid)
   if (osd_timeout > timespan(0)) {
     c->ontimeout = timer.add_event(osd_timeout,
 				   [this, c, tid]() {
-				     command_op_cancel(
-				       c->session, tid,
-				       osdc_errc::timed_out); });
+				     command_op_cancel(c->session, tid,
+						       -ETIMEDOUT); });
   }
 
   if (!c->session->is_homeless()) {
@@ -4769,14 +4835,12 @@ void Objecter::submit_command(CommandOp *c, ceph_tid_t *ptid)
   }
   if (c->map_check_error)
     _send_command_map_check(c);
-  if (ptid)
-    *ptid = tid;
+  *ptid = tid;
 
   logger->inc(l_osdc_command_active);
 }
 
-int Objecter::_calc_command_target(CommandOp *c,
-				   shunique_lock<ceph::shared_mutex>& sul)
+int Objecter::_calc_command_target(CommandOp *c, shunique_lock& sul)
 {
   ceph_assert(sul.owns_lock() && sul.mutex() == &rwlock);
 
@@ -4832,7 +4896,7 @@ int Objecter::_calc_command_target(CommandOp *c,
 }
 
 void Objecter::_assign_command_session(CommandOp *c,
-				       shunique_lock<ceph::shared_mutex>& sul)
+				       shunique_lock& sul)
 {
   ceph_assert(sul.owns_lock() && sul.mutex() == &rwlock);
 
@@ -4843,11 +4907,11 @@ void Objecter::_assign_command_session(CommandOp *c,
   if (c->session != s) {
     if (c->session) {
       OSDSession *cs = c->session;
-      unique_lock csl(cs->lock);
+      OSDSession::unique_lock csl(cs->lock);
       _session_command_op_remove(c->session, c);
       csl.unlock();
     }
-    unique_lock sl(s->lock);
+    OSDSession::unique_lock sl(s->lock);
     _session_command_op_assign(s, c);
   }
 
@@ -4859,7 +4923,7 @@ void Objecter::_send_command(CommandOp *c)
   ldout(cct, 10) << "_send_command " << c->tid << dendl;
   ceph_assert(c->session);
   ceph_assert(c->session->con);
-  auto m = new MCommand(monc->monmap.fsid);
+  MCommand *m = new MCommand(monc->monmap.fsid);
   m->cmd = c->cmd;
   m->set_data(c->inbl);
   m->set_tid(c->tid);
@@ -4867,14 +4931,13 @@ void Objecter::_send_command(CommandOp *c)
   logger->inc(l_osdc_command_send);
 }
 
-int Objecter::command_op_cancel(OSDSession *s, ceph_tid_t tid,
-				bs::error_code ec)
+int Objecter::command_op_cancel(OSDSession *s, ceph_tid_t tid, int r)
 {
   ceph_assert(initialized);
 
   unique_lock wl(rwlock);
 
-  auto it = s->command_ops.find(tid);
+  map<ceph_tid_t, CommandOp*>::iterator it = s->command_ops.find(tid);
   if (it == s->command_ops.end()) {
     ldout(cct, 10) << __func__ << " tid " << tid << " dne" << dendl;
     return -ENOENT;
@@ -4884,25 +4947,25 @@ int Objecter::command_op_cancel(OSDSession *s, ceph_tid_t tid,
 
   CommandOp *op = it->second;
   _command_cancel_map_check(op);
-  unique_lock sl(op->session->lock);
-  _finish_command(op, ec, {}, {});
+  OSDSession::unique_lock sl(op->session->lock);
+  _finish_command(op, r, "");
   sl.unlock();
   return 0;
 }
 
-void Objecter::_finish_command(CommandOp *c, bs::error_code ec,
-			       string&& rs, cb::list&& bl)
+void Objecter::_finish_command(CommandOp *c, int r, string rs)
 {
   // rwlock is locked unique
   // session lock is locked
 
-  ldout(cct, 10) << "_finish_command " << c->tid << " = " << ec << " "
+  ldout(cct, 10) << "_finish_command " << c->tid << " = " << r << " "
 		 << rs << dendl;
-
+  if (c->prs)
+    *c->prs = rs;
   if (c->onfinish)
-    c->onfinish->defer(std::move(c->onfinish), ec, std::move(rs), std::move(bl));
+    c->onfinish->complete(r);
 
-  if (c->ontimeout && ec != bs::errc::timed_out)
+  if (c->ontimeout && r != -ETIMEDOUT)
     timer.cancel_event(c->ontimeout);
 
   _session_command_op_remove(c->session, c);
@@ -4921,14 +4984,20 @@ Objecter::OSDSession::~OSDSession()
   ceph_assert(command_ops.empty());
 }
 
-Objecter::Objecter(CephContext *cct,
-		   Messenger *m, MonClient *mc,
-		   boost::asio::io_context& service,
+Objecter::Objecter(CephContext *cct_, Messenger *m, MonClient *mc,
+		   Finisher *fin,
 		   double mon_timeout,
 		   double osd_timeout) :
-  Dispatcher(cct), messenger(m), monc(mc), service(service),
+  Dispatcher(cct_), messenger(m), monc(mc), finisher(fin),
+  trace_endpoint("0.0.0.0", 0, "Objecter"),
+  osdmap{std::make_unique<OSDMap>()},
+  homeless_session(new OSDSession(cct, -1)),
   mon_timeout(ceph::make_timespan(mon_timeout)),
-  osd_timeout(ceph::make_timespan(osd_timeout))
+  osd_timeout(ceph::make_timespan(osd_timeout)),
+  op_throttle_bytes(cct, "objecter_bytes",
+		    cct->_conf->objecter_inflight_op_bytes),
+  op_throttle_ops(cct, "objecter_ops", cct->_conf->objecter_inflight_ops),
+  retry_writes_after_first_reply(cct->_conf->objecter_retry_writes_after_first_reply)
 {}
 
 Objecter::~Objecter()
@@ -4983,79 +5052,60 @@ hobject_t Objecter::enumerate_objects_end()
   return hobject_t::get_max();
 }
 
-struct EnumerationContext {
-  Objecter* objecter;
-  const hobject_t end;
-  const cb::list filter;
-  uint32_t max;
-  const object_locator_t oloc;
-  std::vector<librados::ListObjectImpl> ls;
-private:
-  fu2::unique_function<void(bs::error_code,
-			    std::vector<librados::ListObjectImpl>,
-			    hobject_t) &&> on_finish;
-public:
-  epoch_t epoch = 0;
-  int budget = -1;
-
-  EnumerationContext(Objecter* objecter,
-		     hobject_t end, cb::list filter,
-		     uint32_t max, object_locator_t oloc,
-		     decltype(on_finish) on_finish)
-    : objecter(objecter), end(std::move(end)), filter(std::move(filter)),
-      max(max), oloc(std::move(oloc)), on_finish(std::move(on_finish)) {}
-
-  void operator()(bs::error_code ec,
-		  std::vector<librados::ListObjectImpl> v,
-		  hobject_t h) && {
-    if (budget >= 0) {
-      objecter->put_op_budget_bytes(budget);
-      budget = -1;
-    }
-
-    std::move(on_finish)(ec, std::move(v), std::move(h));
-  }
-};
+struct C_EnumerateReply : public Context {
+  ceph::buffer::list bl;
 
-struct CB_EnumerateReply {
-  cb::list bl;
+  Objecter *objecter;
+  hobject_t *next;
+  std::list<librados::ListObjectImpl> *result;
+  const hobject_t end;
+  const int64_t pool_id;
+  Context *on_finish;
 
-  Objecter* objecter;
-  std::unique_ptr<EnumerationContext> ctx;
+  epoch_t epoch;
+  int budget;
 
-  CB_EnumerateReply(Objecter* objecter,
-		    std::unique_ptr<EnumerationContext>&& ctx) :
-    objecter(objecter), ctx(std::move(ctx)) {}
+  C_EnumerateReply(Objecter *objecter_, hobject_t *next_,
+      std::list<librados::ListObjectImpl> *result_,
+      const hobject_t end_, const int64_t pool_id_, Context *on_finish_) :
+    objecter(objecter_), next(next_), result(result_),
+    end(end_), pool_id(pool_id_), on_finish(on_finish_),
+    epoch(0), budget(-1)
+  {}
 
-  void operator()(bs::error_code ec) {
-    objecter->_enumerate_reply(std::move(bl), ec, std::move(ctx));
+  void finish(int r) override {
+    objecter->_enumerate_reply(
+      bl, r, end, pool_id, budget, epoch, result, next, on_finish);
   }
 };
 
 void Objecter::enumerate_objects(
-  int64_t pool_id,
-  std::string_view ns,
-  hobject_t start,
-  hobject_t end,
-  const uint32_t max,
-  const cb::list& filter_bl,
-  fu2::unique_function<void(bs::error_code,
-			    std::vector<librados::ListObjectImpl>,
-			    hobject_t) &&> on_finish) {
+    int64_t pool_id,
+    const std::string &ns,
+    const hobject_t &start,
+    const hobject_t &end,
+    const uint32_t max,
+    const ceph::buffer::list &filter_bl,
+    std::list<librados::ListObjectImpl> *result, 
+    hobject_t *next,
+    Context *on_finish)
+{
+  ceph_assert(result);
+
   if (!end.is_max() && start > end) {
     lderr(cct) << __func__ << ": start " << start << " > end " << end << dendl;
-    std::move(on_finish)(osdc_errc::precondition_violated, {}, {});
+    on_finish->complete(-EINVAL);
     return;
   }
 
   if (max < 1) {
     lderr(cct) << __func__ << ": result size may not be zero" << dendl;
-    std::move(on_finish)(osdc_errc::precondition_violated, {}, {});
+    on_finish->complete(-EINVAL);
     return;
   }
 
   if (start.is_max()) {
-    std::move(on_finish)({}, {}, {});
+    on_finish->complete(0);
     return;
   }
 
@@ -5064,92 +5114,92 @@ void Objecter::enumerate_objects(
   if (!osdmap->test_flag(CEPH_OSDMAP_SORTBITWISE)) {
     rl.unlock();
     lderr(cct) << __func__ << ": SORTBITWISE cluster flag not set" << dendl;
-    std::move(on_finish)(osdc_errc::not_supported, {}, {});
+    on_finish->complete(-EOPNOTSUPP);
     return;
   }
-  const pg_pool_t* p = osdmap->get_pg_pool(pool_id);
+  const pg_pool_t *p = osdmap->get_pg_pool(pool_id);
   if (!p) {
     lderr(cct) << __func__ << ": pool " << pool_id << " DNE in osd epoch "
 	       << osdmap->get_epoch() << dendl;
     rl.unlock();
-    std::move(on_finish)(osdc_errc::pool_dne, {}, {});
+    on_finish->complete(-ENOENT);
     return;
   } else {
     rl.unlock();
   }
 
-  _issue_enumerate(start,
-		   std::make_unique<EnumerationContext>(
-		     this, std::move(end), filter_bl,
-		     max, object_locator_t{pool_id, ns},
-		     std::move(on_finish)));
-}
+  ldout(cct, 20) << __func__ << ": start=" << start << " end=" << end << dendl;
+
+  // Stash completion state
+  C_EnumerateReply *on_ack = new C_EnumerateReply(
+      this, next, result, end, pool_id, on_finish);
 
-void Objecter::_issue_enumerate(hobject_t start,
-				std::unique_ptr<EnumerationContext> ctx) {
   ObjectOperation op;
-  auto c = ctx.get();
-  op.pg_nls(c->max, c->filter, start, osdmap->get_epoch());
-  auto on_ack = std::make_unique<CB_EnumerateReply>(this, std::move(ctx));
-  // I hate having to do this. Try to find a cleaner way
-  // later.
-  auto epoch = &c->epoch;
-  auto budget = &c->budget;
-  auto pbl = &on_ack->bl;
+  op.pg_nls(max, filter_bl, start, 0);
 
   // Issue.  See you later in _enumerate_reply
-  pg_read(start.get_hash(),
-	  c->oloc, op, pbl, 0,
-	  Op::OpComp::create(service.get_executor(),
-			     [c = std::move(on_ack)]
-			     (bs::error_code ec) mutable {
-			       (*c)(ec);
-			     }), epoch, budget);
+  object_locator_t oloc(pool_id, ns);
+  pg_read(start.get_hash(), oloc, op,
+	  &on_ack->bl, 0, on_ack, &on_ack->epoch, &on_ack->budget);
 }
 
 void Objecter::_enumerate_reply(
-  cb::list&& bl,
-  bs::error_code ec,
-  std::unique_ptr<EnumerationContext>&& ctx)
-{
-  if (ec) {
-    std::move(*ctx)(ec, {}, {});
+    ceph::buffer::list &bl,
+    int r,
+    const hobject_t &end,
+    const int64_t pool_id,
+    int budget,
+    epoch_t reply_epoch,
+    std::list<librados::ListObjectImpl> *result,
+    hobject_t *next,
+    Context *on_finish)
+{
+  if (budget >= 0) {
+    put_op_budget_bytes(budget);
+  }
+
+  if (r < 0) {
+    ldout(cct, 4) << __func__ << ": remote error " << r << dendl;
+    on_finish->complete(r);
     return;
   }
 
+  ceph_assert(next != NULL);
+
   // Decode the results
   auto iter = bl.cbegin();
   pg_nls_response_t response;
 
-  try {
-    decode(response, iter);
-    if (!iter.end()) {
-      // extra_info isn't used anywhere. We do this solely to preserve
-      // backward compatibility
-      cb::list legacy_extra_info;
-      decode(legacy_extra_info, iter);
-    }
-  } catch (const bs::system_error& e) {
-    std::move(*ctx)(e.code(), {}, {});
-    return;
-  }
-
-  shared_lock rl(rwlock);
-  auto pool = osdmap->get_pg_pool(ctx->oloc.get_pool());
-  rl.unlock();
-  if (!pool) {
-    // pool is gone, drop any results which are now meaningless.
-    std::move(*ctx)(osdc_errc::pool_dne, {}, {});
-    return;
+  decode(response, iter);
+  if (!iter.end()) {
+    // extra_info isn't used anywhere. We do this solely to preserve
+    // backward compatibility
+    ceph::buffer::list legacy_extra_info;
+    decode(legacy_extra_info, iter);
   }
 
-  hobject_t next;
-  if ((response.handle <= ctx->end)) {
-    next = response.handle;
+  ldout(cct, 10) << __func__ << ": got " << response.entries.size()
+		 << " handle " << response.handle
+		 << " reply_epoch " << reply_epoch << dendl;
+  ldout(cct, 20) << __func__ << ": response.entries.size "
+		 << response.entries.size() << ", response.entries "
+		 << response.entries << dendl;
+  if (response.handle <= end) {
+    *next = response.handle;
   } else {
-    next = ctx->end;
+    ldout(cct, 10) << __func__ << ": adjusted next down to end " << end
+		   << dendl;
+    *next = end;
 
     // drop anything after 'end'
+    shared_lock rl(rwlock);
+    const pg_pool_t *pool = osdmap->get_pg_pool(pool_id);
+    if (!pool) {
+      // pool is gone, drop any results which are now meaningless.
+      rl.unlock();
+      on_finish->complete(-ENOENT);
+      return;
+    }
     while (!response.entries.empty()) {
       uint32_t hash = response.entries.back().locator.empty() ?
 	pool->hash_key(response.entries.back().oid,
@@ -5160,49 +5210,34 @@ void Objecter::_enumerate_reply(
 		     response.entries.back().locator,
 		     CEPH_NOSNAP,
 		     hash,
-		     ctx->oloc.get_pool(),
+		     pool_id,
 		     response.entries.back().nspace);
-      if (last < ctx->end)
+      if (last < end)
 	break;
+      ldout(cct, 20) << __func__ << " dropping item " << last
+		     << " >= end " << end << dendl;
       response.entries.pop_back();
     }
+    rl.unlock();
   }
-
-  if (response.entries.size() <= ctx->max) {
-    ctx->max -= response.entries.size();
-    std::move(response.entries.begin(), response.entries.end(),
-	      std::back_inserter(ctx->ls));
-  } else {
-    auto i = response.entries.begin();
-    while (ctx->max > 0) {
-      ctx->ls.push_back(std::move(*i));
-      --(ctx->max);
-      ++i;
-    }
-    uint32_t hash =
-      i->locator.empty() ?
-      pool->hash_key(i->oid, i->nspace) :
-      pool->hash_key(i->locator, i->nspace);
-
-    next = hobject_t{i->oid, i->locator,
-		     CEPH_NOSNAP,
-		     hash,
-		     ctx->oloc.get_pool(),
-		     i->nspace};
+  if (!response.entries.empty()) {
+    result->merge(response.entries);
   }
 
-  if (next == ctx->end || ctx->max == 0) {
-    std::move(*ctx)(ec, std::move(ctx->ls), std::move(next));
-  } else {
-    _issue_enumerate(next, std::move(ctx));
-  }
+  // release the listing context's budget once all
+  // OPs (in the session) are finished
+#if 0
+  put_nlist_context_budget(list_context);
+#endif
+  on_finish->complete(r);
+  return;
 }
 
 namespace {
   using namespace librados;
 
   template <typename T>
-  void do_decode(std::vector<T>& items, std::vector<cb::list>& bls)
+  void do_decode(std::vector<T>& items, std::vector<ceph::buffer::list>& bls)
   {
     for (auto bl : bls) {
       auto p = bl.cbegin();
@@ -5213,19 +5248,19 @@ namespace {
   }
 
   struct C_ObjectOperation_scrub_ls : public Context {
-    cb::list bl;
-    uint32_t* interval;
+    ceph::buffer::list bl;
+    uint32_t *interval;
     std::vector<inconsistent_obj_t> *objects = nullptr;
     std::vector<inconsistent_snapset_t> *snapsets = nullptr;
-    int* rval;
+    int *rval;
 
-    C_ObjectOperation_scrub_ls(uint32_t* interval,
-			       std::vector<inconsistent_obj_t>* objects,
-			       int* rval)
+    C_ObjectOperation_scrub_ls(uint32_t *interval,
+			       std::vector<inconsistent_obj_t> *objects,
+			       int *rval)
       : interval(interval), objects(objects), rval(rval) {}
-    C_ObjectOperation_scrub_ls(uint32_t* interval,
-			       std::vector<inconsistent_snapset_t>* snapsets,
-			       int* rval)
+    C_ObjectOperation_scrub_ls(uint32_t *interval,
+			       std::vector<inconsistent_snapset_t> *snapsets,
+			       int *rval)
       : interval(interval), snapsets(snapsets), rval(rval) {}
     void finish(int r) override {
       if (r < 0 && r != -EAGAIN) {
@@ -5239,7 +5274,7 @@ namespace {
 
       try {
 	decode();
-      } catch (cb::error&) {
+      } catch (ceph::buffer::error&) {
 	if (rval)
 	  *rval = -EIO;
       }
@@ -5259,19 +5294,19 @@ namespace {
   };
 
   template <typename T>
-  void do_scrub_ls(::ObjectOperation* op,
+  void do_scrub_ls(::ObjectOperation *op,
 		   const scrub_ls_arg_t& arg,
 		   std::vector<T> *items,
-		   uint32_t* interval,
-		   int* rval)
+		   uint32_t *interval,
+		   int *rval)
   {
     OSDOp& osd_op = op->add_op(CEPH_OSD_OP_SCRUBLS);
     op->flags |= CEPH_OSD_FLAG_PGOP;
     ceph_assert(interval);
     arg.encode(osd_op.indata);
     unsigned p = op->ops.size() - 1;
-    auto h = new C_ObjectOperation_scrub_ls{interval, items, rval};
-    op->set_handler(h);
+    auto *h = new C_ObjectOperation_scrub_ls{interval, items, rval};
+    op->out_handler[p] = h;
     op->out_bl[p] = &h->bl;
     op->out_rval[p] = rval;
   }
@@ -5279,9 +5314,9 @@ namespace {
 
 void ::ObjectOperation::scrub_ls(const librados::object_id_t& start_after,
 				 uint64_t max_to_get,
-				 std::vector<librados::inconsistent_obj_t>* objects,
-				 uint32_t* interval,
-				 int* rval)
+				 std::vector<librados::inconsistent_obj_t> *objects,
+				 uint32_t *interval,
+				 int *rval)
 {
   scrub_ls_arg_t arg = {*interval, 0, start_after, max_to_get};
   do_scrub_ls(this, arg, objects, interval, rval);
diff --git a/src/osdc/Objecter.h b/src/osdc/Objecter.h
index a684fbcc83e..f669ce31c5f 100644
--- a/src/osdc/Objecter.h
+++ b/src/osdc/Objecter.h
@@ -21,33 +21,24 @@
 #include <mutex>
 #include <memory>
 #include <sstream>
-#include <string>
-#include <string_view>
 #include <type_traits>
 
-#include <boost/container/small_vector.hpp>
-#include <boost/asio.hpp>
+#include <boost/thread/shared_mutex.hpp>
 
-#include "include/buffer.h"
 #include "include/ceph_assert.h"
-#include "include/ceph_fs.h"
-#include "include/expected.hpp"
+#include "include/buffer.h"
 #include "include/types.h"
 #include "include/rados/rados_types.hpp"
-#include "include/function2.hpp"
 
 #include "common/admin_socket.h"
-#include "common/async/completion.h"
 #include "common/ceph_time.h"
-#include "common/ceph_mutex.h"
 #include "common/ceph_timer.h"
 #include "common/config_obs.h"
 #include "common/shunique_lock.h"
 #include "common/zipkin_trace.h"
+#include "common/Finisher.h"
 #include "common/Throttle.h"
 
-#include "mon/MonClient.h"
-
 #include "messages/MOSDOp.h"
 #include "msg/Dispatcher.h"
 
@@ -57,6 +48,7 @@ class Context;
 class Messenger;
 class MonClient;
 class Message;
+class Finisher;
 
 class MPoolOpReply;
 
@@ -66,47 +58,28 @@ class MCommandReply;
 class MWatchNotify;
 
 class PerfCounters;
-struct EnumerationContext;
-
-inline constexpr std::size_t osdc_opvec_len = 4;
-using osdc_opvec = boost::container::small_vector<OSDOp, osdc_opvec_len>;
-
 
 // -----------------------------------------
 
 struct ObjectOperation {
-  osdc_opvec ops;
-  int flags = 0;
-  int priority = 0;
-
-  boost::container::small_vector<ceph::buffer::list*, osdc_opvec_len> out_bl;
-  boost::container::small_vector<
-    fu2::unique_function<void(boost::system::error_code, int,
-			      const ceph::buffer::list& bl) &&>,
-    osdc_opvec_len> out_handler;
-  boost::container::small_vector<int*, osdc_opvec_len> out_rval;
-  boost::container::small_vector<boost::system::error_code*,
-				 osdc_opvec_len> out_ec;
-
-  ObjectOperation() = default;
-  ObjectOperation(const ObjectOperation&) = delete;
-  ObjectOperation& operator =(const ObjectOperation&) = delete;
-  ObjectOperation(ObjectOperation&&) = default;
-  ObjectOperation& operator =(ObjectOperation&&) = default;
-  ~ObjectOperation() = default;
-
-  size_t size() const {
-    return ops.size();
+  std::vector<OSDOp> ops;
+  int flags;
+  int priority;
+
+  std::vector<ceph::buffer::list*> out_bl;
+  std::vector<Context*> out_handler;
+  std::vector<int*> out_rval;
+
+  ObjectOperation() : flags(0), priority(0) {}
+  ~ObjectOperation() {
+    while (!out_handler.empty()) {
+      delete out_handler.back();
+      out_handler.pop_back();
+    }
   }
 
-  void clear() {
-    ops.clear();
-    flags = 0;
-    priority = 0;
-    out_bl.clear();
-    out_handler.clear();
-    out_rval.clear();
-    out_ec.clear();
+  size_t size() {
+    return ops.size();
   }
 
   void set_last_op_flags(int flags) {
@@ -114,51 +87,24 @@ struct ObjectOperation {
     ops.rbegin()->op.flags = flags;
   }
 
-
-  void set_handler(fu2::unique_function<void(boost::system::error_code, int,
-					     const ceph::buffer::list&) &&> f) {
-    if (f) {
-      if (out_handler.back()) {
-	// This happens seldom enough that we may as well keep folding
-	// functions together when we get another one rather than
-	// using a container.
-	out_handler.back() =
-	  [f = std::move(f),
-	   g = std::move(std::move(out_handler.back()))]
-	  (boost::system::error_code ec, int r,
-	   const ceph::buffer::list& bl) mutable {
-	    std::move(g)(ec, r, bl);
-	    std::move(f)(ec, r, bl);
-	  };
-      } else {
-	out_handler.back() = std::move(f);
-      }
-    }
-    ceph_assert(ops.size() == out_handler.size());
-  }
-
-  void set_handler(Context *c) {
-    if (c)
-      set_handler([c = std::unique_ptr<Context>(c)](boost::system::error_code,
-						    int r,
-						    const ceph::buffer::list&) mutable {
-		    c.release()->complete(r);
-		  });
-
-  }
+  class C_TwoContexts;
+  /**
+   * Add a callback to run when this operation completes,
+   * after any other callbacks for it.
+   */
+  void add_handler(Context *extra);
 
   OSDOp& add_op(int op) {
-    ops.emplace_back();
-    ops.back().op.op = op;
-    out_bl.push_back(nullptr);
-    ceph_assert(ops.size() == out_bl.size());
-    out_handler.emplace_back();
-    ceph_assert(ops.size() == out_handler.size());
-    out_rval.push_back(nullptr);
-    ceph_assert(ops.size() == out_rval.size());
-    out_ec.push_back(nullptr);
-    ceph_assert(ops.size() == out_ec.size());
-    return ops.back();
+    int s = ops.size();
+    ops.resize(s+1);
+    ops[s].op.op = op;
+    out_bl.resize(s+1);
+    out_bl[s] = NULL;
+    out_handler.resize(s+1);
+    out_handler[s] = NULL;
+    out_rval.resize(s+1);
+    out_rval[s] = NULL;
+    return ops[s];
   }
   void add_data(int op, uint64_t off, uint64_t len, ceph::buffer::list& bl) {
     OSDOp& osd_op = add_op(op);
@@ -193,58 +139,21 @@ struct ObjectOperation {
       osd_op.indata.append(name, osd_op.op.xattr.name_len);
     osd_op.indata.append(data);
   }
-  void add_xattr(int op, std::string_view name, const ceph::buffer::list& data) {
-    OSDOp& osd_op = add_op(op);
-    osd_op.op.xattr.name_len = name.size();
-    osd_op.op.xattr.value_len = data.length();
-    osd_op.indata.append(name.data(), osd_op.op.xattr.name_len);
-    osd_op.indata.append(data);
-  }
-  void add_xattr_cmp(int op, std::string_view name, uint8_t cmp_op,
-		     uint8_t cmp_mode, const ceph::buffer::list& data) {
-    OSDOp& osd_op = add_op(op);
-    osd_op.op.xattr.name_len = name.size();
-    osd_op.op.xattr.value_len = data.length();
-    osd_op.op.xattr.cmp_op = cmp_op;
-    osd_op.op.xattr.cmp_mode = cmp_mode;
-    if (!name.empty())
-      osd_op.indata.append(name.data(), osd_op.op.xattr.name_len);
-    osd_op.indata.append(data);
-  }
-  void add_call(int op, std::string_view cname, std::string_view method,
-		const ceph::buffer::list &indata,
+  void add_call(int op, const char *cname, const char *method,
+		ceph::buffer::list &indata,
 		ceph::buffer::list *outbl, Context *ctx, int *prval) {
     OSDOp& osd_op = add_op(op);
 
     unsigned p = ops.size() - 1;
-    set_handler(ctx);
+    out_handler[p] = ctx;
     out_bl[p] = outbl;
     out_rval[p] = prval;
 
-    osd_op.op.cls.class_len = cname.size();
-    osd_op.op.cls.method_len = method.size();
-    osd_op.op.cls.indata_len = indata.length();
-    osd_op.indata.append(cname.data(), osd_op.op.cls.class_len);
-    osd_op.indata.append(method.data(), osd_op.op.cls.method_len);
-    osd_op.indata.append(indata);
-  }
-  void add_call(int op, std::string_view cname, std::string_view method,
-		const ceph::buffer::list &indata,
-		fu2::unique_function<void(boost::system::error_code,
-					  const ceph::buffer::list&) &&> f) {
-    OSDOp& osd_op = add_op(op);
-
-    set_handler([f = std::move(f)](boost::system::error_code ec,
-				   int,
-				   const ceph::buffer::list& bl) mutable {
-		  std::move(f)(ec, bl);
-		});
-
-    osd_op.op.cls.class_len = cname.size();
-    osd_op.op.cls.method_len = method.size();
+    osd_op.op.cls.class_len = strlen(cname);
+    osd_op.op.cls.method_len = strlen(method);
     osd_op.op.cls.indata_len = indata.length();
-    osd_op.indata.append(cname.data(), osd_op.op.cls.class_len);
-    osd_op.indata.append(method.data(), osd_op.op.cls.method_len);
+    osd_op.indata.append(cname, osd_op.op.cls.class_len);
+    osd_op.indata.append(method, osd_op.op.cls.method_len);
     osd_op.indata.append(indata);
   }
   void add_pgls(int op, uint64_t count, collection_list_handle_t cookie,
@@ -316,18 +225,17 @@ struct ObjectOperation {
     o.op.flags = (excl ? CEPH_OSD_OP_FLAG_EXCL : 0);
   }
 
-  struct CB_ObjectOperation_stat {
+  struct C_ObjectOperation_stat : public Context {
     ceph::buffer::list bl;
     uint64_t *psize;
     ceph::real_time *pmtime;
     time_t *ptime;
     struct timespec *pts;
     int *prval;
-    boost::system::error_code* pec;
-    CB_ObjectOperation_stat(uint64_t *ps, ceph::real_time *pm, time_t *pt, struct timespec *_pts,
-			    int *prval, boost::system::error_code* pec)
-      : psize(ps), pmtime(pm), ptime(pt), pts(_pts), prval(prval), pec(pec) {}
-    void operator()(boost::system::error_code ec, int r, const ceph::buffer::list& bl) {
+    C_ObjectOperation_stat(uint64_t *ps, ceph::real_time *pm, time_t *pt, struct timespec *_pts,
+			   int *prval)
+      : psize(ps), pmtime(pm), ptime(pt), pts(_pts), prval(prval) {}
+    void finish(int r) override {
       using ceph::decode;
       if (r >= 0) {
 	auto p = bl.cbegin();
@@ -344,91 +252,58 @@ struct ObjectOperation {
 	    *ptime = ceph::real_clock::to_time_t(mtime);
 	  if (pts)
 	    *pts = ceph::real_clock::to_timespec(mtime);
-	} catch (const ceph::buffer::error& e) {
+	} catch (ceph::buffer::error& e) {
 	  if (prval)
 	    *prval = -EIO;
-	  if (pec)
-	    *pec = e.code();
 	}
       }
     }
   };
   void stat(uint64_t *psize, ceph::real_time *pmtime, int *prval) {
     add_op(CEPH_OSD_OP_STAT);
-    set_handler(CB_ObjectOperation_stat(psize, pmtime, nullptr, nullptr, prval,
-					nullptr));
-    out_rval.back() = prval;
-  }
-  void stat(uint64_t *psize, ceph::real_time *pmtime,
-	    boost::system::error_code* ec) {
-    add_op(CEPH_OSD_OP_STAT);
-    set_handler(CB_ObjectOperation_stat(psize, pmtime, nullptr, nullptr,
-					nullptr, ec));
-    out_ec.back() = ec;
+    unsigned p = ops.size() - 1;
+    C_ObjectOperation_stat *h = new C_ObjectOperation_stat(psize, pmtime, NULL, NULL,
+							   prval);
+    out_bl[p] = &h->bl;
+    out_handler[p] = h;
+    out_rval[p] = prval;
   }
   void stat(uint64_t *psize, time_t *ptime, int *prval) {
     add_op(CEPH_OSD_OP_STAT);
-    set_handler(CB_ObjectOperation_stat(psize, nullptr, ptime, nullptr, prval,
-					nullptr));
-    out_rval.back() = prval;
+    unsigned p = ops.size() - 1;
+    C_ObjectOperation_stat *h = new C_ObjectOperation_stat(psize, NULL, ptime, NULL,
+							   prval);
+    out_bl[p] = &h->bl;
+    out_handler[p] = h;
+    out_rval[p] = prval;
   }
   void stat(uint64_t *psize, struct timespec *pts, int *prval) {
     add_op(CEPH_OSD_OP_STAT);
-    set_handler(CB_ObjectOperation_stat(psize, nullptr, nullptr, pts, prval, nullptr));
-    out_rval.back() = prval;
-  }
-  void stat(uint64_t *psize, ceph::real_time *pmtime, nullptr_t) {
-    add_op(CEPH_OSD_OP_STAT);
-    set_handler(CB_ObjectOperation_stat(psize, pmtime, nullptr, nullptr, nullptr,
-					nullptr));
-  }
-  void stat(uint64_t *psize, time_t *ptime, nullptr_t) {
-    add_op(CEPH_OSD_OP_STAT);
-    set_handler(CB_ObjectOperation_stat(psize, nullptr, ptime, nullptr, nullptr,
-					nullptr));
-  }
-  void stat(uint64_t *psize, struct timespec *pts, nullptr_t) {
-    add_op(CEPH_OSD_OP_STAT);
-    set_handler(CB_ObjectOperation_stat(psize, nullptr, nullptr, pts, nullptr,
-					nullptr));
-  }
-  void stat(uint64_t *psize, nullptr_t, nullptr_t) {
-    add_op(CEPH_OSD_OP_STAT);
-    set_handler(CB_ObjectOperation_stat(psize, nullptr, nullptr, nullptr,
-					nullptr, nullptr));
+    unsigned p = ops.size() - 1;
+    C_ObjectOperation_stat *h = new C_ObjectOperation_stat(psize, NULL, NULL, pts,
+							   prval);
+    out_bl[p] = &h->bl;
+    out_handler[p] = h;
+    out_rval[p] = prval;
   }
-
   // object cmpext
-  struct CB_ObjectOperation_cmpext {
-    int* prval = nullptr;
-    boost::system::error_code* ec = nullptr;
-    std::size_t* s = nullptr;
-    explicit CB_ObjectOperation_cmpext(int *prval)
+  struct C_ObjectOperation_cmpext : public Context {
+    int *prval;
+    explicit C_ObjectOperation_cmpext(int *prval)
       : prval(prval) {}
-    CB_ObjectOperation_cmpext(boost::system::error_code* ec, std::size_t* s)
-      : ec(ec), s(s) {}
 
-    void operator()(boost::system::error_code ec, int r, const ceph::buffer::list&) {
+    void finish(int r) {
       if (prval)
         *prval = r;
-      if (this->ec)
-	*this->ec = ec;
-      if (s)
-	*s = static_cast<std::size_t>(-(MAX_ERRNO - r));
     }
   };
 
   void cmpext(uint64_t off, ceph::buffer::list& cmp_bl, int *prval) {
     add_data(CEPH_OSD_OP_CMPEXT, off, cmp_bl.length(), cmp_bl);
-    set_handler(CB_ObjectOperation_cmpext(prval));
-    out_rval.back() = prval;
-  }
-
-  void cmpext(uint64_t off, ceph::buffer::list&& cmp_bl, boost::system::error_code* ec,
-	      std::size_t* s) {
-    add_data(CEPH_OSD_OP_CMPEXT, off, cmp_bl.length(), cmp_bl);
-    set_handler(CB_ObjectOperation_cmpext(ec, s));
-    out_ec.back() = ec;
+    unsigned p = ops.size() - 1;
+    C_ObjectOperation_cmpext *h = new C_ObjectOperation_cmpext(prval);
+    out_handler[p] = h;
+    out_rval[p] = prval;
   }
 
   // Used by C API
@@ -436,8 +311,10 @@ struct ObjectOperation {
     ceph::buffer::list cmp_bl;
     cmp_bl.append(cmp_buf, cmp_len);
     add_data(CEPH_OSD_OP_CMPEXT, off, cmp_len, cmp_bl);
-    set_handler(CB_ObjectOperation_cmpext(prval));
-    out_rval.back() = prval;
+    unsigned p = ops.size() - 1;
+    C_ObjectOperation_cmpext *h = new C_ObjectOperation_cmpext(prval);
+    out_handler[p] = h;
+    out_rval[p] = prval;
   }
 
   void read(uint64_t off, uint64_t len, ceph::buffer::list *pbl, int *prval,
@@ -447,29 +324,20 @@ struct ObjectOperation {
     unsigned p = ops.size() - 1;
     out_bl[p] = pbl;
     out_rval[p] = prval;
-    set_handler(ctx);
+    out_handler[p] = ctx;
   }
 
-  void read(uint64_t off, uint64_t len, boost::system::error_code* ec,
-	    ceph::buffer::list* pbl) {
+  struct C_ObjectOperation_sparse_read : public Context {
     ceph::buffer::list bl;
-    add_data(CEPH_OSD_OP_READ, off, len, bl);
-    out_ec.back() = ec;
-    out_bl.back() = pbl;
-  }
-
-  template<typename Ex>
-  struct CB_ObjectOperation_sparse_read {
-    ceph::buffer::list* data_bl;
-    Ex* extents;
-    int* prval;
-    boost::system::error_code* pec;
-    CB_ObjectOperation_sparse_read(ceph::buffer::list* data_bl,
-				   Ex* extents,
-				   int* prval,
-				   boost::system::error_code* pec)
-      : data_bl(data_bl), extents(extents), prval(prval), pec(pec) {}
-    void operator()(boost::system::error_code ec, int r, const ceph::buffer::list& bl) {
+    ceph::buffer::list *data_bl;
+    std::map<uint64_t, uint64_t> *extents;
+    int *prval;
+    C_ObjectOperation_sparse_read(ceph::buffer::list *data_bl,
+				  std::map<uint64_t, uint64_t> *extents,
+				  int *prval)
+      : data_bl(data_bl), extents(extents), prval(prval) {}
+    void finish(int r) override {
+      using ceph::decode;
       auto iter = bl.cbegin();
       if (r >= 0) {
         // NOTE: it's possible the sub-op has not been executed but the result
@@ -479,35 +347,26 @@ struct ObjectOperation {
 	  try {
 	    decode(*extents, iter);
 	    decode(*data_bl, iter);
-	  } catch (const ceph::buffer::error& e) {
+	  } catch (ceph::buffer::error& e) {
 	    if (prval)
               *prval = -EIO;
-	    if (pec)
-	      *pec = e.code();
 	  }
         } else if (prval) {
           *prval = -EIO;
-	  if (pec)
-	    *pec = buffer::errc::end_of_buffer;
-	}
+        }
       }
     }
   };
-  void sparse_read(uint64_t off, uint64_t len, std::map<uint64_t, uint64_t>* m,
-		   ceph::buffer::list* data_bl, int* prval) {
+  void sparse_read(uint64_t off, uint64_t len, std::map<uint64_t,uint64_t> *m,
+		   ceph::buffer::list *data_bl, int *prval) {
     ceph::buffer::list bl;
     add_data(CEPH_OSD_OP_SPARSE_READ, off, len, bl);
-    set_handler(CB_ObjectOperation_sparse_read(data_bl, m, prval, nullptr));
-    out_rval.back() = prval;
-  }
-  void sparse_read(uint64_t off, uint64_t len,
-		   boost::system::error_code* ec,
-		   std::vector<std::pair<uint64_t, uint64_t>>* m,
-		   ceph::buffer::list* data_bl) {
-    ceph::buffer::list bl;
-    add_data(CEPH_OSD_OP_SPARSE_READ, off, len, bl);
-    set_handler(CB_ObjectOperation_sparse_read(data_bl, m, nullptr, ec));
-    out_ec.back() = ec;
+    unsigned p = ops.size() - 1;
+    C_ObjectOperation_sparse_read *h =
+      new C_ObjectOperation_sparse_read(data_bl, m, prval);
+    out_bl[p] = &h->bl;
+    out_handler[p] = h;
+    out_rval[p] = prval;
   }
   void write(uint64_t off, ceph::buffer::list& bl,
 	     uint64_t truncate_size,
@@ -563,7 +422,7 @@ struct ObjectOperation {
     unsigned p = ops.size() - 1;
     out_bl[p] = pbl;
     out_rval[p] = prval;
-    set_handler(ctx);
+    out_handler[p] = ctx;
   }
 
   // object attrs
@@ -574,37 +433,28 @@ struct ObjectOperation {
     out_bl[p] = pbl;
     out_rval[p] = prval;
   }
-  void getxattr(std::string_view name, boost::system::error_code* ec,
-		buffer::list *pbl) {
-    ceph::buffer::list bl;
-    add_xattr(CEPH_OSD_OP_GETXATTR, name, bl);
-    out_bl.back() = pbl;
-    out_ec.back() = ec;
-  }
-
-  template<typename Vals>
-  struct CB_ObjectOperation_decodevals {
+  struct C_ObjectOperation_decodevals : public Context {
     uint64_t max_entries;
-    Vals* pattrs;
-    bool* ptruncated;
-    int* prval;
-    boost::system::error_code* pec;
-    CB_ObjectOperation_decodevals(uint64_t m, Vals* pa,
-				  bool *pt, int *pr,
-				  boost::system::error_code* pec)
-      : max_entries(m), pattrs(pa), ptruncated(pt), prval(pr), pec(pec) {
+    ceph::buffer::list bl;
+    std::map<std::string,ceph::buffer::list> *pattrs;
+    bool *ptruncated;
+    int *prval;
+    C_ObjectOperation_decodevals(uint64_t m, std::map<std::string,ceph::buffer::list> *pa,
+				 bool *pt, int *pr)
+      : max_entries(m), pattrs(pa), ptruncated(pt), prval(pr) {
       if (ptruncated) {
 	*ptruncated = false;
       }
     }
-    void operator()(boost::system::error_code ec, int r, const ceph::buffer::list& bl) {
+    void finish(int r) override {
+      using ceph::decode;
       if (r >= 0) {
 	auto p = bl.cbegin();
 	try {
 	  if (pattrs)
 	    decode(*pattrs, p);
 	  if (ptruncated) {
-	    Vals ignore;
+	    std::map<std::string,ceph::buffer::list> ignore;
 	    if (!pattrs) {
 	      decode(ignore, p);
 	      pattrs = &ignore;
@@ -612,36 +462,34 @@ struct ObjectOperation {
 	    if (!p.end()) {
 	      decode(*ptruncated, p);
 	    } else {
-	      // the OSD did not provide this.  Since old OSDs do not
+	      // the OSD did not provide this.  since old OSDs do not
 	      // enfoce omap result limits either, we can infer it from
 	      // the size of the result
 	      *ptruncated = (pattrs->size() == max_entries);
 	    }
 	  }
-	} catch (const ceph::buffer::error& e) {
+	}
+	catch (ceph::buffer::error& e) {
 	  if (prval)
 	    *prval = -EIO;
-	  if (pec)
-	    *pec = e.code();
 	}
       }
     }
   };
-  template<typename Keys>
-  struct CB_ObjectOperation_decodekeys {
+  struct C_ObjectOperation_decodekeys : public Context {
     uint64_t max_entries;
-    Keys* pattrs;
+    ceph::buffer::list bl;
+    std::set<std::string> *pattrs;
     bool *ptruncated;
     int *prval;
-    boost::system::error_code* pec;
-    CB_ObjectOperation_decodekeys(uint64_t m, Keys* pa, bool *pt,
-				  int *pr, boost::system::error_code* pec)
-      : max_entries(m), pattrs(pa), ptruncated(pt), prval(pr), pec(pec) {
+    C_ObjectOperation_decodekeys(uint64_t m, std::set<std::string> *pa, bool *pt,
+				 int *pr)
+      : max_entries(m), pattrs(pa), ptruncated(pt), prval(pr) {
       if (ptruncated) {
 	*ptruncated = false;
       }
     }
-    void operator()(boost::system::error_code ec, int r, const ceph::buffer::list& bl) {
+    void finish(int r) override {
       if (r >= 0) {
 	using ceph::decode;
 	auto p = bl.cbegin();
@@ -649,7 +497,7 @@ struct ObjectOperation {
 	  if (pattrs)
 	    decode(*pattrs, p);
 	  if (ptruncated) {
-	    Keys ignore;
+	    std::set<std::string> ignore;
 	    if (!pattrs) {
 	      decode(ignore, p);
 	      pattrs = &ignore;
@@ -663,24 +511,22 @@ struct ObjectOperation {
 	      *ptruncated = (pattrs->size() == max_entries);
 	    }
 	  }
-	} catch (const ceph::buffer::error& e) {
+	}
+	catch (ceph::buffer::error& e) {
 	  if (prval)
 	    *prval = -EIO;
-	  if (pec)
-	    *pec = e.code();
 	}
       }
     }
   };
-  template<typename Watchers>
-  struct CB_ObjectOperation_decodewatchers {
-    Watchers* pwatchers;
-    int* prval;
-    boost::system::error_code* pec;
-    CB_ObjectOperation_decodewatchers(Watchers* pw, int* pr,
-				      boost::system::error_code* pec)
-      : pwatchers(pw), prval(pr), pec(pec) {}
-    void operator()(boost::system::error_code ec, int r, const ceph::buffer::list& bl) {
+  struct C_ObjectOperation_decodewatchers : public Context {
+    ceph::buffer::list bl;
+    std::list<obj_watch_t> *pwatchers;
+    int *prval;
+    C_ObjectOperation_decodewatchers(std::list<obj_watch_t> *pw, int *pr)
+      : pwatchers(pw), prval(pr) {}
+    void finish(int r) override {
+      using ceph::decode;
       if (r >= 0) {
 	auto p = bl.cbegin();
 	try {
@@ -698,23 +544,21 @@ struct ObjectOperation {
 	      pwatchers->push_back(std::move(ow));
 	    }
 	  }
-	} catch (const ceph::buffer::error& e) {
+	}
+	catch (ceph::buffer::error& e) {
 	  if (prval)
 	    *prval = -EIO;
-	  if (pec)
-	    *pec = e.code();
 	}
       }
     }
   };
-  struct CB_ObjectOperation_decodesnaps {
+  struct C_ObjectOperation_decodesnaps : public Context {
+    ceph::buffer::list bl;
     librados::snap_set_t *psnaps;
     int *prval;
-    boost::system::error_code* pec;
-    CB_ObjectOperation_decodesnaps(librados::snap_set_t* ps, int* pr,
-				   boost::system::error_code* pec)
-      : psnaps(ps), prval(pr), pec(pec) {}
-    void operator()(boost::system::error_code ec, int r, const ceph::buffer::list& bl) {
+    C_ObjectOperation_decodesnaps(librados::snap_set_t *ps, int *pr)
+      : psnaps(ps), prval(pr) {}
+    void finish(int r) override {
       if (r >= 0) {
 	using ceph::decode;
 	auto p = bl.cbegin();
@@ -723,9 +567,7 @@ struct ObjectOperation {
 	  decode(resp, p);
 	  if (psnaps) {
 	    psnaps->clones.clear();
-	    for (auto ci = resp.clones.begin();
-		 ci != resp.clones.end();
-		 ++ci) {
+	    for (auto ci = resp.clones.begin(); ci != resp.clones.end(); ++ci) {
 	      librados::clone_info_t clone;
 
 	      clone.cloneid = ci->cloneid;
@@ -739,11 +581,9 @@ struct ObjectOperation {
 	    }
 	    psnaps->seq = resp.seq;
 	  }
-	} catch (const ceph::buffer::error& e) {
+	} catch (ceph::buffer::error& e) {
 	  if (prval)
 	    *prval = -EIO;
-	  if (pec)
-	    *pec = e.code();
 	}
       }
     }
@@ -751,23 +591,17 @@ struct ObjectOperation {
   void getxattrs(std::map<std::string,ceph::buffer::list> *pattrs, int *prval) {
     add_op(CEPH_OSD_OP_GETXATTRS);
     if (pattrs || prval) {
-      set_handler(CB_ObjectOperation_decodevals(0, pattrs, nullptr, prval,
-						nullptr));
-      out_rval.back() = prval;
+      unsigned p = ops.size() - 1;
+      C_ObjectOperation_decodevals *h
+	= new C_ObjectOperation_decodevals(0, pattrs, nullptr, prval);
+      out_handler[p] = h;
+      out_bl[p] = &h->bl;
+      out_rval[p] = prval;
     }
   }
-  void getxattrs(boost::system::error_code* ec,
-		 boost::container::flat_map<std::string, ceph::buffer::list> *pattrs) {
-    add_op(CEPH_OSD_OP_GETXATTRS);
-    set_handler(CB_ObjectOperation_decodevals(0, pattrs, nullptr, nullptr, ec));
-    out_ec.back() = ec;
-  }
   void setxattr(const char *name, const ceph::buffer::list& bl) {
     add_xattr(CEPH_OSD_OP_SETXATTR, name, bl);
   }
-  void setxattr(std::string_view name, const ceph::buffer::list& bl) {
-    add_xattr(CEPH_OSD_OP_SETXATTR, name, bl);
-  }
   void setxattr(const char *name, const std::string& s) {
     ceph::buffer::list bl;
     bl.append(s);
@@ -777,19 +611,11 @@ struct ObjectOperation {
 		const ceph::buffer::list& bl) {
     add_xattr_cmp(CEPH_OSD_OP_CMPXATTR, name, cmp_op, cmp_mode, bl);
   }
-  void cmpxattr(std::string_view name, uint8_t cmp_op, uint8_t cmp_mode,
-		const ceph::buffer::list& bl) {
-    add_xattr_cmp(CEPH_OSD_OP_CMPXATTR, name, cmp_op, cmp_mode, bl);
-  }
   void rmxattr(const char *name) {
     ceph::buffer::list bl;
     add_xattr(CEPH_OSD_OP_RMXATTR, name, bl);
   }
-  void rmxattr(std::string_view name) {
-    ceph::buffer::list bl;
-    add_xattr(CEPH_OSD_OP_RMXATTR, name, bl);
-  }
-  void setxattrs(map<string, ceph::buffer::list>& attrs) {
+  void setxattrs(std::map<std::string, ceph::buffer::list>& attrs) {
     using ceph::encode;
     ceph::buffer::list bl;
     encode(attrs, bl);
@@ -822,28 +648,14 @@ struct ObjectOperation {
     op.op.extent.length = bl.length();
     op.indata.claim_append(bl);
     if (prval || ptruncated || out_set) {
-      set_handler(CB_ObjectOperation_decodekeys(max_to_get, out_set, ptruncated, prval,
-						nullptr));
-      out_rval.back() = prval;
+      unsigned p = ops.size() - 1;
+      C_ObjectOperation_decodekeys *h =
+	new C_ObjectOperation_decodekeys(max_to_get, out_set, ptruncated, prval);
+      out_handler[p] = h;
+      out_bl[p] = &h->bl;
+      out_rval[p] = prval;
     }
   }
-  void omap_get_keys(std::optional<std::string_view> start_after,
-		     uint64_t max_to_get,
-		     boost::system::error_code* ec,
-		     boost::container::flat_set<std::string> *out_set,
-		     bool *ptruncated) {
-    OSDOp& op = add_op(CEPH_OSD_OP_OMAPGETKEYS);
-    ceph::buffer::list bl;
-    encode(start_after ? *start_after : std::string_view{}, bl);
-    encode(max_to_get, bl);
-    op.op.extent.offset = 0;
-    op.op.extent.length = bl.length();
-    op.indata.claim_append(bl);
-    set_handler(
-      CB_ObjectOperation_decodekeys(max_to_get, out_set, ptruncated, nullptr,
-				    ec));
-    out_ec.back() = ec;
-  }
 
   void omap_get_vals(const std::string &start_after,
 		     const std::string &filter_prefix,
@@ -861,34 +673,19 @@ struct ObjectOperation {
     op.op.extent.length = bl.length();
     op.indata.claim_append(bl);
     if (prval || out_set || ptruncated) {
-      set_handler(CB_ObjectOperation_decodevals(max_to_get, out_set, ptruncated,
-						prval, nullptr));
-      out_rval.back() = prval;
+      unsigned p = ops.size() - 1;
+      C_ObjectOperation_decodevals *h =
+	new C_ObjectOperation_decodevals(max_to_get, out_set, ptruncated, prval);
+      out_handler[p] = h;
+      out_bl[p] = &h->bl;
+      out_rval[p] = prval;
     }
   }
 
-  void omap_get_vals(std::optional<std::string_view> start_after,
-		     std::optional<std::string_view> filter_prefix,
-		     uint64_t max_to_get,
-		     boost::system::error_code* ec,
-		     boost::container::flat_map<std::string, ceph::buffer::list> *out_set,
-		     bool *ptruncated) {
-    OSDOp &op = add_op(CEPH_OSD_OP_OMAPGETVALS);
-    ceph::buffer::list bl;
-    encode(start_after ? *start_after : std::string_view{}, bl);
-    encode(max_to_get, bl);
-    encode(filter_prefix ? *start_after : std::string_view{}, bl);
-    op.op.extent.offset = 0;
-    op.op.extent.length = bl.length();
-    op.indata.claim_append(bl);
-    set_handler(CB_ObjectOperation_decodevals(max_to_get, out_set, ptruncated,
-					      nullptr, ec));
-    out_ec.back() = ec;
-  }
-
   void omap_get_vals_by_keys(const std::set<std::string> &to_get,
-			     std::map<std::string, ceph::buffer::list> *out_set,
-			     int *prval) {
+			    std::map<std::string, ceph::buffer::list> *out_set,
+			    int *prval) {
+    using ceph::encode;
     OSDOp &op = add_op(CEPH_OSD_OP_OMAPGETVALSBYKEYS);
     ceph::buffer::list bl;
     encode(to_get, bl);
@@ -896,28 +693,16 @@ struct ObjectOperation {
     op.op.extent.length = bl.length();
     op.indata.claim_append(bl);
     if (prval || out_set) {
-      set_handler(CB_ObjectOperation_decodevals(0, out_set, nullptr, prval,
-						nullptr));
-      out_rval.back() = prval;
+      unsigned p = ops.size() - 1;
+      C_ObjectOperation_decodevals *h =
+	new C_ObjectOperation_decodevals(0, out_set, nullptr, prval);
+      out_handler[p] = h;
+      out_bl[p] = &h->bl;
+      out_rval[p] = prval;
     }
   }
 
-  void omap_get_vals_by_keys(
-    const boost::container::flat_set<std::string>& to_get,
-    boost::system::error_code* ec,
-    boost::container::flat_map<std::string, ceph::buffer::list> *out_set) {
-    OSDOp &op = add_op(CEPH_OSD_OP_OMAPGETVALSBYKEYS);
-    ceph::buffer::list bl;
-    encode(to_get, bl);
-    op.op.extent.offset = 0;
-    op.op.extent.length = bl.length();
-    op.indata.claim_append(bl);
-    set_handler(CB_ObjectOperation_decodevals(0, out_set, nullptr, nullptr,
-					      ec));
-    out_ec.back() = ec;
-  }
-
-  void omap_cmp(const std::map<std::string, pair<ceph::buffer::list,int> > &assertions,
+  void omap_cmp(const std::map<std::string, std::pair<ceph::buffer::list,int> > &assertions,
 		int *prval) {
     using ceph::encode;
     OSDOp &op = add_op(CEPH_OSD_OP_OMAP_CMP);
@@ -932,18 +717,6 @@ struct ObjectOperation {
     }
   }
 
-  void omap_cmp(const boost::container::flat_map<
-		  std::string, pair<ceph::buffer::list, int>>& assertions,
-		boost::system::error_code *ec) {
-    OSDOp &op = add_op(CEPH_OSD_OP_OMAP_CMP);
-    ceph::buffer::list bl;
-    encode(assertions, bl);
-    op.op.extent.offset = 0;
-    op.op.extent.length = bl.length();
-    op.indata.claim_append(bl);
-    out_ec.back() = ec;
-  }
-
   struct C_ObjectOperation_copyget : public Context {
     ceph::buffer::list bl;
     object_copy_cursor_t *cursor;
@@ -1032,7 +805,7 @@ struct ObjectOperation {
 	if (out_truncate_size)
 	  *out_truncate_size = copy_reply.truncate_size;
 	*cursor = copy_reply.cursor;
-      } catch (const ceph::buffer::error& e) {
+      } catch (ceph::buffer::error& e) {
 	if (prval)
 	  *prval = -EIO;
       }
@@ -1073,7 +846,7 @@ struct ObjectOperation {
 				    out_reqid_return_codes, truncate_seq,
 				    truncate_size, prval);
     out_bl[p] = &h->bl;
-    set_handler(h);
+    out_handler[p] = h;
   }
 
   void undirty() {
@@ -1096,7 +869,7 @@ struct ObjectOperation {
 	decode(isdirty, p);
 	if (pisdirty)
 	  *pisdirty = isdirty;
-      } catch (const ceph::buffer::error& e) {
+      } catch (ceph::buffer::error& e) {
 	if (prval)
 	  *prval = -EIO;
       }
@@ -1110,7 +883,7 @@ struct ObjectOperation {
     C_ObjectOperation_isdirty *h =
       new C_ObjectOperation_isdirty(pisdirty, prval);
     out_bl[p] = &h->bl;
-    set_handler(h);
+    out_handler[p] = h;
   }
 
   struct C_ObjectOperation_hit_set_ls : public Context {
@@ -1145,7 +918,7 @@ struct ObjectOperation {
 	}
 	if (putls)
 	  putls->swap(ls);
-      } catch (const ceph::buffer::error& e) {
+      } catch (ceph::buffer::error& e) {
 	r = -EIO;
       }
       if (prval)
@@ -1170,7 +943,7 @@ struct ObjectOperation {
     C_ObjectOperation_hit_set_ls *h =
       new C_ObjectOperation_hit_set_ls(pls, NULL, prval);
     out_bl[p] = &h->bl;
-    set_handler(h);
+    out_handler[p] = h;
   }
   void hit_set_ls(std::list<std::pair<ceph::real_time, ceph::real_time> > *pls,
 		  int *prval) {
@@ -1180,7 +953,7 @@ struct ObjectOperation {
     C_ObjectOperation_hit_set_ls *h =
       new C_ObjectOperation_hit_set_ls(NULL, pls, prval);
     out_bl[p] = &h->bl;
-    set_handler(h);
+    out_handler[p] = h;
   }
 
   /**
@@ -1208,19 +981,8 @@ struct ObjectOperation {
     out_rval[p] = prval;
   }
 
-  void omap_get_header(boost::system::error_code* ec, ceph::buffer::list *bl) {
-    add_op(CEPH_OSD_OP_OMAPGETHEADER);
-    out_bl.back() = bl;
-    out_ec.back() = ec;
-  }
-
-  void omap_set(const map<string, ceph::buffer::list> &map) {
-    ceph::buffer::list bl;
-    encode(map, bl);
-    add_data(CEPH_OSD_OP_OMAPSETVALS, 0, bl.length(), bl);
-  }
-
-  void omap_set(const boost::container::flat_map<string, ceph::buffer::list>& map) {
+  void omap_set(const std::map<std::string, ceph::buffer::list>& map) {
+    using ceph::encode;
     ceph::buffer::list bl;
     encode(map, bl);
     add_data(CEPH_OSD_OP_OMAPSETVALS, 0, bl.length(), bl);
@@ -1240,11 +1002,6 @@ struct ObjectOperation {
     encode(to_remove, bl);
     add_data(CEPH_OSD_OP_OMAPRMKEYS, 0, bl.length(), bl);
   }
-  void omap_rm_keys(const boost::container::flat_set<std::string>& to_remove) {
-    ceph::buffer::list bl;
-    encode(to_remove, bl);
-    add_data(CEPH_OSD_OP_OMAPRMKEYS, 0, bl.length(), bl);
-  }
 
   void omap_rm_range(std::string_view key_begin, std::string_view key_end) {
     bufferlist bl;
@@ -1263,24 +1020,6 @@ struct ObjectOperation {
     add_call(CEPH_OSD_OP_CALL, cname, method, indata, outdata, ctx, prval);
   }
 
-  void call(std::string_view cname, std::string_view method,
-	    const ceph::buffer::list& indata, boost::system::error_code* ec) {
-    add_call(CEPH_OSD_OP_CALL, cname, method, indata, NULL, NULL, NULL);
-    out_ec.back() = ec;
-  }
-
-  void call(std::string_view cname, std::string_view method, const ceph::buffer::list& indata,
-	    boost::system::error_code* ec, ceph::buffer::list *outdata) {
-    add_call(CEPH_OSD_OP_CALL, cname, method, indata, outdata, nullptr, nullptr);
-    out_ec.back() = ec;
-  }
-  void call(std::string_view cname, std::string_view method,
-	    const ceph::buffer::list& indata,
-	    fu2::unique_function<void (boost::system::error_code,
-				       const ceph::buffer::list&) &&> f) {
-    add_call(CEPH_OSD_OP_CALL, cname, method, indata, std::move(f));
-  }
-
   // watch/notify
   void watch(uint64_t cookie, __u8 op, uint32_t timeout = 0) {
     OSDOp& osd_op = add_op(CEPH_OSD_OP_WATCH);
@@ -1313,26 +1052,26 @@ struct ObjectOperation {
 
   void list_watchers(std::list<obj_watch_t> *out,
 		     int *prval) {
-    add_op(CEPH_OSD_OP_LIST_WATCHERS);
+    (void)add_op(CEPH_OSD_OP_LIST_WATCHERS);
     if (prval || out) {
-      set_handler(CB_ObjectOperation_decodewatchers(out, prval, nullptr));
-      out_rval.back() = prval;
+      unsigned p = ops.size() - 1;
+      C_ObjectOperation_decodewatchers *h =
+	new C_ObjectOperation_decodewatchers(out, prval);
+      out_handler[p] = h;
+      out_bl[p] = &h->bl;
+      out_rval[p] = prval;
     }
   }
-  void list_watchers(vector<obj_watch_t>* out,
-		     boost::system::error_code* ec) {
-    add_op(CEPH_OSD_OP_LIST_WATCHERS);
-    set_handler(CB_ObjectOperation_decodewatchers(out, nullptr, ec));
-    out_ec.back() = ec;
-  }
 
-  void list_snaps(librados::snap_set_t *out, int *prval,
-		  boost::system::error_code* ec = nullptr) {
-    add_op(CEPH_OSD_OP_LIST_SNAPS);
-    if (prval || out || ec) {
-      set_handler(CB_ObjectOperation_decodesnaps(out, prval, ec));
-      out_rval.back() = prval;
-      out_ec.back() = ec;
+  void list_snaps(librados::snap_set_t *out, int *prval) {
+    (void)add_op(CEPH_OSD_OP_LIST_SNAPS);
+    if (prval || out) {
+      unsigned p = ops.size() - 1;
+      C_ObjectOperation_decodesnaps *h =
+	new C_ObjectOperation_decodesnaps(out, prval);
+      out_handler[p] = h;
+      out_bl[p] = &h->bl;
+      out_rval[p] = prval;
     }
   }
 
@@ -1458,19 +1197,15 @@ struct ObjectOperation {
     set_last_op_flags(CEPH_OSD_OP_FLAG_FAILOK);
   }
 
-  template<typename V>
-  void dup(V& sops) {
-    ops.clear();
-    std::copy(sops.begin(), sops.end(),
-	      std::back_inserter(ops));
+  void dup(std::vector<OSDOp>& sops) {
+    ops = sops;
     out_bl.resize(sops.size());
     out_handler.resize(sops.size());
     out_rval.resize(sops.size());
-    out_ec.resize(sops.size());
     for (uint32_t i = 0; i < sops.size(); i++) {
       out_bl[i] = &sops[i].outdata;
+      out_handler[i] = NULL;
       out_rval[i] = &sops[i].rval;
-      out_ec[i] = nullptr;
     }
   }
 
@@ -1486,28 +1221,12 @@ struct ObjectOperation {
   }
 };
 
-inline std::ostream& operator <<(std::ostream& m, const ObjectOperation& oo) {
-  auto i = oo.ops.cbegin();
-  m << '[';
-  while (i != oo.ops.cend()) {
-    if (i != oo.ops.cbegin())
-      m << ' ';
-    m << *i;
-    ++i;
-  }
-  m << ']';
-  return m;
-}
-
 
 // ----------------
 
+
 class Objecter : public md_config_obs_t, public Dispatcher {
-  using MOSDOp = _mosdop::MOSDOp<osdc_opvec>;
 public:
-  using OpSignature = void(boost::system::error_code);
-  using OpCompletion = ceph::async::Completion<OpSignature>;
-
   // config observer bits
   const char** get_tracked_conf_keys() const override;
   void handle_conf_change(const ConfigProxy& conf,
@@ -1516,13 +1235,10 @@ public:
 public:
   Messenger *messenger;
   MonClient *monc;
-  boost::asio::io_context& service;
-  // The guaranteed sequenced, one-at-a-time execution and apparently
-  // people sometimes depend on this.
-  boost::asio::io_context::strand finish_strand{service};
-  ZTracer::Endpoint trace_endpoint{"0.0.0.0", 0, "Objecter"};
+  Finisher *finisher;
+  ZTracer::Endpoint trace_endpoint;
 private:
-  std::unique_ptr<OSDMap> osdmap{std::make_unique<OSDMap>()};
+  std::unique_ptr<OSDMap> osdmap;
 public:
   using Dispatcher::cct;
   std::multimap<std::string,std::string> crush_location;
@@ -1557,8 +1273,7 @@ private:
                : epoch(epoch), up(up), up_primary(up_primary),
                  acting(acting), acting_primary(acting_primary) {}
   };
-  ceph::shared_mutex pg_mapping_lock =
-    ceph::make_shared_mutex("Objecter::pg_mapping_lock");
+  std::shared_mutex pg_mapping_lock;
   // pool -> pg mapping
   std::map<int64_t, std::vector<pg_mapping_t>> pg_mappings;
 
@@ -1613,11 +1328,14 @@ private:
   version_t last_seen_osdmap_version = 0;
   version_t last_seen_pgmap_version = 0;
 
-  mutable ceph::shared_mutex rwlock =
-	   ceph::make_shared_mutex("Objecter::rwlock");
+  mutable std::shared_mutex rwlock;
+  using lock_guard = std::lock_guard<decltype(rwlock)>;
+  using unique_lock = std::unique_lock<decltype(rwlock)>;
+  using shared_lock = boost::shared_lock<decltype(rwlock)>;
+  using shunique_lock = ceph::shunique_lock<decltype(rwlock)>;
   ceph::timer<ceph::coarse_mono_clock> timer;
 
-  PerfCounters* logger = nullptr;
+  PerfCounters *logger = nullptr;
 
   uint64_t tick_event = 0;
 
@@ -1707,175 +1425,92 @@ public:
     void dump(ceph::Formatter *f) const;
   };
 
-  std::unique_ptr<ceph::async::Completion<void(boost::system::error_code)>>
-  OpContextVert(Context* c) {
-    if (c)
-      return ceph::async::Completion<void(boost::system::error_code)>::create(
-	service.get_executor(),
-	[c = std::unique_ptr<Context>(c)]
-	(boost::system::error_code e) mutable {
-	  c.release()->complete(e);
-	});
-    else
-      return nullptr;
-  }
-
-  template<typename T>
-  std::unique_ptr<ceph::async::Completion<void(boost::system::error_code, T)>>
-  OpContextVert(Context* c, T* p) {
-
-    if (c || p)
-      return
-	ceph::async::Completion<void(boost::system::error_code, T)>::create(
-	  service.get_executor(),
-	  [c = std::unique_ptr<Context>(c), p]
-	  (boost::system::error_code e, T r) mutable {
-	      if (p)
-		*p = std::move(r);
-	      if (c)
-		c.release()->complete(ceph::from_error_code(e));
-	    });
-    else
-      return nullptr;
-  }
-
-  template<typename T>
-  std::unique_ptr<ceph::async::Completion<void(boost::system::error_code, T)>>
-  OpContextVert(Context* c, T& p) {
-    if (c)
-      return ceph::async::Completion<
-	void(boost::system::error_code, T)>::create(
-	  service.get_executor(),
-	  [c = std::unique_ptr<Context>(c), &p]
-	  (boost::system::error_code e, T r) mutable {
-	    p = std::move(r);
-	    if (c)
-	      c.release()->complete(ceph::from_error_code(e));
-	  });
-    else
-      return nullptr;
-  }
-
   struct Op : public RefCountedObject {
-    OSDSession *session = nullptr;
-    int incarnation = 0;
+    OSDSession *session;
+    int incarnation;
 
     op_target_t target;
 
-    ConnectionRef con = nullptr;  // for rx buffer only
-    uint64_t features = CEPH_FEATURES_SUPPORTED_DEFAULT; // explicitly specified op features
+    ConnectionRef con;  // for rx buffer only
+    uint64_t features;  // explicitly specified op features
 
-    osdc_opvec ops;
+    std::vector<OSDOp> ops;
 
-    snapid_t snapid = CEPH_NOSNAP;
+    snapid_t snapid;
     SnapContext snapc;
     ceph::real_time mtime;
 
-    ceph::buffer::list *outbl = nullptr;
-    boost::container::small_vector<ceph::buffer::list*, osdc_opvec_len> out_bl;
-    boost::container::small_vector<
-      fu2::unique_function<void(boost::system::error_code, int,
-				const ceph::buffer::list& bl) &&>,
-      osdc_opvec_len> out_handler;
-    boost::container::small_vector<int*, osdc_opvec_len> out_rval;
-    boost::container::small_vector<boost::system::error_code*,
-				   osdc_opvec_len> out_ec;
-
-    int priority = 0;
-    using OpSig = void(boost::system::error_code);
-    using OpComp = ceph::async::Completion<OpSig>;
-    // Due to an irregularity of cmpxattr, we actualy need the 'int'
-    // value for onfinish for legacy librados users. As such just
-    // preserve the Context* in this one case. That way we can have
-    // our callers just pass in a unique_ptr<OpComp> and not deal with
-    // our signature in Objecter being different than the exposed
-    // signature in RADOS.
-    std::variant<std::unique_ptr<OpComp>, Context*> onfinish;
-    uint64_t ontimeout = 0;
+    ceph::buffer::list *outbl;
+    std::vector<ceph::buffer::list*> out_bl;
+    std::vector<Context*> out_handler;
+    std::vector<int*> out_rval;
 
-    ceph_tid_t tid = 0;
-    int attempts = 0;
+    int priority;
+    Context *onfinish;
+    uint64_t ontimeout;
+
+    ceph_tid_t tid;
+    int attempts;
 
     version_t *objver;
-    epoch_t *reply_epoch = nullptr;
+    epoch_t *reply_epoch;
 
     ceph::coarse_mono_time stamp;
 
-    epoch_t map_dne_bound = 0;
+    epoch_t map_dne_bound;
 
-    int budget = -1;
+    int budget;
 
     /// true if we should resend this message on failure
-    bool should_resend = true;
+    bool should_resend;
 
     /// true if the throttle budget is get/put on a series of OPs,
     /// instead of per OP basis, when this flag is set, the budget is
     /// acquired before sending the very first OP of the series and
     /// released upon receiving the last OP reply.
-    bool ctx_budgeted = false;
+    bool ctx_budgeted;
 
     int *data_offset;
 
     osd_reqid_t reqid; // explicitly setting reqid
     ZTracer::Trace trace;
 
-    static bool has_completion(decltype(onfinish)& f) {
-      return std::visit([](auto&& arg) { return bool(arg);}, f);
-    }
-    bool has_completion() {
-      return has_completion(onfinish);
-    }
-
-    static void complete(decltype(onfinish)&& f, boost::system::error_code ec,
-			 int r) {
-      std::visit([ec, r](auto&& arg) {
-		   if constexpr (std::is_same_v<std::decay_t<decltype(arg)>,
-				 Context*>) {
-		     arg->complete(r);
-                   } else {
-		     arg->defer(std::move(arg), ec);
-		   }
-		 }, std::move(f));
-    }
-    void complete(boost::system::error_code ec, int r) {
-      complete(std::move(onfinish), ec, r);
-    }
-
-    Op(const object_t& o, const object_locator_t& ol,  osdc_opvec&& _ops,
-       int f, std::unique_ptr<OpComp>&& fin,
-       version_t *ov, int *offset = nullptr,
+    Op(const object_t& o, const object_locator_t& ol, std::vector<OSDOp>& op,
+       int f, Context *fin, version_t *ov, int *offset = NULL,
        ZTracer::Trace *parent_trace = nullptr) :
+      session(NULL), incarnation(0),
       target(o, ol, f),
-      ops(std::move(_ops)),
-      out_bl(ops.size(), nullptr),
-      out_handler(ops.size()),
-      out_rval(ops.size(), nullptr),
-      out_ec(ops.size(), nullptr),
-      onfinish(std::move(fin)),
+      con(NULL),
+      features(CEPH_FEATURES_SUPPORTED_DEFAULT),
+      snapid(CEPH_NOSNAP),
+      outbl(NULL),
+      priority(0),
+      onfinish(fin),
+      ontimeout(0),
+      tid(0),
+      attempts(0),
       objver(ov),
+      reply_epoch(NULL),
+      map_dne_bound(0),
+      budget(-1),
+      should_resend(true),
+      ctx_budgeted(false),
       data_offset(offset) {
-      if (target.base_oloc.key == o)
-	target.base_oloc.key.clear();
-      if (parent_trace && parent_trace->valid()) {
-        trace.init("op", nullptr, parent_trace);
-        trace.event("start");
+      ops.swap(op);
+
+      /* initialize out_* to match op std::vector */
+      out_bl.resize(ops.size());
+      out_rval.resize(ops.size());
+      out_handler.resize(ops.size());
+      for (unsigned i = 0; i < ops.size(); i++) {
+	out_bl[i] = NULL;
+	out_handler[i] = NULL;
+	out_rval[i] = NULL;
       }
-    }
 
-    Op(const object_t& o, const object_locator_t& ol, osdc_opvec&& _ops,
-       int f, Context* fin, version_t *ov, int *offset = nullptr,
-       ZTracer::Trace *parent_trace = nullptr) :
-      target(o, ol, f),
-      ops(std::move(_ops)),
-      out_bl(ops.size(), nullptr),
-      out_handler(ops.size()),
-      out_rval(ops.size(), nullptr),
-      out_ec(ops.size(), nullptr),
-      onfinish(fin),
-      objver(ov),
-      data_offset(offset) {
       if (target.base_oloc.key == o)
 	target.base_oloc.key.clear();
+
       if (parent_trace && parent_trace->valid()) {
         trace.init("op", nullptr, parent_trace);
         trace.event("start");
@@ -1894,22 +1529,30 @@ public:
 
   private:
     ~Op() override {
+      while (!out_handler.empty()) {
+	delete out_handler.back();
+	out_handler.pop_back();
+      }
       trace.event("finish");
     }
   };
 
-  struct CB_Op_Map_Latest {
+  struct C_Op_Map_Latest : public Context {
     Objecter *objecter;
     ceph_tid_t tid;
-    CB_Op_Map_Latest(Objecter *o, ceph_tid_t t) : objecter(o), tid(t) {}
-    void operator()(boost::system::error_code err, version_t latest, version_t);
+    version_t latest;
+    C_Op_Map_Latest(Objecter *o, ceph_tid_t t) : objecter(o), tid(t),
+						 latest(0) {}
+    void finish(int r) override;
   };
 
-  struct CB_Command_Map_Latest {
+  struct C_Command_Map_Latest : public Context {
     Objecter *objecter;
     uint64_t tid;
-    CB_Command_Map_Latest(Objecter *o, ceph_tid_t t) :  objecter(o), tid(t) {}
-    void operator()(boost::system::error_code err, version_t latest, version_t);
+    version_t latest;
+    C_Command_Map_Latest(Objecter *o, ceph_tid_t t) :  objecter(o), tid(t),
+						       latest(0) {}
+    void finish(int r) override;
   };
 
   struct C_Stat : public Context {
@@ -2007,43 +1650,40 @@ public:
 
   struct PoolStatOp {
     ceph_tid_t tid;
-    std::vector<std::string> pools;
-    using OpSig = void(boost::system::error_code,
-		       boost::container::flat_map<std::string, pool_stat_t>,
-		       bool);
-    using OpComp = ceph::async::Completion<OpSig>;
-    std::unique_ptr<OpComp> onfinish;
-    std::uint64_t ontimeout;
+    std::list<std::string> pools;
+
+    std::map<std::string,pool_stat_t> *pool_stats;
+    bool *per_pool;
+    Context *onfinish;
+    uint64_t ontimeout;
+
     ceph::coarse_mono_time last_submit;
   };
 
   struct StatfsOp {
     ceph_tid_t tid;
+    struct ceph_statfs *stats;
     boost::optional<int64_t> data_pool;
-    using OpSig = void(boost::system::error_code,
-		       const struct ceph_statfs);
-    using OpComp = ceph::async::Completion<OpSig>;
-
-    std::unique_ptr<OpComp> onfinish;
+    Context *onfinish;
     uint64_t ontimeout;
 
     ceph::coarse_mono_time last_submit;
   };
 
   struct PoolOp {
-    ceph_tid_t tid = 0;
-    int64_t pool = 0;
+    ceph_tid_t tid;
+    int64_t pool;
     std::string name;
-    using OpSig = void(boost::system::error_code, ceph::buffer::list);
-    using OpComp = ceph::async::Completion<OpSig>;
-    std::unique_ptr<OpComp> onfinish;
-    uint64_t ontimeout = 0;
-    int pool_op = 0;
-    int16_t crush_rule = 0;
-    snapid_t snapid = 0;
-    ceph::coarse_mono_time last_submit;
+    Context *onfinish;
+    uint64_t ontimeout;
+    int pool_op;
+    int16_t crush_rule;
+    snapid_t snapid;
+    ceph::buffer::list *blp;
 
-    PoolOp() {}
+    ceph::coarse_mono_time last_submit;
+    PoolOp() : tid(0), pool(0), onfinish(NULL), ontimeout(0), pool_op(0),
+	       crush_rule(0), snapid(0), blp(NULL) {}
   };
 
   // -- osd commands --
@@ -2052,6 +1692,8 @@ public:
     ceph_tid_t tid = 0;
     std::vector<std::string> cmd;
     ceph::buffer::list inbl;
+    ceph::buffer::list *poutbl = nullptr;
+    std::string *prs = nullptr;
 
     // target_osd == -1 means target_pg is valid
     const int target_osd = -1;
@@ -2063,92 +1705,108 @@ public:
     int map_check_error = 0; // error to return if std::map check fails
     const char *map_check_error_str = nullptr;
 
-    using OpSig = void(boost::system::error_code, std::string,
-		       ceph::buffer::list);
-    using OpComp = ceph::async::Completion<OpSig>;
-    std::unique_ptr<OpComp> onfinish;
-
+    Context *onfinish = nullptr;
     uint64_t ontimeout = 0;
     ceph::coarse_mono_time last_submit;
 
     CommandOp(
       int target_osd,
-      std::vector<string>&& cmd,
-      ceph::buffer::list&& inbl,
-      decltype(onfinish)&& onfinish)
-      : cmd(std::move(cmd)),
-	inbl(std::move(inbl)),
+      const std::vector<std::string> &cmd,
+      ceph::buffer::list inbl,
+      ceph::buffer::list *poutbl,
+      std::string *prs,
+      Context *onfinish)
+      : cmd(cmd),
+	inbl(inbl),
+	poutbl(poutbl),
+	prs(prs),
 	target_osd(target_osd),
-	onfinish(std::move(onfinish)) {}
+	onfinish(onfinish) {}
 
     CommandOp(
       pg_t pgid,
-      std::vector<string>&& cmd,
-      ceph::buffer::list&& inbl,
-      decltype(onfinish)&& onfinish)
-      : cmd(std::move(cmd)),
-	inbl(std::move(inbl)),
+      const std::vector<std::string> &cmd,
+      ceph::buffer::list inbl,
+      ceph::buffer::list *poutbl,
+      std::string *prs,
+      Context *onfinish)
+      : cmd(cmd),
+	inbl(inbl),
+	poutbl(poutbl),
+	prs(prs),
 	target_pg(pgid),
 	target(pgid),
-	onfinish(std::move(onfinish)) {}
+	onfinish(onfinish) {}
+
   };
 
   void submit_command(CommandOp *c, ceph_tid_t *ptid);
-  int _calc_command_target(CommandOp *c,
-			   ceph::shunique_lock<ceph::shared_mutex> &sul);
-  void _assign_command_session(CommandOp *c,
-			       ceph::shunique_lock<ceph::shared_mutex> &sul);
+  int _calc_command_target(CommandOp *c, shunique_lock &sul);
+  void _assign_command_session(CommandOp *c, shunique_lock &sul);
   void _send_command(CommandOp *c);
-  int command_op_cancel(OSDSession *s, ceph_tid_t tid,
-			boost::system::error_code ec);
-  void _finish_command(CommandOp *c, boost::system::error_code ec,
-		       std::string&& rs, ceph::buffer::list&& bl);
+  int command_op_cancel(OSDSession *s, ceph_tid_t tid, int r);
+  void _finish_command(CommandOp *c, int r, std::string rs);
   void handle_command_reply(MCommandReply *m);
 
+
   // -- lingering ops --
 
+  struct WatchContext {
+    // this simply mirrors librados WatchCtx2
+    virtual void handle_notify(uint64_t notify_id,
+			       uint64_t cookie,
+			       uint64_t notifier_id,
+			       ceph::buffer::list& bl) = 0;
+    virtual void handle_error(uint64_t cookie, int err) = 0;
+    virtual ~WatchContext() {}
+  };
+
   struct LingerOp : public RefCountedObject {
-    uint64_t linger_id{0};
-    op_target_t target{object_t(), object_locator_t(), 0};
-    snapid_t snap{CEPH_NOSNAP};
+    uint64_t linger_id;
+
+    op_target_t target;
+
+    snapid_t snap;
     SnapContext snapc;
     ceph::real_time mtime;
 
-    osdc_opvec ops;
+    std::vector<OSDOp> ops;
     ceph::buffer::list inbl;
-    version_t *pobjver{nullptr};
+    ceph::buffer::list *poutbl;
+    version_t *pobjver;
 
-    bool is_watch{false};
+    bool is_watch;
     ceph::coarse_mono_time watch_valid_thru; ///< send time for last acked ping
-    boost::system::error_code last_error;  ///< error from last failed ping|reconnect, if any
-    ceph::shared_mutex watch_lock =
-      ceph::make_shared_mutex("LingerOp::watch_lock");
+    int last_error;  ///< error from last failed ping|reconnect, if any
+    std::shared_mutex watch_lock;
+    using lock_guard = std::unique_lock<decltype(watch_lock)>;
+    using unique_lock = std::unique_lock<decltype(watch_lock)>;
+    using shared_lock = boost::shared_lock<decltype(watch_lock)>;
+    using shunique_lock = ceph::shunique_lock<decltype(watch_lock)>;
 
     // queue of pending async operations, with the timestamp of
     // when they were queued.
     std::list<ceph::coarse_mono_time> watch_pending_async;
 
-    uint32_t register_gen{0};
-    bool registered{false};
-    bool canceled{false};
-    using OpSig = void(boost::system::error_code, ceph::buffer::list);
-    using OpComp = ceph::async::Completion<OpSig>;
-    std::unique_ptr<OpComp> on_reg_commit;
-    std::unique_ptr<OpComp> on_notify_finish;
-    uint64_t notify_id{0};
-
-    fu2::unique_function<void(boost::system::error_code,
-			      uint64_t notify_id,
-			      uint64_t cookie,
-			      uint64_t notifier_id,
-			      ceph::buffer::list&& bl)> handle;
-    OSDSession *session{nullptr};
+    uint32_t register_gen;
+    bool registered;
+    bool canceled;
+    Context *on_reg_commit;
+
+    // we trigger these from an async finisher
+    Context *on_notify_finish;
+    ceph::buffer::list *notify_result_bl;
+    uint64_t notify_id;
+
+    WatchContext *watch_context;
+
+    OSDSession *session;
 
     Objecter *objecter;
-    int ctx_budget{-1};
-    ceph_tid_t register_tid{0};
-    ceph_tid_t ping_tid{0};
-    epoch_t map_dne_bound{0};
+    int ctx_budget;
+    ceph_tid_t register_tid;
+    ceph_tid_t ping_tid;
+    epoch_t map_dne_bound;
 
     void _queued_async() {
       // watch_lock ust be locked unique
@@ -2160,8 +1818,24 @@ public:
       watch_pending_async.pop_front();
     }
 
-    explicit LingerOp(Objecter *o)
-      : objecter(o) {}
+    explicit LingerOp(Objecter *o) : linger_id(0),
+			    target(object_t(), object_locator_t(), 0),
+			    snap(CEPH_NOSNAP), poutbl(NULL), pobjver(NULL),
+			    is_watch(false), last_error(0),
+			    register_gen(0),
+			    registered(false),
+			    canceled(false),
+			    on_reg_commit(NULL),
+			    on_notify_finish(NULL),
+			    notify_result_bl(NULL),
+			    notify_id(0),
+			    watch_context(NULL),
+			    session(NULL),
+			    objecter(o),
+			    ctx_budget(-1),
+			    register_tid(0),
+			    ping_tid(0),
+			    map_dne_bound(0) {}
 
     const LingerOp &operator=(const LingerOp& r) = delete;
     LingerOp(const LingerOp& o) = delete;
@@ -2169,60 +1843,66 @@ public:
     uint64_t get_cookie() {
       return reinterpret_cast<uint64_t>(this);
     }
+
+  private:
+    ~LingerOp() override {
+      delete watch_context;
+    }
   };
 
-  struct CB_Linger_Commit {
+  struct C_Linger_Commit : public Context {
     Objecter *objecter;
     LingerOp *info;
     ceph::buffer::list outbl;  // used for notify only
-    CB_Linger_Commit(Objecter *o, LingerOp *l) : objecter(o), info(l) {
+    C_Linger_Commit(Objecter *o, LingerOp *l) : objecter(o), info(l) {
       info->get();
     }
-    ~CB_Linger_Commit() {
+    ~C_Linger_Commit() override {
       info->put();
     }
-    void operator()(boost::system::error_code ec) {
-      objecter->_linger_commit(info, ec, outbl);
+    void finish(int r) override {
+      objecter->_linger_commit(info, r, outbl);
     }
   };
 
-  struct CB_Linger_Reconnect {
+  struct C_Linger_Reconnect : public Context {
     Objecter *objecter;
     LingerOp *info;
-    CB_Linger_Reconnect(Objecter *o, LingerOp *l)
-      : objecter(o), info(l) {
+    C_Linger_Reconnect(Objecter *o, LingerOp *l) : objecter(o), info(l) {
       info->get();
     }
-    ~CB_Linger_Reconnect() {
+    ~C_Linger_Reconnect() override {
       info->put();
     }
-    void operator()(boost::system::error_code ec) {
-      objecter->_linger_reconnect(info, ec);
+    void finish(int r) override {
+      objecter->_linger_reconnect(info, r);
     }
   };
 
-  struct CB_Linger_Ping {
+  struct C_Linger_Ping : public Context {
     Objecter *objecter;
     LingerOp *info;
     ceph::coarse_mono_time sent;
     uint32_t register_gen;
-    CB_Linger_Ping(Objecter *o, LingerOp *l)
+    C_Linger_Ping(Objecter *o, LingerOp *l)
       : objecter(o), info(l), register_gen(info->register_gen) {
       info->get();
     }
-    CB_Linger_Ping() {
+    ~C_Linger_Ping() override {
       info->put();
     }
-    void operator()(boost::system::error_code ec) {
-      objecter->_linger_ping(info, ec, sent, register_gen);
+    void finish(int r) override {
+      objecter->_linger_ping(info, r, sent, register_gen);
     }
   };
 
-  struct CB_Linger_Map_Latest {
+  struct C_Linger_Map_Latest : public Context {
     Objecter *objecter;
     uint64_t linger_id;
-    CB_Linger_Map_Latest(Objecter *o, uint64_t id) : objecter(o), linger_id(id) {}
-    void operator()(boost::system::error_code err, version_t latest, version_t);
+    version_t latest;
+    C_Linger_Map_Latest(Objecter *o, uint64_t id) :
+      objecter(o), linger_id(id), latest(0) {}
+    void finish(int r) override;
   };
 
   // -- osd sessions --
@@ -2233,8 +1913,11 @@ public:
   };
 
   struct OSDSession : public RefCountedObject {
-    ceph::shared_mutex lock =
-      ceph::make_shared_mutex("OSDSession::lock");
+    std::shared_mutex lock;
+    using lock_guard = std::lock_guard<decltype(lock)>;
+    using unique_lock = std::unique_lock<decltype(lock)>;
+    using shared_lock = boost::shared_lock<decltype(lock)>;
+    using shunique_lock = ceph::shunique_lock<decltype(lock)>;
 
     // pending ops
     std::map<ceph_tid_t,Op*> ops;
@@ -2250,6 +1933,9 @@ public:
     ConnectionRef con;
     int num_locks;
     std::unique_ptr<std::mutex[]> completion_locks;
+    using unique_completion_lock = std::unique_lock<
+      decltype(completion_locks)::element_type>;
+
 
     OSDSession(CephContext *cct, int o) :
       osd(o), incarnation(0), con(NULL),
@@ -2260,7 +1946,7 @@ public:
 
     bool is_homeless() { return (osd == -1); }
 
-    std::unique_lock<std::mutex> get_lock(object_t& oid);
+    unique_completion_lock get_lock(object_t& oid);
   };
   std::map<int,OSDSession*> osd_sessions;
 
@@ -2288,8 +1974,7 @@ public:
   std::map<ceph_tid_t,PoolOp*> pool_ops;
   std::atomic<unsigned> num_homeless_ops{0};
 
-  OSDSession* homeless_session = new OSDSession(cct, -1);
-
+  OSDSession *homeless_session;
 
   // ops waiting for an osdmap with a new pool or confirmation that
   // the pool does not exist (may be expanded to other uses later)
@@ -2297,9 +1982,7 @@ public:
   std::map<ceph_tid_t, Op*> check_latest_map_ops;
   std::map<ceph_tid_t, CommandOp*> check_latest_map_commands;
 
-  std::map<epoch_t,
-	   std::vector<std::pair<std::unique_ptr<OpCompletion>,
-				 boost::system::error_code>>> waiting_for_map;
+  std::map<epoch_t,std::list< std::pair<Context*, int> > > waiting_for_map;
 
   ceph::timespan mon_timeout;
   ceph::timespan osd_timeout;
@@ -2332,7 +2015,7 @@ public:
   int _calc_target(op_target_t *t, Connection *con,
 		   bool any_change = false);
   int _map_session(op_target_t *op, OSDSession **s,
-		   ceph::shunique_lock<ceph::shared_mutex>& lc);
+		   shunique_lock& lc);
 
   void _session_op_assign(OSDSession *s, Op *op);
   void _session_op_remove(OSDSession *s, Op *op);
@@ -2341,35 +2024,28 @@ public:
   void _session_command_op_assign(OSDSession *to, CommandOp *op);
   void _session_command_op_remove(OSDSession *from, CommandOp *op);
 
-  int _assign_op_target_session(Op *op, ceph::shunique_lock<ceph::shared_mutex>& lc,
+  int _assign_op_target_session(Op *op, shunique_lock& lc,
 				bool src_session_locked,
 				bool dst_session_locked);
-  int _recalc_linger_op_target(LingerOp *op,
-			       ceph::shunique_lock<ceph::shared_mutex>& lc);
-
-  void _linger_submit(LingerOp *info,
-		      ceph::shunique_lock<ceph::shared_mutex>& sul);
-  void _send_linger(LingerOp *info,
-		    ceph::shunique_lock<ceph::shared_mutex>& sul);
-  void _linger_commit(LingerOp *info, boost::system::error_code ec,
-		      ceph::buffer::list& outbl);
-  void _linger_reconnect(LingerOp *info, boost::system::error_code ec);
+  int _recalc_linger_op_target(LingerOp *op, shunique_lock& lc);
+
+  void _linger_submit(LingerOp *info, shunique_lock& sul);
+  void _send_linger(LingerOp *info, shunique_lock& sul);
+  void _linger_commit(LingerOp *info, int r, ceph::buffer::list& outbl);
+  void _linger_reconnect(LingerOp *info, int r);
   void _send_linger_ping(LingerOp *info);
-  void _linger_ping(LingerOp *info, boost::system::error_code ec,
-		    ceph::coarse_mono_time sent, uint32_t register_gen);
-  boost::system::error_code _normalize_watch_error(boost::system::error_code ec);
+  void _linger_ping(LingerOp *info, int r, ceph::coarse_mono_time sent,
+		    uint32_t register_gen);
+  int _normalize_watch_error(int r);
 
-  friend class CB_DoWatchError;
+  friend class C_DoWatchError;
 public:
-  template<typename CT>
-  auto linger_callback_flush(CT&& ct) {
-    boost::asio::async_completion<CT, void(void)> init(ct);
-    boost::asio::defer(finish_strand, std::move(init.completion_handler));
-    return init.result.get();
+  void linger_callback_flush(Context *ctx) {
+    finisher->queue(ctx);
   }
 
 private:
-  void _check_op_pool_dne(Op *op, std::unique_lock<ceph::shared_mutex> *sl);
+  void _check_op_pool_dne(Op *op, unique_lock *sl);
   void _send_op_map_check(Op *op);
   void _op_cancel_map_check(Op *op);
   void _check_linger_pool_dne(LingerOp *op, bool *need_unregister);
@@ -2380,11 +2056,9 @@ private:
   void _command_cancel_map_check(CommandOp *op);
 
   void _kick_requests(OSDSession *session, std::map<uint64_t, LingerOp *>& lresend);
-  void _linger_ops_resend(std::map<uint64_t, LingerOp *>& lresend,
-			  std::unique_lock<ceph::shared_mutex>& ul);
+  void _linger_ops_resend(std::map<uint64_t, LingerOp *>& lresend, unique_lock& ul);
 
-  int _get_session(int osd, OSDSession **session,
-		   ceph::shunique_lock<ceph::shared_mutex>& sul);
+  int _get_session(int osd, OSDSession **session, shunique_lock& sul);
   void put_session(OSDSession *s);
   void get_session(OSDSession *s);
   void _reopen_session(OSDSession *session);
@@ -2401,10 +2075,9 @@ private:
    * and returned whenever an op is removed from the std::map
    * If throttle_op needs to throttle it will unlock client_lock.
    */
-  int calc_op_budget(const boost::container::small_vector_base<OSDOp>& ops);
-  void _throttle_op(Op *op, ceph::shunique_lock<ceph::shared_mutex>& sul,
-		    int op_size = 0);
-  int _take_op_budget(Op *op, ceph::shunique_lock<ceph::shared_mutex>& sul) {
+  int calc_op_budget(const std::vector<OSDOp>& ops);
+  void _throttle_op(Op *op, shunique_lock& sul, int op_size = 0);
+  int _take_op_budget(Op *op, shunique_lock& sul) {
     ceph_assert(sul && sul.mutex() == &rwlock);
     int op_budget = calc_op_budget(op->ops);
     if (keep_balanced_budget) {
@@ -2417,21 +2090,18 @@ private:
     return op_budget;
   }
   int take_linger_budget(LingerOp *info);
+  friend class WatchContext; // to invoke put_up_budget_bytes
   void put_op_budget_bytes(int op_budget) {
     ceph_assert(op_budget >= 0);
     op_throttle_bytes.put(op_budget);
     op_throttle_ops.put(1);
   }
   void put_nlist_context_budget(NListContext *list_context);
-  Throttle op_throttle_bytes{cct, "objecter_bytes",
-			     static_cast<int64_t>(
-			       cct->_conf->objecter_inflight_op_bytes)};
-  Throttle op_throttle_ops{cct, "objecter_ops",
-			   static_cast<int64_t>(
-			     cct->_conf->objecter_inflight_ops)};
+  Throttle op_throttle_bytes, op_throttle_ops;
+
  public:
-  Objecter(CephContext *cct, Messenger *m, MonClient *mc,
-	   boost::asio::io_context& service,
+  Objecter(CephContext *cct_, Messenger *m, MonClient *mc,
+	   Finisher *fin,
 	   double mon_timeout,
 	   double osd_timeout);
   ~Objecter() override;
@@ -2483,7 +2153,7 @@ private:
     std::map<ceph_tid_t, Op*>& need_resend,
     std::list<LingerOp*>& need_resend_linger,
     std::map<ceph_tid_t, CommandOp*>& need_resend_command,
-    ceph::shunique_lock<ceph::shared_mutex>& sul);
+    shunique_lock& sul);
 
   int64_t get_object_hash_position(int64_t pool, const std::string& key,
 				   const std::string& ns);
@@ -2517,27 +2187,6 @@ private:
   void handle_osd_map(class MOSDMap *m);
   void wait_for_osd_map();
 
-  template<typename CompletionToken>
-  auto wait_for_osd_map(CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, void()> init(token);
-    unique_lock l(rwlock);
-    if (osdmap->get_epoch()) {
-      l.unlock();
-      boost::asio::dispatch(std::move(init.completion_handler));
-    } else {
-      waiting_for_map[0].emplace_back(
-	OpCompletion::create(
-	  service.get_executor(),
-	  [c = std::move(init.completion_handler)]
-	  (boost::system::error_code) mutable {
-	    std::move(c)();
-	  }), boost::system::error_code{});
-      l.unlock();
-    }
-    return init.result.get();
-  }
-
-
   /**
    * Get std::list of entities blacklisted since this was last called,
    * and reset the std::list.
@@ -2562,17 +2211,15 @@ private:
                              const OSDMap &new_osd_map);
 
   // low-level
-  void _op_submit(Op *op, ceph::shunique_lock<ceph::shared_mutex>& lc,
-		  ceph_tid_t *ptid);
-  void _op_submit_with_budget(Op *op,
-			      ceph::shunique_lock<ceph::shared_mutex>& lc,
+  void _op_submit(Op *op, shunique_lock& lc, ceph_tid_t *ptid);
+  void _op_submit_with_budget(Op *op, shunique_lock& lc,
 			      ceph_tid_t *ptid,
 			      int *ctx_budget = NULL);
   // public interface
 public:
   void op_submit(Op *op, ceph_tid_t *ptid = NULL, int *ctx_budget = NULL);
   bool is_active() {
-    std::shared_lock l(rwlock);
+    shared_lock l(rwlock);
     return !((!inflight_ops) && linger_ops.empty() &&
 	     poolstat_ops.empty() && statfs_ops.empty());
   }
@@ -2598,87 +2245,11 @@ public:
   void set_client_incarnation(int inc) { client_inc = inc; }
 
   bool have_map(epoch_t epoch);
-
-  struct CB_Objecter_GetVersion {
-    Objecter *objecter;
-    std::unique_ptr<OpCompletion> fin;
-
-    CB_Objecter_GetVersion(Objecter *o, std::unique_ptr<OpCompletion> c)
-      : objecter(o), fin(std::move(c)) {}
-    void operator()(boost::system::error_code ec, version_t newest,
-		    version_t oldest) {
-      if (ec == boost::system::errc::resource_unavailable_try_again) {
-	// try again as instructed
-	objecter->wait_for_latest_osdmap(std::move(fin));
-      } else if (ec) {
-	ceph::async::post(std::move(fin), ec);
-      } else {
-	auto l = std::unique_lock(objecter->rwlock);
-	objecter->_get_latest_version(oldest, newest, std::move(fin),
-				      std::move(l));
-      }
-    }
-  };
-
-  template<typename CompletionToken>
-  typename boost::asio::async_result<CompletionToken, OpSignature>::return_type
-  wait_for_map(epoch_t epoch, CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, OpSignature> init(token);
-
-    if (osdmap->get_epoch() >= epoch) {
-      boost::asio::post(service,
-			ceph::async::bind_handler(
-			  std::move(init.completion_handler),
-			  boost::system::error_code()));
-    } else {
-      monc->get_version("osdmap",
-			CB_Objecter_GetVersion(
-			  this,
-			  OpCompletion::create(service.get_executor(),
-					       std::move(init.completion_handler))));
-    }
-    return init.result.get();
-  }
-
-  void _wait_for_new_map(std::unique_ptr<OpCompletion>, epoch_t epoch,
-			 boost::system::error_code = {});
-
-  template<typename CompletionToken>
-  typename boost::asio::async_result<CompletionToken, OpSignature>::return_type
-  wait_for_latest_osdmap(CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, OpSignature> init(token);
-
-    monc->get_version("osdmap",
-		      CB_Objecter_GetVersion(
-			this,
-			OpCompletion::create(service.get_executor(),
-					     std::move(init.completion_handler))));
-    return init.result.get();
-  }
-
-  void wait_for_latest_osdmap(std::unique_ptr<OpCompletion> c) {
-    monc->get_version("osdmap",
-		      CB_Objecter_GetVersion(this, std::move(c)));
-  }
-
-  template<typename CompletionToken>
-  auto get_latest_version(epoch_t oldest, epoch_t newest,
-			  CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken, OpSignature> init(token);
-    {
-      std::unique_lock wl(rwlock);
-      _get_latest_version(oldest, newest,
-			  OpCompletion::create(
-			    service.get_executor(),
-			    std::move(init.completion_handler)),
-			  std::move(wl));
-    }
-    return init.result.get();
-  }
-
-  void _get_latest_version(epoch_t oldest, epoch_t neweset,
-			   std::unique_ptr<OpCompletion> fin,
-			   std::unique_lock<ceph::shared_mutex>&& ul);
+  /// wait for epoch; true if we already have it
+  bool wait_for_map(epoch_t epoch, Context *c, int err=0);
+  void _wait_for_new_map(Context *c, epoch_t epoch, int err=0);
+  void wait_for_latest_osdmap(Context *fin);
+  void get_latest_version(epoch_t oldest, epoch_t neweset, Context *fin);
 
   /** Get the current set of global op flags */
   int get_global_op_flags() const { return global_op_flags; }
@@ -2710,52 +2281,32 @@ public:
   epoch_t op_cancel_writes(int r, int64_t pool=-1);
 
   // commands
-  void osd_command(int osd, std::vector<std::string> cmd,
-		   ceph::buffer::list inbl, ceph_tid_t *ptid,
-		   decltype(CommandOp::onfinish)&& onfinish) {
+  void osd_command(int osd, const std::vector<std::string>& cmd,
+		  const ceph::buffer::list& inbl, ceph_tid_t *ptid,
+		  ceph::buffer::list *poutbl, std::string *prs, Context *onfinish) {
     ceph_assert(osd >= 0);
-    auto c = new CommandOp(
+    CommandOp *c = new CommandOp(
       osd,
-      std::move(cmd),
-      std::move(inbl),
-      std::move(onfinish));
+      cmd,
+      inbl,
+      poutbl,
+      prs,
+      onfinish);
     submit_command(c, ptid);
   }
-  template<typename CompletionToken>
-  auto osd_command(int osd, std::vector<std::string> cmd,
-		   ceph::buffer::list inbl, ceph_tid_t *ptid,
-		   CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken,
-				  CommandOp::OpSig> init(token);
-    osd_command(osd, std::move(cmd), std::move(inbl), ptid,
-		CommandOp::OpComp::create(service.get_executor(),
-					  std::move(init.completion_handler)));
-    return init.result.get();
-  }
-
-  void pg_command(pg_t pgid, std::vector<std::string> cmd,
-		  ceph::buffer::list inbl, ceph_tid_t *ptid,
-		  decltype(CommandOp::onfinish)&& onfinish) {
-    auto *c = new CommandOp(
+  void pg_command(pg_t pgid, const std::vector<std::string>& cmd,
+		 const ceph::buffer::list& inbl, ceph_tid_t *ptid,
+		 ceph::buffer::list *poutbl, std::string *prs, Context *onfinish) {
+    CommandOp *c = new CommandOp(
       pgid,
-      std::move(cmd),
-      std::move(inbl),
-      std::move(onfinish));
+      cmd,
+      inbl,
+      poutbl,
+      prs,
+      onfinish);
     submit_command(c, ptid);
   }
 
-  template<typename CompletionToken>
-  auto pg_command(pg_t pgid, std::vector<std::string> cmd,
-		  ceph::buffer::list inbl, ceph_tid_t *ptid,
-		  CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken,
-				  CommandOp::OpSig> init(token);
-    pg_command(pgid, std::move(cmd), std::move(inbl), ptid,
-	       CommandOp::OpComp::create(service.get_executor(),
-					 std::move(init.completion_handler)));
-    return init.result.get();
-  }
-
   // mid-level helpers
   Op *prepare_mutate_op(
     const object_t& oid, const object_locator_t& oloc,
@@ -2764,18 +2315,15 @@ public:
     Context *oncommit, version_t *objver = NULL,
     osd_reqid_t reqid = osd_reqid_t(),
     ZTracer::Trace *parent_trace = nullptr) {
-    Op *o = new Op(oid, oloc, std::move(op.ops), flags | global_op_flags |
-		   CEPH_OSD_FLAG_WRITE, oncommit, objver,
-		   nullptr, parent_trace);
+    Op *o = new Op(oid, oloc, op.ops, flags | global_op_flags |
+		   CEPH_OSD_FLAG_WRITE, oncommit, objver, nullptr, parent_trace);
     o->priority = op.priority;
     o->mtime = mtime;
     o->snapc = snapc;
     o->out_rval.swap(op.out_rval);
     o->out_bl.swap(op.out_bl);
     o->out_handler.swap(op.out_handler);
-    o->out_ec.swap(op.out_ec);
     o->reqid = reqid;
-    op.clear();
     return o;
   }
   ceph_tid_t mutate(
@@ -2790,27 +2338,6 @@ public:
     op_submit(o, &tid);
     return tid;
   }
-
-  void mutate(const object_t& oid, const object_locator_t& oloc,
-	      ObjectOperation&& op, const SnapContext& snapc,
-	      ceph::real_time mtime, int flags,
-	      std::unique_ptr<Op::OpComp>&& oncommit,
-	      version_t *objver = NULL, osd_reqid_t reqid = osd_reqid_t()) {
-    Op *o = new Op(oid, oloc, std::move(op.ops), flags | global_op_flags |
-		   CEPH_OSD_FLAG_WRITE, std::move(oncommit), objver,
-		   nullptr);
-    o->priority = op.priority;
-    o->mtime = mtime;
-    o->snapc = snapc;
-    o->out_bl.swap(op.out_bl);
-    o->out_handler.swap(op.out_handler);
-    o->out_rval.swap(op.out_rval);
-    o->out_ec.swap(op.out_ec);
-    o->reqid = reqid;
-    op.clear();
-    op_submit(o);
-  }
-
   Op *prepare_read_op(
     const object_t& oid, const object_locator_t& oloc,
     ObjectOperation& op,
@@ -2819,19 +2346,16 @@ public:
     int *data_offset = NULL,
     uint64_t features = 0,
     ZTracer::Trace *parent_trace = nullptr) {
-    Op *o = new Op(oid, oloc, std::move(op.ops), flags | global_op_flags |
-		   CEPH_OSD_FLAG_READ, onack, objver,
-		   data_offset, parent_trace);
+    Op *o = new Op(oid, oloc, op.ops, flags | global_op_flags |
+		   CEPH_OSD_FLAG_READ, onack, objver, data_offset, parent_trace);
     o->priority = op.priority;
     o->snapid = snapid;
     o->outbl = pbl;
-    if (!o->outbl && op.size() == 1 && op.out_bl[0] && op.out_bl[0]->length())
+    if (!o->outbl && op.size() == 1 && op.out_bl[0]->length())
 	o->outbl = op.out_bl[0];
     o->out_bl.swap(op.out_bl);
     o->out_handler.swap(op.out_handler);
     o->out_rval.swap(op.out_rval);
-    o->out_ec.swap(op.out_ec);
-    op.clear();
     return o;
   }
   ceph_tid_t read(
@@ -2849,40 +2373,13 @@ public:
     op_submit(o, &tid);
     return tid;
   }
-
-  void read(const object_t& oid, const object_locator_t& oloc,
-	    ObjectOperation&& op, snapid_t snapid, ceph::buffer::list *pbl,
-	    int flags, std::unique_ptr<Op::OpComp>&& onack,
-	    version_t *objver = nullptr, int *data_offset = nullptr,
-	    uint64_t features = 0) {
-    Op *o = new Op(oid, oloc, std::move(op.ops), flags | global_op_flags |
-		   CEPH_OSD_FLAG_READ, std::move(onack), objver,
-		   data_offset);
-    o->priority = op.priority;
-    o->snapid = snapid;
-    o->outbl = pbl;
-    // XXX
-    if (!o->outbl && op.size() == 1 && op.out_bl[0] && op.out_bl[0]->length()) {
-      o->outbl = op.out_bl[0];
-    }
-    o->out_bl.swap(op.out_bl);
-    o->out_handler.swap(op.out_handler);
-    o->out_rval.swap(op.out_rval);
-    o->out_ec.swap(op.out_ec);
-    if (features)
-      o->features = features;
-    op.clear();
-    op_submit(o);
-  }
-
-
   Op *prepare_pg_read_op(
     uint32_t hash, object_locator_t oloc,
     ObjectOperation& op, ceph::buffer::list *pbl, int flags,
     Context *onack, epoch_t *reply_epoch,
     int *ctx_budget) {
     Op *o = new Op(object_t(), oloc,
-		   std::move(op.ops),
+		   op.ops,
 		   flags | global_op_flags | CEPH_OSD_FLAG_READ |
 		   CEPH_OSD_FLAG_IGNORE_OVERLAY,
 		   onack, NULL);
@@ -2894,13 +2391,11 @@ public:
     o->out_bl.swap(op.out_bl);
     o->out_handler.swap(op.out_handler);
     o->out_rval.swap(op.out_rval);
-    o->out_ec.swap(op.out_ec);
     o->reply_epoch = reply_epoch;
     if (ctx_budget) {
       // budget is tracked by listing context
       o->ctx_budgeted = true;
     }
-    op.clear();
     return o;
   }
   ceph_tid_t pg_read(
@@ -2915,35 +2410,6 @@ public:
     return tid;
   }
 
-  ceph_tid_t pg_read(
-    uint32_t hash, object_locator_t oloc,
-    ObjectOperation& op, ceph::buffer::list *pbl, int flags,
-    std::unique_ptr<Op::OpComp>&& onack, epoch_t *reply_epoch, int *ctx_budget) {
-    ceph_tid_t tid;
-    Op *o = new Op(object_t(), oloc,
-		   std::move(op.ops),
-		   flags | global_op_flags | CEPH_OSD_FLAG_READ |
-		   CEPH_OSD_FLAG_IGNORE_OVERLAY,
-		   std::move(onack), nullptr);
-    o->target.precalc_pgid = true;
-    o->target.base_pgid = pg_t(hash, oloc.pool);
-    o->priority = op.priority;
-    o->snapid = CEPH_NOSNAP;
-    o->outbl = pbl;
-    o->out_bl.swap(op.out_bl);
-    o->out_handler.swap(op.out_handler);
-    o->out_rval.swap(op.out_rval);
-    o->out_ec.swap(op.out_ec);
-    o->reply_epoch = reply_epoch;
-    if (ctx_budget) {
-      // budget is tracked by listing context
-      o->ctx_budgeted = true;
-    }
-    op_submit(o, &tid, ctx_budget);
-    op.clear();
-    return tid;
-  }
-
   // caller owns a ref
   LingerOp *linger_register(const object_t& oid, const object_locator_t& oloc,
 			    int flags);
@@ -2951,34 +2417,15 @@ public:
 			  ObjectOperation& op,
 			  const SnapContext& snapc, ceph::real_time mtime,
 			  ceph::buffer::list& inbl,
-			  decltype(info->on_reg_commit)&& oncommit,
+			  Context *onfinish,
 			  version_t *objver);
-  ceph_tid_t linger_watch(LingerOp *info,
-			  ObjectOperation& op,
-			  const SnapContext& snapc, ceph::real_time mtime,
-			  ceph::buffer::list& inbl,
-			  Context* onfinish,
-			  version_t *objver) {
-    return linger_watch(info, op, snapc, mtime, inbl,
-			OpContextVert<ceph::buffer::list>(onfinish, nullptr), objver);
-  }
-  ceph_tid_t linger_notify(LingerOp *info,
-			   ObjectOperation& op,
-			   snapid_t snap, ceph::buffer::list& inbl,
-			   decltype(LingerOp::on_reg_commit)&& onfinish,
-			   version_t *objver);
   ceph_tid_t linger_notify(LingerOp *info,
 			   ObjectOperation& op,
 			   snapid_t snap, ceph::buffer::list& inbl,
 			   ceph::buffer::list *poutbl,
-			   Context* onack,
-			   version_t *objver) {
-    return linger_notify(info, op, snap, inbl,
-			 OpContextVert(onack, poutbl),
-			 objver);
-  }
-  tl::expected<ceph::timespan,
-	       boost::system::error_code> linger_check(LingerOp *info);
+			   Context *onack,
+			   version_t *objver);
+  int linger_check(LingerOp *info);
   void linger_cancel(LingerOp *info);  // releases a reference
   void _linger_cancel(LingerOp *info);
 
@@ -2994,8 +2441,7 @@ public:
    * @param extra_ops pointer to [array of] initial op[s]
    * @return index of final op (for caller to fill in)
    */
-  int init_ops(boost::container::small_vector_base<OSDOp>& ops, int ops_count,
-	       ObjectOperation *extra_ops) {
+  int init_ops(std::vector<OSDOp>& ops, int ops_count, ObjectOperation *extra_ops) {
     int i;
     int extra = 0;
 
@@ -3018,11 +2464,11 @@ public:
     snapid_t snap, uint64_t *psize, ceph::real_time *pmtime,
     int flags, Context *onfinish, version_t *objver = NULL,
     ObjectOperation *extra_ops = NULL) {
-    osdc_opvec ops;
+    std::vector<OSDOp> ops;
     int i = init_ops(ops, 1, extra_ops);
     ops[i].op.op = CEPH_OSD_OP_STAT;
     C_Stat *fin = new C_Stat(psize, pmtime, onfinish);
-    Op *o = new Op(oid, oloc, std::move(ops), flags | global_op_flags |
+    Op *o = new Op(oid, oloc, ops, flags | global_op_flags |
 		   CEPH_OSD_FLAG_READ, fin, objver);
     o->snapid = snap;
     o->outbl = &fin->bl;
@@ -3046,7 +2492,7 @@ public:
     int flags, Context *onfinish, version_t *objver = NULL,
     ObjectOperation *extra_ops = NULL, int op_flags = 0,
     ZTracer::Trace *parent_trace = nullptr) {
-    osdc_opvec ops;
+    std::vector<OSDOp> ops;
     int i = init_ops(ops, 1, extra_ops);
     ops[i].op.op = CEPH_OSD_OP_READ;
     ops[i].op.extent.offset = off;
@@ -3054,9 +2500,8 @@ public:
     ops[i].op.extent.truncate_size = 0;
     ops[i].op.extent.truncate_seq = 0;
     ops[i].op.flags = op_flags;
-    Op *o = new Op(oid, oloc, std::move(ops), flags | global_op_flags |
-		   CEPH_OSD_FLAG_READ, onfinish, objver,
-		   nullptr, parent_trace);
+    Op *o = new Op(oid, oloc, ops, flags | global_op_flags |
+		   CEPH_OSD_FLAG_READ, onfinish, objver, nullptr, parent_trace);
     o->snapid = snap;
     o->outbl = pbl;
     return o;
@@ -3078,7 +2523,7 @@ public:
     uint64_t off, ceph::buffer::list &cmp_bl,
     snapid_t snap, int flags, Context *onfinish, version_t *objver = NULL,
     ObjectOperation *extra_ops = NULL, int op_flags = 0) {
-    osdc_opvec ops;
+    std::vector<OSDOp> ops;
     int i = init_ops(ops, 1, extra_ops);
     ops[i].op.op = CEPH_OSD_OP_CMPEXT;
     ops[i].op.extent.offset = off;
@@ -3087,7 +2532,7 @@ public:
     ops[i].op.extent.truncate_seq = 0;
     ops[i].indata = cmp_bl;
     ops[i].op.flags = op_flags;
-    Op *o = new Op(oid, oloc, std::move(ops), flags | global_op_flags |
+    Op *o = new Op(oid, oloc, ops, flags | global_op_flags |
 		   CEPH_OSD_FLAG_READ, onfinish, objver);
     o->snapid = snap;
     return o;
@@ -3111,7 +2556,7 @@ public:
 			__u32 trunc_seq, Context *onfinish,
 			version_t *objver = NULL,
 			ObjectOperation *extra_ops = NULL, int op_flags = 0) {
-    osdc_opvec ops;
+    std::vector<OSDOp> ops;
     int i = init_ops(ops, 1, extra_ops);
     ops[i].op.op = CEPH_OSD_OP_READ;
     ops[i].op.extent.offset = off;
@@ -3119,7 +2564,7 @@ public:
     ops[i].op.extent.truncate_size = trunc_size;
     ops[i].op.extent.truncate_seq = trunc_seq;
     ops[i].op.flags = op_flags;
-    Op *o = new Op(oid, oloc, std::move(ops), flags | global_op_flags |
+    Op *o = new Op(oid, oloc, ops, flags | global_op_flags |
 		   CEPH_OSD_FLAG_READ, onfinish, objver);
     o->snapid = snap;
     o->outbl = pbl;
@@ -3131,14 +2576,14 @@ public:
 		    uint64_t off, uint64_t len, snapid_t snap, ceph::buffer::list *pbl,
 		    int flags, Context *onfinish, version_t *objver = NULL,
 		    ObjectOperation *extra_ops = NULL) {
-    osdc_opvec ops;
+    std::vector<OSDOp> ops;
     int i = init_ops(ops, 1, extra_ops);
     ops[i].op.op = CEPH_OSD_OP_MAPEXT;
     ops[i].op.extent.offset = off;
     ops[i].op.extent.length = len;
     ops[i].op.extent.truncate_size = 0;
     ops[i].op.extent.truncate_seq = 0;
-    Op *o = new Op(oid, oloc, std::move(ops), flags | global_op_flags |
+    Op *o = new Op(oid, oloc, ops, flags | global_op_flags |
 		   CEPH_OSD_FLAG_READ, onfinish, objver);
     o->snapid = snap;
     o->outbl = pbl;
@@ -3150,14 +2595,14 @@ public:
 	     const char *name, snapid_t snap, ceph::buffer::list *pbl, int flags,
 	     Context *onfinish,
 	     version_t *objver = NULL, ObjectOperation *extra_ops = NULL) {
-    osdc_opvec ops;
+    std::vector<OSDOp> ops;
     int i = init_ops(ops, 1, extra_ops);
     ops[i].op.op = CEPH_OSD_OP_GETXATTR;
     ops[i].op.xattr.name_len = (name ? strlen(name) : 0);
     ops[i].op.xattr.value_len = 0;
     if (name)
       ops[i].indata.append(name, ops[i].op.xattr.name_len);
-    Op *o = new Op(oid, oloc, std::move(ops), flags | global_op_flags |
+    Op *o = new Op(oid, oloc, ops, flags | global_op_flags |
 		   CEPH_OSD_FLAG_READ, onfinish, objver);
     o->snapid = snap;
     o->outbl = pbl;
@@ -3170,11 +2615,11 @@ public:
 		       snapid_t snap, std::map<std::string,ceph::buffer::list>& attrset,
 		       int flags, Context *onfinish, version_t *objver = NULL,
 		       ObjectOperation *extra_ops = NULL) {
-    osdc_opvec ops;
+    std::vector<OSDOp> ops;
     int i = init_ops(ops, 1, extra_ops);
     ops[i].op.op = CEPH_OSD_OP_GETXATTRS;
     C_GetAttrs *fin = new C_GetAttrs(attrset, onfinish);
-    Op *o = new Op(oid, oloc, std::move(ops), flags | global_op_flags |
+    Op *o = new Op(oid, oloc, ops, flags | global_op_flags |
 		   CEPH_OSD_FLAG_READ, fin, objver);
     o->snapid = snap;
     o->outbl = &fin->bl;
@@ -3194,12 +2639,11 @@ public:
 
   // writes
   ceph_tid_t _modify(const object_t& oid, const object_locator_t& oloc,
-		     osdc_opvec& ops,
-		     ceph::real_time mtime,
+		     std::vector<OSDOp>& ops, ceph::real_time mtime,
 		     const SnapContext& snapc, int flags,
 		     Context *oncommit,
 		     version_t *objver = NULL) {
-    Op *o = new Op(oid, oloc, std::move(ops), flags | global_op_flags |
+    Op *o = new Op(oid, oloc, ops, flags | global_op_flags |
 		   CEPH_OSD_FLAG_WRITE, oncommit, objver);
     o->mtime = mtime;
     o->snapc = snapc;
@@ -3214,7 +2658,7 @@ public:
     Context *oncommit, version_t *objver = NULL,
     ObjectOperation *extra_ops = NULL, int op_flags = 0,
     ZTracer::Trace *parent_trace = nullptr) {
-    osdc_opvec ops;
+    std::vector<OSDOp> ops;
     int i = init_ops(ops, 1, extra_ops);
     ops[i].op.op = CEPH_OSD_OP_WRITE;
     ops[i].op.extent.offset = off;
@@ -3223,8 +2667,8 @@ public:
     ops[i].op.extent.truncate_seq = 0;
     ops[i].indata = bl;
     ops[i].op.flags = op_flags;
-    Op *o = new Op(oid, oloc, std::move(ops), flags | global_op_flags |
-		   CEPH_OSD_FLAG_WRITE, std::move(oncommit), objver,
+    Op *o = new Op(oid, oloc, ops, flags | global_op_flags |
+		   CEPH_OSD_FLAG_WRITE, oncommit, objver,
                    nullptr, parent_trace);
     o->mtime = mtime;
     o->snapc = snapc;
@@ -3249,7 +2693,7 @@ public:
     Context *oncommit,
     version_t *objver = NULL,
     ObjectOperation *extra_ops = NULL) {
-    osdc_opvec ops;
+    std::vector<OSDOp> ops;
     int i = init_ops(ops, 1, extra_ops);
     ops[i].op.op = CEPH_OSD_OP_APPEND;
     ops[i].op.extent.offset = 0;
@@ -3257,7 +2701,7 @@ public:
     ops[i].op.extent.truncate_size = 0;
     ops[i].op.extent.truncate_seq = 0;
     ops[i].indata = bl;
-    Op *o = new Op(oid, oloc, std::move(ops), flags | global_op_flags |
+    Op *o = new Op(oid, oloc, ops, flags | global_op_flags |
 		   CEPH_OSD_FLAG_WRITE, oncommit, objver);
     o->mtime = mtime;
     o->snapc = snapc;
@@ -3283,7 +2727,7 @@ public:
 			 Context *oncommit,
 			 version_t *objver = NULL,
 			 ObjectOperation *extra_ops = NULL, int op_flags = 0) {
-    osdc_opvec ops;
+    std::vector<OSDOp> ops;
     int i = init_ops(ops, 1, extra_ops);
     ops[i].op.op = CEPH_OSD_OP_WRITE;
     ops[i].op.extent.offset = off;
@@ -3292,7 +2736,7 @@ public:
     ops[i].op.extent.truncate_seq = trunc_seq;
     ops[i].indata = bl;
     ops[i].op.flags = op_flags;
-    Op *o = new Op(oid, oloc, std::move(ops), flags | global_op_flags |
+    Op *o = new Op(oid, oloc, ops, flags | global_op_flags |
 		   CEPH_OSD_FLAG_WRITE, oncommit, objver);
     o->mtime = mtime;
     o->snapc = snapc;
@@ -3306,14 +2750,14 @@ public:
     ceph::real_time mtime, int flags,
     Context *oncommit, version_t *objver = NULL,
     ObjectOperation *extra_ops = NULL, int op_flags = 0) {
-    osdc_opvec ops;
+    std::vector<OSDOp> ops;
     int i = init_ops(ops, 1, extra_ops);
     ops[i].op.op = CEPH_OSD_OP_WRITEFULL;
     ops[i].op.extent.offset = 0;
     ops[i].op.extent.length = bl.length();
     ops[i].indata = bl;
     ops[i].op.flags = op_flags;
-    Op *o = new Op(oid, oloc, std::move(ops), flags | global_op_flags |
+    Op *o = new Op(oid, oloc, ops, flags | global_op_flags |
 		   CEPH_OSD_FLAG_WRITE, oncommit, objver);
     o->mtime = mtime;
     o->snapc = snapc;
@@ -3339,7 +2783,7 @@ public:
     Context *oncommit, version_t *objver = NULL,
     ObjectOperation *extra_ops = NULL, int op_flags = 0) {
 
-    osdc_opvec ops;
+    std::vector<OSDOp> ops;
     int i = init_ops(ops, 1, extra_ops);
     ops[i].op.op = CEPH_OSD_OP_WRITESAME;
     ops[i].op.writesame.offset = off;
@@ -3347,7 +2791,7 @@ public:
     ops[i].op.writesame.data_length = bl.length();
     ops[i].indata = bl;
     ops[i].op.flags = op_flags;
-    Op *o = new Op(oid, oloc, std::move(ops), flags | global_op_flags |
+    Op *o = new Op(oid, oloc, ops, flags | global_op_flags |
 		   CEPH_OSD_FLAG_WRITE, oncommit, objver);
     o->mtime = mtime;
     o->snapc = snapc;
@@ -3374,13 +2818,13 @@ public:
 		   uint64_t trunc_size, __u32 trunc_seq,
 		   Context *oncommit, version_t *objver = NULL,
 		   ObjectOperation *extra_ops = NULL) {
-    osdc_opvec ops;
+    std::vector<OSDOp> ops;
     int i = init_ops(ops, 1, extra_ops);
     ops[i].op.op = CEPH_OSD_OP_TRUNCATE;
     ops[i].op.extent.offset = trunc_size;
     ops[i].op.extent.truncate_size = trunc_size;
     ops[i].op.extent.truncate_seq = trunc_seq;
-    Op *o = new Op(oid, oloc, std::move(ops), flags | global_op_flags |
+    Op *o = new Op(oid, oloc, ops, flags | global_op_flags |
 		   CEPH_OSD_FLAG_WRITE, oncommit, objver);
     o->mtime = mtime;
     o->snapc = snapc;
@@ -3392,12 +2836,12 @@ public:
 		  uint64_t off, uint64_t len, const SnapContext& snapc,
 		  ceph::real_time mtime, int flags, Context *oncommit,
 	     version_t *objver = NULL, ObjectOperation *extra_ops = NULL) {
-    osdc_opvec ops;
+    std::vector<OSDOp> ops;
     int i = init_ops(ops, 1, extra_ops);
     ops[i].op.op = CEPH_OSD_OP_ZERO;
     ops[i].op.extent.offset = off;
     ops[i].op.extent.length = len;
-    Op *o = new Op(oid, oloc, std::move(ops), flags | global_op_flags |
+    Op *o = new Op(oid, oloc, ops, flags | global_op_flags |
 		   CEPH_OSD_FLAG_WRITE, oncommit, objver);
     o->mtime = mtime;
     o->snapc = snapc;
@@ -3410,11 +2854,11 @@ public:
 			     ceph::real_time mtime, Context *oncommit,
 			     version_t *objver = NULL,
 			     ObjectOperation *extra_ops = NULL) {
-    osdc_opvec ops;
+    std::vector<OSDOp> ops;
     int i = init_ops(ops, 1, extra_ops);
     ops[i].op.op = CEPH_OSD_OP_ROLLBACK;
     ops[i].op.snap.snapid = snapid;
-    Op *o = new Op(oid, oloc, std::move(ops), CEPH_OSD_FLAG_WRITE, oncommit, objver);
+    Op *o = new Op(oid, oloc, ops, CEPH_OSD_FLAG_WRITE, oncommit, objver);
     o->mtime = mtime;
     o->snapc = snapc;
     ceph_tid_t tid;
@@ -3426,11 +2870,11 @@ public:
 		    int create_flags, Context *oncommit,
 		    version_t *objver = NULL,
 		    ObjectOperation *extra_ops = NULL) {
-    osdc_opvec ops;
+    std::vector<OSDOp> ops;
     int i = init_ops(ops, 1, extra_ops);
     ops[i].op.op = CEPH_OSD_OP_CREATE;
     ops[i].op.flags = create_flags;
-    Op *o = new Op(oid, oloc, std::move(ops), global_flags | global_op_flags |
+    Op *o = new Op(oid, oloc, ops, global_flags | global_op_flags |
 		   CEPH_OSD_FLAG_WRITE, oncommit, objver);
     o->mtime = mtime;
     o->snapc = snapc;
@@ -3443,10 +2887,10 @@ public:
     const SnapContext& snapc, ceph::real_time mtime, int flags,
     Context *oncommit,
     version_t *objver = NULL, ObjectOperation *extra_ops = NULL) {
-    osdc_opvec ops;
+    std::vector<OSDOp> ops;
     int i = init_ops(ops, 1, extra_ops);
     ops[i].op.op = CEPH_OSD_OP_DELETE;
-    Op *o = new Op(oid, oloc, std::move(ops), flags | global_op_flags |
+    Op *o = new Op(oid, oloc, ops, flags | global_op_flags |
 		   CEPH_OSD_FLAG_WRITE, oncommit, objver);
     o->mtime = mtime;
     o->snapc = snapc;
@@ -3469,7 +2913,7 @@ public:
 	      ceph::real_time mtime, int flags,
 	      Context *oncommit,
 	      version_t *objver = NULL, ObjectOperation *extra_ops = NULL) {
-    osdc_opvec ops;
+    std::vector<OSDOp> ops;
     int i = init_ops(ops, 1, extra_ops);
     ops[i].op.op = CEPH_OSD_OP_SETXATTR;
     ops[i].op.xattr.name_len = (name ? strlen(name) : 0);
@@ -3477,9 +2921,8 @@ public:
     if (name)
       ops[i].indata.append(name, ops[i].op.xattr.name_len);
     ops[i].indata.append(bl);
-    Op *o = new Op(oid, oloc, std::move(ops), flags | global_op_flags |
-		   CEPH_OSD_FLAG_WRITE, oncommit,
-		   objver);
+    Op *o = new Op(oid, oloc, ops, flags | global_op_flags |
+		   CEPH_OSD_FLAG_WRITE, oncommit, objver);
     o->mtime = mtime;
     o->snapc = snapc;
     ceph_tid_t tid;
@@ -3491,14 +2934,14 @@ public:
 	      ceph::real_time mtime, int flags,
 	      Context *oncommit,
 	      version_t *objver = NULL, ObjectOperation *extra_ops = NULL) {
-    osdc_opvec ops;
+    std::vector<OSDOp> ops;
     int i = init_ops(ops, 1, extra_ops);
     ops[i].op.op = CEPH_OSD_OP_RMXATTR;
     ops[i].op.xattr.name_len = (name ? strlen(name) : 0);
     ops[i].op.xattr.value_len = 0;
     if (name)
       ops[i].indata.append(name, ops[i].op.xattr.name_len);
-    Op *o = new Op(oid, oloc, std::move(ops), flags | global_op_flags |
+    Op *o = new Op(oid, oloc, ops, flags | global_op_flags |
 		   CEPH_OSD_FLAG_WRITE, oncommit, objver);
     o->mtime = mtime;
     o->snapc = snapc;
@@ -3514,25 +2957,29 @@ public:
 
   hobject_t enumerate_objects_begin();
   hobject_t enumerate_objects_end();
-
-  friend EnumerationContext;
-  friend struct CB_EnumerateReply;
+  //hobject_t enumerate_objects_begin(int n, int m);
   void enumerate_objects(
     int64_t pool_id,
-    std::string_view ns,
-    hobject_t start,
-    hobject_t end,
+    const std::string &ns,
+    const hobject_t &start,
+    const hobject_t &end,
     const uint32_t max,
-    const ceph::buffer::list& filter_bl,
-    fu2::unique_function<void(boost::system::error_code,
-			      std::vector<librados::ListObjectImpl>,
-			      hobject_t) &&> on_finish);
-  void _issue_enumerate(hobject_t start,
-			std::unique_ptr<EnumerationContext>);
+    const ceph::buffer::list &filter_bl,
+    std::list<librados::ListObjectImpl> *result, 
+    hobject_t *next,
+    Context *on_finish);
+
   void _enumerate_reply(
-    ceph::buffer::list&& bl,
-    boost::system::error_code ec,
-    std::unique_ptr<EnumerationContext>&& ectx);
+      ceph::buffer::list &bl,
+      int r,
+      const hobject_t &end,
+      const int64_t pool_id,
+      int budget,
+      epoch_t reply_epoch,
+      std::list<librados::ListObjectImpl> *result, 
+      hobject_t *next,
+      Context *on_finish);
+  friend class C_EnumerateReply;
 
   // -------------------------
   // pool ops
@@ -3540,66 +2987,18 @@ private:
   void pool_op_submit(PoolOp *op);
   void _pool_op_submit(PoolOp *op);
   void _finish_pool_op(PoolOp *op, int r);
-  void _do_delete_pool(int64_t pool,
-		       decltype(PoolOp::onfinish)&& onfinish);
-
+  void _do_delete_pool(int64_t pool, Context *onfinish);
 public:
-  void create_pool_snap(int64_t pool, std::string_view snapName,
-			decltype(PoolOp::onfinish)&& onfinish);
-  void create_pool_snap(int64_t pool, std::string_view snapName,
-			Context* c) {
-    create_pool_snap(pool, snapName,
-		     OpContextVert<ceph::buffer::list>(c, nullptr));
-  }
-  void allocate_selfmanaged_snap(int64_t pool,
-				 std::unique_ptr<ceph::async::Completion<
-				 void(boost::system::error_code,
-				      snapid_t)>> onfinish);
-  void allocate_selfmanaged_snap(int64_t pool, snapid_t* psnapid,
-				 Context* c) {
-    allocate_selfmanaged_snap(pool,
-			      OpContextVert(c, psnapid));
-  }
-  void delete_pool_snap(int64_t pool, std::string_view snapName,
-			decltype(PoolOp::onfinish)&& onfinish);
-  void delete_pool_snap(int64_t pool, std::string_view snapName,
-			Context* c) {
-    delete_pool_snap(pool, snapName,
-		     OpContextVert<ceph::buffer::list>(c, nullptr));
-  }
-
-  void delete_selfmanaged_snap(int64_t pool, snapid_t snap,
-			       decltype(PoolOp::onfinish)&& onfinish);
-  void delete_selfmanaged_snap(int64_t pool, snapid_t snap,
-			       Context* c) {
-    delete_selfmanaged_snap(pool, snap,
-			    OpContextVert<ceph::buffer::list>(c, nullptr));
-  }
-
-
-  void create_pool(std::string_view name,
-		   decltype(PoolOp::onfinish)&& onfinish,
-		   int crush_rule=-1);
-  void create_pool(std::string_view name, Context *onfinish,
-		  int crush_rule=-1) {
-    create_pool(name,
-		OpContextVert<ceph::buffer::list>(onfinish, nullptr),
-		crush_rule);
-  }
-  void delete_pool(int64_t pool,
-		   decltype(PoolOp::onfinish)&& onfinish);
-  void delete_pool(int64_t pool,
-		   Context* onfinish) {
-    delete_pool(pool, OpContextVert<ceph::buffer::list>(onfinish, nullptr));
-  }
-
-  void delete_pool(std::string_view name,
-		   decltype(PoolOp::onfinish)&& onfinish);
-
-  void delete_pool(std::string_view name,
-		   Context* onfinish) {
-    delete_pool(name, OpContextVert<ceph::buffer::list>(onfinish, nullptr));
-  }
+  int create_pool_snap(int64_t pool, std::string& snapName, Context *onfinish);
+  int allocate_selfmanaged_snap(int64_t pool, snapid_t *psnapid,
+				Context *onfinish);
+  int delete_pool_snap(int64_t pool, std::string& snapName, Context *onfinish);
+  int delete_selfmanaged_snap(int64_t pool, snapid_t snap, Context *onfinish);
+
+  int create_pool(std::string& name, Context *onfinish,
+		  int crush_rule=-1);
+  int delete_pool(int64_t pool, Context *onfinish);
+  int delete_pool(const std::string& name, Context *onfinish);
 
   void handle_pool_op_reply(MPoolOpReply *m);
   int pool_op_cancel(ceph_tid_t tid, int r);
@@ -3610,19 +3009,10 @@ private:
   void _poolstat_submit(PoolStatOp *op);
 public:
   void handle_get_pool_stats_reply(MGetPoolStatsReply *m);
-  void get_pool_stats(const std::vector<std::string>& pools,
-		      decltype(PoolStatOp::onfinish)&& onfinish);
-  template<typename CompletionToken>
-  auto get_pool_stats(const std::vector<std::string>& pools,
-		      CompletionToken&& token) {
-    boost::asio::async_completion<CompletionToken,
-				  PoolStatOp::OpSig> init(token);
-    get_pool_stats(pools,
-		   PoolStatOp::OpComp::create(
-		     service.get_executor(),
-		     std::move(init.completion_handler)));
-    return init.result.get();
-  }
+  void get_pool_stats(std::list<std::string>& pools,
+		      std::map<std::string,pool_stat_t> *result,
+		      bool *per_pool,
+		      Context *onfinish);
   int pool_stat_op_cancel(ceph_tid_t tid, int r);
   void _finish_pool_stat_op(PoolStatOp *op, int r);
 
@@ -3632,12 +3022,8 @@ private:
   void _fs_stats_submit(StatfsOp *op);
 public:
   void handle_fs_stats_reply(MStatfsReply *m);
-  void get_fs_stats(boost::optional<int64_t> poolid,
-		    decltype(StatfsOp::onfinish)&& onfinish);
   void get_fs_stats(struct ceph_statfs& result, boost::optional<int64_t> poolid,
-		    Context *onfinish) {
-    get_fs_stats(poolid, OpContextVert(onfinish, result));
-  }
+		    Context *onfinish);
   int statfs_op_cancel(ceph_tid_t tid, int r);
   void _finish_statfs_op(StatfsOp *op, int r);
 
@@ -3735,9 +3121,7 @@ public:
 
 private:
   epoch_t epoch_barrier = 0;
-  bool retry_writes_after_first_reply =
-    cct->_conf->objecter_retry_writes_after_first_reply;
-
+  bool retry_writes_after_first_reply;
 public:
   void set_epoch_barrier(epoch_t epoch);
 
diff --git a/src/osdc/error_code.cc b/src/osdc/error_code.cc
deleted file mode 100644
index 8017f7a4ce5..00000000000
--- a/src/osdc/error_code.cc
+++ /dev/null
@@ -1,147 +0,0 @@
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2019 Red Hat <contact@redhat.com>
- * Author: Adam C. Emerson <aemerson@redhat.com>
- *
- * This is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License version 2.1, as published by the Free Software
- * Foundation.  See file COPYING.
- *
- */
-
-#include <string>
-
-#include "common/error_code.h"
-#include "error_code.h"
-
-namespace bs = boost::system;
-
-class osdc_error_category : public ceph::converting_category {
-public:
-  osdc_error_category(){}
-  const char* name() const noexcept override;
-  std::string message(int ev) const override;
-  bs::error_condition default_error_condition(int ev) const noexcept
-    override;
-  bool equivalent(int ev, const bs::error_condition& c) const
-    noexcept override;
-  using ceph::converting_category::equivalent;
-  int from_code(int ev) const noexcept override;
-};
-
-const char* osdc_error_category::name() const noexcept {
-  return "osdc";
-}
-
-std::string osdc_error_category::message(int ev) const {
-  if (ev == 0)
-    return "No error";
-
-  switch (static_cast<osdc_errc>(ev)) {
-  case osdc_errc::pool_dne:
-    return "Pool does not exist";
-
-  case osdc_errc::pool_exists:
-    return "Pool already exists";
-
-  case osdc_errc::precondition_violated:
-    return "Precondition for operation not satisfied";
-
-  case osdc_errc::not_supported:
-    return "Operation not supported";
-
-  case osdc_errc::snapshot_exists:
-    return "Snapshot already exists";
-
-  case osdc_errc::snapshot_dne:
-    return "Snapshot does not exist";
-
-  case osdc_errc::timed_out:
-    return "Operation timed out";
-  }
-
-  return "Unknown error";
-}
-
-bs::error_condition
-osdc_error_category::default_error_condition(int ev) const noexcept {
-  switch (static_cast<osdc_errc>(ev)) {
-  case osdc_errc::pool_dne:
-    return ceph::errc::does_not_exist;
-  case osdc_errc::pool_exists:
-    return ceph::errc::exists;
-  case osdc_errc::precondition_violated:
-    return bs::errc::invalid_argument;
-  case osdc_errc::not_supported:
-    return bs::errc::operation_not_supported;
-  case osdc_errc::snapshot_exists:
-    return ceph::errc::exists;
-  case osdc_errc::snapshot_dne:
-    return ceph::errc::does_not_exist;
-  case osdc_errc::timed_out:
-    return bs::errc::timed_out;
-  }
-
-  return { ev, *this };
-}
-
-bool osdc_error_category::equivalent(int ev,
-                                     const bs::error_condition& c) const noexcept {
-  if (ev == osdc_errc::pool_dne) {
-    if (c == bs::errc::no_such_file_or_directory) {
-      return true;
-    }
-    if (c == ceph::errc::not_in_map) {
-      return true;
-    }
-  }
-  if (ev == osdc_errc::pool_exists) {
-    if (c == bs::errc::file_exists) {
-      return true;
-    }
-  }
-  if (ev == osdc_errc::snapshot_exists) {
-    if (c == bs::errc::file_exists) {
-      return true;
-    }
-  }
-  if (ev == osdc_errc::snapshot_dne) {
-    if (c == bs::errc::no_such_file_or_directory) {
-      return true;
-    }
-    if (c == ceph::errc::not_in_map) {
-      return true;
-    }
-  }
-
-  return default_error_condition(ev) == c;
-}
-
-int osdc_error_category::from_code(int ev) const noexcept {
-  switch (static_cast<osdc_errc>(ev)) {
-  case osdc_errc::pool_dne:
-    return -ENOENT;
-  case osdc_errc::pool_exists:
-    return -EEXIST;
-  case osdc_errc::precondition_violated:
-    return -EINVAL;
-  case osdc_errc::not_supported:
-    return -EOPNOTSUPP;
-  case osdc_errc::snapshot_exists:
-    return -EEXIST;
-  case osdc_errc::snapshot_dne:
-    return -ENOENT;
-  case osdc_errc::timed_out:
-    return -ETIMEDOUT;
-  }
-  return -EDOM;
-}
-
-const bs::error_category& osdc_category() noexcept {
-  static const osdc_error_category c;
-  return c;
-}
diff --git a/src/osdc/error_code.h b/src/osdc/error_code.h
deleted file mode 100644
index 224c9768879..00000000000
--- a/src/osdc/error_code.h
+++ /dev/null
@@ -1,53 +0,0 @@
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2019 Red Hat <contact@redhat.com>
- * Author: Adam C. Emerson <aemerson@redhat.com>
- *
- * This is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License version 2.1, as published by the Free Software
- * Foundation.  See file COPYING.
- *
- */
-
-#pragma once
-
-#include <boost/system/error_code.hpp>
-
-#include "include/rados.h"
-
-const boost::system::error_category& osdc_category() noexcept;
-
-enum osdc_errc {
-  pool_dne = 1,
-  pool_exists,
-  // Come the revolution, we'll just kill your program. Maybe.
-  precondition_violated,
-  not_supported,
-  snapshot_exists,
-  snapshot_dne,
-  timed_out
-};
-
-namespace boost {
-namespace system {
-template<>
-struct is_error_code_enum<::osdc_errc> {
-  static const bool value = true;
-};
-}
-}
-
-//  explicit conversion:
-inline boost::system::error_code make_error_code(osdc_errc e) noexcept {
-  return { e, osdc_category() };
-}
-
-// implicit conversion:
-inline boost::system::error_condition make_error_condition(osdc_errc e)
-  noexcept {
-  return { e, osdc_category() };
-}
diff --git a/src/rgw/rgw_bucket.cc b/src/rgw/rgw_bucket.cc
index f702d8d5e12..1a3b6b1a4c5 100644
--- a/src/rgw/rgw_bucket.cc
+++ b/src/rgw/rgw_bucket.cc
@@ -3573,8 +3573,7 @@ int RGWBucketCtl::chown(rgw::sal::RGWRadosStore *store, RGWBucketInfo& bucket_in
           decode(policy, bl);
           owner = policy.get_owner();
         } catch (buffer::error& err) {
-          ldout(store->ctx(), 0) << "ERROR: decode policy failed" << err.what()
-				 << dendl;
+          ldout(store->ctx(), 0) << "ERROR: decode policy failed" << err << dendl;
           return -EIO;
         }
 
diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h
index a45730867eb..cdc85016318 100644
--- a/src/rgw/rgw_common.h
+++ b/src/rgw/rgw_common.h
@@ -2299,7 +2299,7 @@ struct rgw_obj {
         } else {
           ssize_t pos = key.name.find('_', 1);
           if (pos < 0) {
-            throw buffer::malformed_input();
+            throw buffer::error();
           }
           key.name = key.name.substr(pos + 1);
         }
diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt
index 401a5151aac..c9bfe939939 100644
--- a/src/test/CMakeLists.txt
+++ b/src/test/CMakeLists.txt
@@ -32,7 +32,6 @@ add_subdirectory(fs)
 add_subdirectory(journal)
 add_subdirectory(libcephfs)
 add_subdirectory(librados)
-add_subdirectory(RADOS)
 add_subdirectory(librados_test_stub)
 if(WITH_LIBRADOSSTRIPER)
   add_subdirectory(libradosstriper)
diff --git a/src/test/RADOS/CMakeLists.txt b/src/test/RADOS/CMakeLists.txt
deleted file mode 100644
index e95fbc47ec3..00000000000
--- a/src/test/RADOS/CMakeLists.txt
+++ /dev/null
@@ -1,15 +0,0 @@
-add_executable(ceph_test_RADOS_start_stop start_stop.cc)
-target_link_libraries(ceph_test_RADOS_start_stop
-  global libRADOS ${unittest_libs})
-
-add_executable(ceph_test_RADOS_completions completions.cc)
-target_link_libraries(ceph_test_RADOS_completions Boost::system pthread
-  ${unittest_libs})
-
-add_executable(ceph_test_RADOS_op_speed op_speed.cc)
-target_link_libraries(ceph_test_RADOS_op_speed
-  global libRADOS fmt::fmt ${unittest_libs})
-
-add_executable(ceph_test_RADOS_list_pool list_pool.cc)
-target_link_libraries(ceph_test_RADOS_list_pool
-  global libRADOS fmt::fmt ${unittest_libs})
diff --git a/src/test/RADOS/completions.cc b/src/test/RADOS/completions.cc
deleted file mode 100644
index d9c0e087005..00000000000
--- a/src/test/RADOS/completions.cc
+++ /dev/null
@@ -1,20 +0,0 @@
-#include <cassert>
-#include <boost/asio.hpp>
-#include <boost/system/system_error.hpp>
-
-constexpr int max_completions = 10'000'000;
-int completed = 0;
-
-boost::asio::io_context c;
-
-void nested_cb() {
-  if (++completed < max_completions)
-    c.post(&nested_cb);
-}
-
-int main(void) {
-  c.post(&nested_cb);
-  c.run();
-  assert(completed == max_completions);
-  return 0;
-}
diff --git a/src/test/RADOS/list_pool.cc b/src/test/RADOS/list_pool.cc
deleted file mode 100644
index 5d5cb3dfd4e..00000000000
--- a/src/test/RADOS/list_pool.cc
+++ /dev/null
@@ -1,192 +0,0 @@
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2019 Red Hat <contact@redhat.com>
- * Author: Adam C. Emerson <aemerson@redhat.com>
- *
- * This is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License version 2.1, as published by the Free Software
- * Foundation.  See file COPYING.
- *
- */
-
-#include <iostream>
-#include <initializer_list>
-#include <optional>
-#include <thread>
-#include <tuple>
-#include <string_view>
-#include <vector>
-
-#include <sys/param.h>
-
-#include <unistd.h>
-
-#include <boost/system/system_error.hpp>
-
-#include <fmt/format.h>
-
-#include "include/RADOS/RADOS.hpp"
-
-#include "include/scope_guard.h"
-
-#include "common/async/context_pool.h"
-#include "common/ceph_time.h"
-#include "common/ceph_argparse.h"
-#include "common/async/waiter.h"
-
-#include "global/global_init.h"
-
-std::string_view hostname() {
-  static char hostname[MAXHOSTNAMELEN] = { 0 };
-  static size_t len = 0;
-  if (!len) {
-    auto r = gethostname(hostname, sizeof(hostname));
-    if (r != 0) {
-      throw boost::system::system_error(
-        errno, boost::system::system_category());
-    }
-    len = std::strlen(hostname);
-  }
-  return {hostname, len};
-}
-
-std::string temp_pool_name(const std::string_view prefix)
-{
-  using namespace std::chrono;
-  static std::uint64_t num = 1;
-  return fmt::format(
-    "{}-{}-{}-{}-{}",
-    prefix,
-    hostname(),
-    getpid(),
-    duration_cast<milliseconds>(ceph::coarse_real_clock::now()
-                                .time_since_epoch()).count(),
-    num++);
-}
-
-boost::system::error_code noisy_list(RADOS::RADOS& r,
-                                     int64_t p) {
-    ceph::async::waiter<boost::system::error_code> w;
-    RADOS::Cursor next;
-    std::vector<RADOS::Entry> v;
-    auto b = RADOS::Cursor::begin();
-    auto e = RADOS::Cursor::end();
-
-    std::cout << "begin = " << b.to_str() << std::endl;
-    std::cout << "end = " << e.to_str() << std::endl;
-    r.enumerate_objects(p, b, e, 1000, {}, &v, &next, w,
-                        RADOS::all_nspaces);
-    auto ec = w.wait();
-    if (ec) {
-      std::cerr << "RADOS::enumerate_objects: " << ec << std::endl;
-      return ec;
-    }
-
-    std::cout << "Got " << v.size() << " entries." << std::endl;
-
-    std::cout << "next cursor = " << next.to_str() << std::endl;
-    std::cout << "next == end: " << (next == e) << std::endl;
-    std::cout << "Returned Objects: ";
-    std::cout << "[";
-    auto o = v.cbegin();
-    while (o != v.cend()) {
-      std::cout << *o;
-      if (++o != v.cend())
-        std::cout << " ";
-    }
-    std::cout << "]" << std::endl;
-    return {};
-}
-
-boost::system::error_code create_several(RADOS::RADOS& r,
-                                         const RADOS::IOContext& i,
-                                         std::initializer_list<std::string> l) {
-  for (const auto& o : l) {
-    ceph::async::waiter<boost::system::error_code> w;
-    RADOS::WriteOp op;
-    std::cout << "Creating " << o << std::endl;
-    ceph::bufferlist bl;
-    bl.append("My bologna has no name.");
-    op.write_full(std::move(bl));
-    r.execute(o, i, std::move(op), w);
-    auto ec = w.wait();
-    if (ec) {
-      std::cerr << "RADOS::execute: " << ec << std::endl;
-      return ec;
-    }
-  }
-  return {};
-}
-
-int main(int argc, char** argv)
-{
-  using namespace std::literals;
-
-  std::vector<const char*> args;
-  argv_to_vec(argc, const_cast<const char**>(argv), args);
-  env_to_vec(args);
-
-  auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT,
-                         CODE_ENVIRONMENT_UTILITY, 0);
-  common_init_finish(cct.get());
-
-  ceph::async::io_context_pool p(1);
-  auto r = RADOS::RADOS::make_with_cct(cct.get(), p,
-				       boost::asio::use_future).get();
-
-  auto pool_name = temp_pool_name("ceph_test_RADOS_list_pool");
-
-  {
-    ceph::async::waiter<boost::system::error_code> w;
-    r.create_pool(pool_name, std::nullopt, w);
-    auto ec = w.wait();
-    if (ec) {
-      std::cerr << "RADOS::create_pool: " << ec << std::endl;
-      return 1;
-    }
-  }
-
-  auto pd = make_scope_guard(
-    [&pool_name, &r]() {
-      ceph::async::waiter<boost::system::error_code> w;
-      r.delete_pool(pool_name, w);
-      auto ec = w.wait();
-      if (ec)
-        std::cerr << "RADOS::delete_pool: " << ec << std::endl;
-    });
-
-  std::int64_t pool;
-
-  {
-    ceph::async::waiter<boost::system::error_code, std::int64_t> w;
-    r.lookup_pool(pool_name, w);
-    boost::system::error_code ec;
-    std::tie(ec, pool) = w.wait();
-    if (ec) {
-      std::cerr << "RADOS::lookup_pool: " << ec << std::endl;
-      return 1;
-    }
-  }
-
-  RADOS::IOContext i(pool);
-
-  if (noisy_list(r, pool)) {
-    return 1;
-  }
-
-  if (create_several(r, i, {"meow", "woof", "squeak"})) {
-    return 1;
-  }
-
-  std::this_thread::sleep_for(5s);
-
-  if (noisy_list(r, pool)) {
-    return 1;
-  }
-
-  return 0;
-}
diff --git a/src/test/RADOS/op_speed.cc b/src/test/RADOS/op_speed.cc
deleted file mode 100644
index 715404bd359..00000000000
--- a/src/test/RADOS/op_speed.cc
+++ /dev/null
@@ -1,34 +0,0 @@
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2019 Red Hat <contact@redhat.com>
- * Author: Adam C. Emerson <aemerson@redhat.com>
- *
- * This is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License version 2.1, as published by the Free Software
- * Foundation.  See file COPYING.
- *
- */
-
-#include "include/RADOS/RADOS.hpp"
-
-constexpr int to_create = 10'000'000;
-
-int main() {
-  for (int i = 0; i < to_create; ++i) {
-    RADOS::ReadOp op;
-    bufferlist bl;
-    std::uint64_t sz;
-    ceph::real_time tm;
-    boost::container::flat_map<std::string, ceph::buffer::list> xattrs;
-    boost::container::flat_map<std::string, ceph::buffer::list> omap;
-    bool trunc;
-    op.read(0, 0, &bl);
-    op.stat(&sz, &tm);
-    op.get_xattrs(&xattrs);
-    op.get_omap_vals(std::nullopt, std::nullopt, 1000, &omap, &trunc);
-  }
-}
diff --git a/src/test/RADOS/start_stop.cc b/src/test/RADOS/start_stop.cc
deleted file mode 100644
index 42132ba1a20..00000000000
--- a/src/test/RADOS/start_stop.cc
+++ /dev/null
@@ -1,173 +0,0 @@
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2019 Red Hat <contact@redhat.com>
- * Author: Adam C. Emerson <aemerson@redhat.com>
- *
- * This is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License version 2.1, as published by the Free Software
- * Foundation.  See file COPYING.
- *
- */
-
-#include <thread>
-#include <vector>
-
-#include "include/RADOS/RADOS.hpp"
-
-#include "common/async/context_pool.h"
-#include "common/ceph_argparse.h"
-
-#include "global/global_init.h"
-
-
-int main(int argc, char** argv)
-{
-  using namespace std::literals;
-
-  std::vector<const char*> args;
-  argv_to_vec(argc, const_cast<const char**>(argv), args);
-  env_to_vec(args);
-
-  auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT,
-                         CODE_ENVIRONMENT_UTILITY, 0);
-  common_init_finish(cct.get());
-
-  {
-    ceph::async::io_context_pool p(1);
-    auto r = RADOS::RADOS::make_with_cct(cct.get(), p,
-					 boost::asio::use_future).get();
-    std::this_thread::sleep_for(30s);
-  }
-  std::this_thread::sleep_for(30s);
-  {
-    ceph::async::io_context_pool p(1);
-    auto r = RADOS::RADOS::make_with_cct(cct.get(), p,
-					 boost::asio::use_future).get();
-    std::this_thread::sleep_for(30s);
-  }
-  {
-    ceph::async::io_context_pool p(1);
-    auto r = RADOS::RADOS::make_with_cct(cct.get(), p,
-					 boost::asio::use_future).get();
-    std::this_thread::sleep_for(1s);
-  }
-  {
-    ceph::async::io_context_pool p(1);
-    auto r = RADOS::RADOS::make_with_cct(cct.get(), p,
-					 boost::asio::use_future).get();
-    std::this_thread::sleep_for(1s);
-  }
-  {
-    ceph::async::io_context_pool p(1);
-    auto r = RADOS::RADOS::make_with_cct(cct.get(), p,
-					 boost::asio::use_future).get();
-    std::this_thread::sleep_for(1s);
-  }
-  {
-    ceph::async::io_context_pool p(1);
-    auto r = RADOS::RADOS::make_with_cct(cct.get(), p,
-					 boost::asio::use_future).get();
-    std::this_thread::sleep_for(1s);
-  }
-  {
-    ceph::async::io_context_pool p(1);
-    auto r = RADOS::RADOS::make_with_cct(cct.get(), p,
-					 boost::asio::use_future).get();
-    std::this_thread::sleep_for(1s);
-  }
-  {
-    ceph::async::io_context_pool p(1);
-    auto r = RADOS::RADOS::make_with_cct(cct.get(), p,
-					 boost::asio::use_future).get();
-    std::this_thread::sleep_for(1s);
-  }
-  {
-    ceph::async::io_context_pool p(1);
-    auto r = RADOS::RADOS::make_with_cct(cct.get(), p,
-					 boost::asio::use_future).get();
-    std::this_thread::sleep_for(1s);
-  }
-  {
-    ceph::async::io_context_pool p(1);
-    auto r = RADOS::RADOS::make_with_cct(cct.get(), p,
-					 boost::asio::use_future).get();
-    std::this_thread::sleep_for(500ms);
-  }
-  {
-    ceph::async::io_context_pool p(1);
-    auto r = RADOS::RADOS::make_with_cct(cct.get(), p,
-					 boost::asio::use_future).get();
-    std::this_thread::sleep_for(500ms);
-  }
-  {
-    ceph::async::io_context_pool p(1);
-    auto r = RADOS::RADOS::make_with_cct(cct.get(), p,
-					 boost::asio::use_future).get();
-    std::this_thread::sleep_for(50ms);
-  }
-  {
-    ceph::async::io_context_pool p(1);
-    auto r = RADOS::RADOS::make_with_cct(cct.get(), p,
-					 boost::asio::use_future).get();
-    std::this_thread::sleep_for(50ms);
-  }
-  {
-    ceph::async::io_context_pool p(1);
-    auto r = RADOS::RADOS::make_with_cct(cct.get(), p,
-					 boost::asio::use_future).get();
-    std::this_thread::sleep_for(50ms);
-  }
-  {
-    ceph::async::io_context_pool p(1);
-    auto r = RADOS::RADOS::make_with_cct(cct.get(), p,
-					 boost::asio::use_future).get();
-    std::this_thread::sleep_for(5ms);
-  }
-  {
-    ceph::async::io_context_pool p(1);
-    auto r = RADOS::RADOS::make_with_cct(cct.get(), p,
-					 boost::asio::use_future).get();
-    std::this_thread::sleep_for(5ms);
-  }
-  {
-    ceph::async::io_context_pool p(1);
-    auto r = RADOS::RADOS::make_with_cct(cct.get(), p,
-					 boost::asio::use_future).get();
-    std::this_thread::sleep_for(5ms);
-  }
-  {
-    ceph::async::io_context_pool p(1);
-    auto r = RADOS::RADOS::make_with_cct(cct.get(), p,
-					 boost::asio::use_future).get();
-    std::this_thread::sleep_for(5ms);
-  }
-  {
-    ceph::async::io_context_pool p(1);
-    auto r = RADOS::RADOS::make_with_cct(cct.get(), p,
-					 boost::asio::use_future).get();
-    std::this_thread::sleep_for(5ms);
-  }
-  {
-    ceph::async::io_context_pool p(1);
-    auto r = RADOS::RADOS::make_with_cct(cct.get(), p,
-					 boost::asio::use_future).get();
-    std::this_thread::sleep_for(5us);
-  }
-  {
-    ceph::async::io_context_pool p(1);
-    auto r = RADOS::RADOS::make_with_cct(cct.get(), p,
-					 boost::asio::use_future).get();
-    std::this_thread::sleep_for(5us);
-  }
-  {
-    ceph::async::io_context_pool p(1);
-    auto r = RADOS::RADOS::make_with_cct(cct.get(), p,
-					 boost::asio::use_future).get();
-    std::this_thread::sleep_for(5us);
-  }
-  return 0;
-}
diff --git a/src/test/crimson/test_socket.cc b/src/test/crimson/test_socket.cc
index 557a0cf6f06..ec306700bd7 100644
--- a/src/test/crimson/test_socket.cc
+++ b/src/test/crimson/test_socket.cc
@@ -32,7 +32,7 @@ class SocketFactoryBase
  public:
   virtual ~SocketFactoryBase() = default;
 
-  virtual seastar::future<> bind_accept() {
+  virtual future<> bind_accept() {
     return this->container().invoke_on_all([] (auto& factory) {
       entity_addr_t addr;
       addr.parse(server_addr, nullptr);
@@ -77,7 +77,7 @@ class SocketFactoryBase
     });
   }
 
-  seastar::future<> shutdown() {
+  future<> shutdown() {
     return this->container().invoke_on_all([] (auto& factory) {
       if (factory.listener) {
         factory.listener.value().abort_accept();
@@ -86,22 +86,22 @@ class SocketFactoryBase
     });
   }
 
-  seastar::future<> stop() { return seastar::now(); }
+  future<> stop() { return seastar::now(); }
 
-  static seastar::future<SocketFRef> connect() {
+  static future<SocketFRef> connect() {
     entity_addr_t addr;
     addr.parse(server_addr, nullptr);
     return Socket::connect(addr);
   }
 
  protected:
-  virtual seastar::future<> handle_server_socket(SocketFRef&& socket) = 0;
+  virtual future<> handle_server_socket(SocketFRef&& socket) = 0;
 };
 
 class AcceptTest final
     : public SocketFactoryBase<AcceptTest> {
  public:
-  seastar::future<> handle_server_socket(SocketFRef&& socket) override {
+  future<> handle_server_socket(SocketFRef&& socket) override {
     return seastar::sleep(100ms
     ).then([socket = std::move(socket)] () mutable {
       return socket->close()
@@ -110,7 +110,7 @@ class AcceptTest final
   }
 };
 
-seastar::future<> test_refused() {
+future<> test_refused() {
   logger.info("test_refused()...");
   return AcceptTest::connect().discard_result(
   ).then([] {
@@ -127,7 +127,7 @@ seastar::future<> test_refused() {
   });
 }
 
-seastar::future<> test_bind_same() {
+future<> test_bind_same() {
   logger.info("test_bind_same()...");
   return crimson::net::create_sharded<AcceptTest>().then(
     [] (AcceptTest* factory) {
@@ -157,7 +157,7 @@ seastar::future<> test_bind_same() {
   });
 }
 
-seastar::future<> test_accept() {
+future<> test_accept() {
   logger.info("test_accept()");
   return crimson::net::create_sharded<AcceptTest>(
   ).then([] (AcceptTest* factory) {
@@ -187,25 +187,25 @@ class SocketFactory final
   const seastar::shard_id target_shard;
   seastar::promise<SocketFRef> socket_promise;
 
-  seastar::future<> bind_accept() override {
+  future<> bind_accept() override {
     return SocketFactoryBase<SocketFactory>::bind_accept();
   }
 
-  seastar::future<SocketFRef> get_accepted() {
+  future<SocketFRef> get_accepted() {
     return socket_promise.get_future();
   }
 
  public:
   SocketFactory(seastar::shard_id shard) : target_shard{shard} {}
 
-  seastar::future<> handle_server_socket(SocketFRef&& socket) override {
+  future<> handle_server_socket(SocketFRef&& socket) override {
     return container().invoke_on(target_shard,
         [socket = std::move(socket)] (auto& factory) mutable {
       factory.socket_promise.set_value(std::move(socket));
     });
   }
 
-static seastar::future<tuple<SocketFRef, SocketFRef>> get_sockets() {
+  static future<tuple<SocketFRef, SocketFRef>> get_sockets() {
     return crimson::net::create_sharded<SocketFactory>(seastar::engine().cpu_id()
     ).then([] (SocketFactory* factory) {
       return factory->bind_accept().then([factory] {
@@ -241,7 +241,7 @@ class Connection {
     data[DATA_SIZE - 1] = DATA_TAIL;
   }
 
-  seastar::future<> dispatch_write(unsigned round = 0, bool force_shut = false) {
+  future<> dispatch_write(unsigned round = 0, bool force_shut = false) {
     return seastar::repeat([this, round, force_shut] {
       if (round != 0 && round <= write_count) {
         return seastar::futurize_apply([this, force_shut] {
@@ -265,7 +265,7 @@ class Connection {
     });
   }
 
-  seastar::future<> dispatch_write_unbounded() {
+  future<> dispatch_write_unbounded() {
     return dispatch_write(
     ).then([] {
       ceph_abort();
@@ -282,7 +282,7 @@ class Connection {
     });
   }
 
-  seastar::future<> dispatch_read(unsigned round = 0, bool force_shut = false) {
+  future<> dispatch_read(unsigned round = 0, bool force_shut = false) {
     return seastar::repeat([this, round, force_shut] {
       if (round != 0 && round <= read_count) {
         return seastar::futurize_apply([this, force_shut] {
@@ -318,7 +318,7 @@ class Connection {
     });
   }
 
-  seastar::future<> dispatch_read_unbounded() {
+  future<> dispatch_read_unbounded() {
     return dispatch_read(
     ).then([] {
       ceph_abort();
@@ -339,12 +339,12 @@ class Connection {
     socket->shutdown();
   }
 
-  seastar::future<> close() {
+  future<> close() {
     return socket->close();
   }
 
  public:
-  static seastar::future<> dispatch_rw_bounded(SocketFRef&& socket, bool is_client,
+  static future<> dispatch_rw_bounded(SocketFRef&& socket, bool is_client,
                                       unsigned round, bool force_shut = false) {
     return seastar::smp::submit_to(is_client ? 0 : 1,
         [socket = std::move(socket), round, force_shut] () mutable {
@@ -365,7 +365,7 @@ class Connection {
     });
   }
 
-  static seastar::future<> dispatch_rw_unbounded(SocketFRef&& socket, bool is_client,
+  static future<> dispatch_rw_unbounded(SocketFRef&& socket, bool is_client,
                                         bool preemptive_shut = false) {
     return seastar::smp::submit_to(is_client ? 0 : 1,
         [socket = std::move(socket), preemptive_shut, is_client] () mutable {
@@ -393,7 +393,7 @@ class Connection {
   }
 };
 
-seastar::future<> test_read_write() {
+future<> test_read_write() {
   logger.info("test_read_write()...");
   return SocketFactory::get_sockets().then([] (auto sockets) {
     auto [client_socket, server_socket] = std::move(sockets);
@@ -407,7 +407,7 @@ seastar::future<> test_read_write() {
   });
 }
 
-seastar::future<> test_unexpected_down() {
+future<> test_unexpected_down() {
   logger.info("test_unexpected_down()...");
   return SocketFactory::get_sockets().then([] (auto sockets) {
     auto [client_socket, server_socket] = std::move(sockets);
@@ -421,7 +421,7 @@ seastar::future<> test_unexpected_down() {
   });
 }
 
-seastar::future<> test_shutdown_propagated() {
+future<> test_shutdown_propagated() {
   logger.info("test_shutdown_propagated()...");
   return SocketFactory::get_sockets().then([] (auto sockets) {
     auto [client_socket, server_socket] = std::move(sockets);
@@ -437,7 +437,7 @@ seastar::future<> test_shutdown_propagated() {
   });
 }
 
-seastar::future<> test_preemptive_down() {
+future<> test_preemptive_down() {
   logger.info("test_preemptive_down()...");
   return SocketFactory::get_sockets().then([] (auto sockets) {
     auto [client_socket, server_socket] = std::move(sockets);
diff --git a/src/test/encoding.cc b/src/test/encoding.cc
index 6d252fae18b..4f91ecc4deb 100644
--- a/src/test/encoding.cc
+++ b/src/test/encoding.cc
@@ -320,8 +320,8 @@ TEST(EncodingRoundTrip, Integers) {
 }
 
 const char* expected_what[] = {
-  "void lame_decoder(int) no longer understand old encoding version 100 < 200: Malformed input",
-  "void lame_decoder(int) decode past end of struct encoding: Malformed input"
+  "buffer::malformed_input: void lame_decoder(int) no longer understand old encoding version 100 < 200",
+  "buffer::malformed_input: void lame_decoder(int) decode past end of struct encoding",
 };
 
 void lame_decoder(int which) {
diff --git a/src/test/librados/CMakeLists.txt b/src/test/librados/CMakeLists.txt
index b0faa7c76ac..4fc53d24066 100644
--- a/src/test/librados/CMakeLists.txt
+++ b/src/test/librados/CMakeLists.txt
@@ -187,15 +187,3 @@ target_link_libraries(unittest_librados_config
   librados
   ${BLKID_LIBRARIES} ${GSSAPI_LIBRARIES} ${OPENLDAP_LIBRARIES})
 
-# Removing this test. We can't shove it into Finisher as it's not a
-# Context any more, and wrapping it to adapt it would be less fair.
-
-#add_executable(ceph_test_rados_completion_speed
-#  completion_speed.cc)
-#target_link_libraries(ceph_test_rados_completion_speed
-#  librados ${UNITTEST_LIBS} radostest-cxx)
-
-add_executable(ceph_test_rados_op_speed
-  op_speed.cc)
-target_link_libraries(ceph_test_rados_op_speed
-  librados ${UNITTEST_LIBS} radostest-cxx)
diff --git a/src/test/librados/completion_speed.cc b/src/test/librados/completion_speed.cc
deleted file mode 100644
index 708a58425e1..00000000000
--- a/src/test/librados/completion_speed.cc
+++ /dev/null
@@ -1,38 +0,0 @@
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-
-#include "include/rados/librados.hpp"
-#include "common/ceph_context.h"
-#include "common/Finisher.h"
-#include "librados/AioCompletionImpl.h"
-
-
-constexpr int max_completions = 10'000'000;
-int completed = 0;
-auto cct = (new CephContext(CEPH_ENTITY_TYPE_CLIENT))->get();
-Finisher f(cct);
-
-void completion_cb(librados::completion_t cb, void* arg) {
-  auto c = static_cast<librados::AioCompletion*>(arg);
-  delete c;
-  if (++completed < max_completions) {
-    auto aio = librados::Rados::aio_create_completion();
-    aio->set_complete_callback(static_cast<void*>(aio), &completion_cb);
-    f.queue(new librados::C_AioComplete(aio->pc));
-  }
-}
-
-int main(void) {
-  auto aio = librados::Rados::aio_create_completion();
-  aio->set_complete_callback(static_cast<void*>(aio), &completion_cb);
-  f.queue(new librados::C_AioComplete(aio->pc));
-  f.start();
-
-  while (completed < max_completions)
-    f.wait_for_empty();
-
-  f.stop();
-
-  assert(completed == max_completions);
-  cct->put();
-}
diff --git a/src/test/librados/op_speed.cc b/src/test/librados/op_speed.cc
deleted file mode 100644
index 90c7bdac571..00000000000
--- a/src/test/librados/op_speed.cc
+++ /dev/null
@@ -1,22 +0,0 @@
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*
-// vim: ts=8 sw=2 smarttab
-
-#include "include/rados/librados.hpp"
-
-constexpr int to_create = 10'000'000;
-
-int main() {
-  for (int i = 0; i < to_create; ++i) {
-    librados::ObjectReadOperation op;
-    bufferlist bl;
-    std::uint64_t sz;
-    struct timespec tm;
-    std::map<std::string, ceph::buffer::list> xattrs;
-    std::map<std::string, ceph::buffer::list> omap;
-    bool more;
-    op.read(0, 0, &bl, nullptr);
-    op.stat2(&sz, &tm, nullptr);
-    op.getxattrs(&xattrs, nullptr);
-    op.omap_get_vals2({}, 1000, &omap, &more, nullptr);
-  }
-}
diff --git a/src/test/mon/test-mon-msg.cc b/src/test/mon/test-mon-msg.cc
index 4c551cab810..ad7ab6c8e23 100644
--- a/src/test/mon/test-mon-msg.cc
+++ b/src/test/mon/test-mon-msg.cc
@@ -20,7 +20,6 @@
 
 #include "global/global_init.h"
 #include "global/global_context.h"
-#include "common/async/context_pool.h"
 #include "common/ceph_argparse.h"
 #include "common/version.h"
 #include "common/dout.h"
@@ -52,7 +51,6 @@ class MonClientHelper : public Dispatcher
 {
 protected:
   CephContext *cct;
-  ceph::async::io_context_pool poolctx;
   Messenger *msg;
   MonClient monc;
 
@@ -65,9 +63,8 @@ public:
   explicit MonClientHelper(CephContext *cct_)
     : Dispatcher(cct_),
       cct(cct_),
-      poolctx(1),
       msg(NULL),
-      monc(cct_, poolctx)
+      monc(cct_)
   { }
 
 
diff --git a/src/test/mon/test_mon_workloadgen.cc b/src/test/mon/test_mon_workloadgen.cc
index 613a18f247e..15792a63af5 100644
--- a/src/test/mon/test_mon_workloadgen.cc
+++ b/src/test/mon/test_mon_workloadgen.cc
@@ -39,7 +39,6 @@
 #include "mon/MonClient.h"
 #include "msg/Dispatcher.h"
 #include "msg/Messenger.h"
-#include "common/async/context_pool.h"
 #include "common/Timer.h"
 #include "common/ceph_argparse.h"
 #include "global/global_init.h"
@@ -83,7 +82,6 @@ class TestStub : public Dispatcher
 {
  protected:
   MessengerRef messenger;
-  ceph::async::io_context_pool poolctx;
   MonClient monc;
 
   ceph::mutex lock;
@@ -165,7 +163,6 @@ class TestStub : public Dispatcher
     monc.shutdown();
     timer.shutdown();
     messenger->shutdown();
-    poolctx.finish();
     return 0;
   }
 
@@ -180,7 +177,7 @@ class TestStub : public Dispatcher
 
   TestStub(CephContext *cct, string who)
     : Dispatcher(cct),
-      monc(cct, poolctx),
+      monc(cct),
       lock(ceph::make_mutex(who.append("::lock"))),
       timer(cct, lock),
       do_shutdown(false),
@@ -247,7 +244,6 @@ class ClientStub : public TestStub
 
   int init() override {
     int err;
-    poolctx.start(1);
     err = monc.build_initial_monmap();
     if (err < 0) {
       derr << "ClientStub::" << __func__ << " ERROR: build initial monmap: "
@@ -263,7 +259,7 @@ class ClientStub : public TestStub
     dout(10) << "ClientStub::" << __func__ << " starting messenger at "
 	    << messenger->get_myaddrs() << dendl;
 
-    objecter.reset(new Objecter(cct, messenger.get(), &monc, poolctx, 0, 0));
+    objecter.reset(new Objecter(cct, messenger.get(), &monc, NULL, 0, 0));
     ceph_assert(objecter.get() != NULL);
     objecter->set_balanced_budget();
 
diff --git a/src/test/osd/TestOSDScrub.cc b/src/test/osd/TestOSDScrub.cc
index 810d77521bb..34a3b8d81af 100644
--- a/src/test/osd/TestOSDScrub.cc
+++ b/src/test/osd/TestOSDScrub.cc
@@ -22,7 +22,6 @@
 #include <stdio.h>
 #include <signal.h>
 #include <gtest/gtest.h>
-#include "common/async/context_pool.h"
 #include "osd/OSD.h"
 #include "os/ObjectStore.h"
 #include "mon/MonClient.h"
@@ -42,9 +41,8 @@ public:
       Messenger *hb_front_server,
       Messenger *hb_back_server,
       Messenger *osdc_messenger,
-      MonClient *mc, const std::string &dev, const std::string &jdev,
-      ceph::async::io_context_pool& ictx) :
-      OSD(cct_, store_, id, internal, external, hb_front_client, hb_back_client, hb_front_server, hb_back_server, osdc_messenger, mc, dev, jdev, ictx)
+      MonClient *mc, const std::string &dev, const std::string &jdev) :
+      OSD(cct_, store_, id, internal, external, hb_front_client, hb_back_client, hb_front_server, hb_back_server, osdc_messenger, mc, dev, jdev)
   {
   }
 
@@ -54,7 +52,6 @@ public:
 };
 
 TEST(TestOSDScrub, scrub_time_permit) {
-  ceph::async::io_context_pool icp(1);
   ObjectStore *store = ObjectStore::create(g_ceph_context,
              g_conf()->osd_objectstore,
              g_conf()->osd_data,
@@ -66,9 +63,9 @@ TEST(TestOSDScrub, scrub_time_permit) {
   ms->set_cluster_protocol(CEPH_OSD_PROTOCOL);
   ms->set_default_policy(Messenger::Policy::stateless_server(0));
   ms->bind(g_conf()->public_addr);
-  MonClient mc(g_ceph_context, icp);
+  MonClient mc(g_ceph_context);
   mc.build_initial_monmap();
-  TestOSDScrub* osd = new TestOSDScrub(g_ceph_context, store, 0, ms, ms, ms, ms, ms, ms, ms, &mc, "", "", icp);
+  TestOSDScrub* osd = new TestOSDScrub(g_ceph_context, store, 0, ms, ms, ms, ms, ms, ms, ms, &mc, "", "");
 
   g_ceph_context->_conf.set_val("osd_scrub_begin_hour", "0");
   g_ceph_context->_conf.set_val("osd_scrub_end_hour", "24");
diff --git a/src/test/rgw/test_rgw_common.h b/src/test/rgw/test_rgw_common.h
index 29ab3e4ac38..c360e3b0ad5 100644
--- a/src/test/rgw/test_rgw_common.h
+++ b/src/test/rgw/test_rgw_common.h
@@ -414,7 +414,7 @@ public:
       } else {
         ssize_t pos = object.find('_', 1);
         if (pos < 0) {
-          throw buffer::malformed_input();
+          throw buffer::error();
         }
         orig_obj = object.substr(pos);
       }
diff --git a/src/tools/CMakeLists.txt b/src/tools/CMakeLists.txt
index c0814c888f7..0f8120a7775 100644
--- a/src/tools/CMakeLists.txt
+++ b/src/tools/CMakeLists.txt
@@ -16,14 +16,6 @@ else()
 endif()
 install(TARGETS rados DESTINATION bin)
 
-set(neorados_srcs
-  neorados.cc)
-add_executable(neorados ${neorados_srcs})
-
-target_link_libraries(neorados libRADOS global Boost::coroutine ${CMAKE_DL_LIBS})
-#install(TARGETS neorados DESTINATION bin)
-
-
 if(WITH_TESTS)
 add_executable(ceph_scratchtool scratchtool.c)
 target_link_libraries(ceph_scratchtool librados global)
diff --git a/src/tools/ceph_monstore_tool.cc b/src/tools/ceph_monstore_tool.cc
index c3b3089bd23..030f8b437a9 100644
--- a/src/tools/ceph_monstore_tool.cc
+++ b/src/tools/ceph_monstore_tool.cc
@@ -924,7 +924,7 @@ int main(int argc, char **argv) {
         }
       } catch (const buffer::error &err) {
         std::cerr << "Could not decode for human readable output (you may still"
-	  " use non-readable mode).  Detail: " << err.what() << std::endl;
+                     " use non-readable mode).  Detail: " << err << std::endl;
       }
 
       out.append(ss);
diff --git a/src/tools/ceph_objectstore_tool.cc b/src/tools/ceph_objectstore_tool.cc
index 2bec6603a87..229c9f628ee 100644
--- a/src/tools/ceph_objectstore_tool.cc
+++ b/src/tools/ceph_objectstore_tool.cc
@@ -676,7 +676,7 @@ int do_trim_pg_log(ObjectStore *store, const coll_t &coll,
       try {
 	e.decode_with_checksum(bp);
       } catch (const buffer::error &e) {
-	cerr << "Error reading pg log entry: " << e.what() << std::endl;
+	cerr << "Error reading pg log entry: " << e << std::endl;
       }
       if (debug) {
 	cerr << "read entry " << e << std::endl;
diff --git a/src/tools/cephfs/DataScan.cc b/src/tools/cephfs/DataScan.cc
index 38e74301095..4dfbe48ccd9 100644
--- a/src/tools/cephfs/DataScan.cc
+++ b/src/tools/cephfs/DataScan.cc
@@ -1288,7 +1288,7 @@ int DataScan::scan_frags()
         auto q = parent_bl.cbegin();
         backtrace.decode(q);
       } catch (buffer::error &e) {
-        dout(4) << "Corrupt backtrace on '" << oid << "': " << e.what() << dendl;
+        dout(4) << "Corrupt backtrace on '" << oid << "': " << e << dendl;
         if (!force_corrupt) {
           return -EINVAL;
         } else {
@@ -1303,7 +1303,7 @@ int DataScan::scan_frags()
         auto q = layout_bl.cbegin();
         decode(loaded_layout, q);
       } catch (buffer::error &e) {
-        dout(4) << "Corrupt layout on '" << oid << "': " << e.what() << dendl;
+        dout(4) << "Corrupt layout on '" << oid << "': " << e << dendl;
         if (!force_corrupt) {
           return -EINVAL;
         }
@@ -1586,8 +1586,7 @@ int MetadataDriver::get_frag_of(
       backtrace.decode(q);
       have_backtrace = true;
     } catch (buffer::error &e) {
-      dout(4) << "Corrupt backtrace on '" << root_frag_oid << "': "
-	      << e.what() << dendl;
+      dout(4) << "Corrupt backtrace on '" << root_frag_oid << "': " << e << dendl;
     }
   }
 
diff --git a/src/tools/cephfs/MDSUtility.cc b/src/tools/cephfs/MDSUtility.cc
index cc4de9ff1b8..839b4aea050 100644
--- a/src/tools/cephfs/MDSUtility.cc
+++ b/src/tools/cephfs/MDSUtility.cc
@@ -25,10 +25,10 @@ MDSUtility::MDSUtility() :
   waiting_for_mds_map(NULL),
   inited(false)
 {
-  monc = new MonClient(g_ceph_context, poolctx);
+  monc = new MonClient(g_ceph_context);
   messenger = Messenger::create_client_messenger(g_ceph_context, "mds");
   fsmap = new FSMap();
-  objecter = new Objecter(g_ceph_context, messenger, monc, poolctx, 0, 0);
+  objecter = new Objecter(g_ceph_context, messenger, monc, NULL, 0, 0);
 }
 
 
@@ -52,7 +52,6 @@ int MDSUtility::init()
   if (r < 0)
     return r;
 
-  poolctx.start(1);
   messenger->start();
 
   objecter->set_client_incarnation(0);
@@ -126,7 +125,6 @@ void MDSUtility::shutdown()
   monc->shutdown();
   messenger->shutdown();
   messenger->wait();
-  poolctx.finish();
 }
 
 
diff --git a/src/tools/cephfs/MDSUtility.h b/src/tools/cephfs/MDSUtility.h
index 09f1918ba44..e5097ec48fe 100644
--- a/src/tools/cephfs/MDSUtility.h
+++ b/src/tools/cephfs/MDSUtility.h
@@ -20,7 +20,6 @@
 #include "msg/Dispatcher.h"
 #include "msg/Messenger.h"
 #include "auth/Auth.h"
-#include "common/async/context_pool.h"
 #include "common/Finisher.h"
 #include "common/Timer.h"
 
@@ -39,7 +38,6 @@ protected:
 
   ceph::mutex lock = ceph::make_mutex("MDSUtility::lock");
   Finisher finisher;
-  ceph::async::io_context_pool poolctx;
 
   Context *waiting_for_mds_map;
 
diff --git a/src/tools/neorados.cc b/src/tools/neorados.cc
deleted file mode 100644
index 6bfd2fb0f42..00000000000
--- a/src/tools/neorados.cc
+++ /dev/null
@@ -1,368 +0,0 @@
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
-// vim: ts=8 sw=2 smarttab
-/*
- * Ceph - scalable distributed file system
- *
- * Copyright (C) 2019 Red Hat <contact@redhat.com>
- * Author: Adam C. Emerson <aemerson@redhat.com>
- *
- * This is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License version 2.1, as published by the Free Software
- * Foundation.  See file COPYING.
- *
- */
-
-#define BOOST_COROUTINES_NO_DEPRECATION_WARNING
-
-#include <algorithm>
-#include <cassert>
-#include <iostream>
-#include <string>
-#include <string_view>
-#include <tuple>
-#include <vector>
-
-#include <boost/asio.hpp>
-#include <boost/asio/spawn.hpp>
-#include <boost/io/ios_state.hpp>
-#include <boost/program_options.hpp>
-#include <boost/system/system_error.hpp>
-
-#include <fmt/format.h>
-#include <fmt/ostream.h>
-
-#include "include/buffer.h" // :(
-
-#include "include/RADOS/RADOS.hpp"
-
-using namespace std::literals;
-
-namespace ba = boost::asio;
-namespace bs = boost::system;
-namespace R = RADOS;
-
-std::string verstr(const std::tuple<uint32_t, uint32_t, uint32_t>& v) {
-  const auto [maj, min, p] = v;
-  return fmt::format("v{}.{}.{}", maj, min, p);
-}
-
-template<typename V>
-void printseq(const V& v, std::ostream& m) {
-  std::for_each(v.cbegin(), v.cend(),
-		[&m](const auto& e) {
-		  fmt::print(m, "{}\n", e);
-		});
-}
-
-template<typename V, typename F>
-void printseq(const V& v, std::ostream& m, F&& f) {
-  std::for_each(v.cbegin(), v.cend(),
-		[&m, &f](const auto& e) {
-		  fmt::print(m, "{}\n", f(e));
-		});
-}
-
-std::int64_t lookup_pool(R::RADOS& r, const std::string& pname,
-			 ba::yield_context y) {
-  bs::error_code ec;
-  auto p = r.lookup_pool(pname, y[ec]);
-  if (ec)
-    throw bs::system_error(
-      ec, fmt::format("when looking up '{}'", pname));
-  return p;
-}
-
-
-void lspools(R::RADOS& r, const std::vector<std::string>&,
-	     ba::yield_context y) {
-  const auto l = r.list_pools(y);
-  printseq(l, std::cout, [](const auto& p) -> const std::string& {
-			   return p.second;
-			 });
-}
-
-
-void ls(R::RADOS& r, const std::vector<std::string>& p, ba::yield_context y) {
-  const auto& pname = p[0];
-  const auto pool = lookup_pool(r, pname, y);
-
-  std::vector<R::Entry> ls;
-  R::Cursor next = R::Cursor::begin();
-  bs::error_code ec;
-  do {
-    r.enumerate_objects(pool, next, R::Cursor::end(),
-			1000, {}, &ls, &next, y[ec], R::all_nspaces);
-    if (ec)
-      throw bs::system_error(ec, fmt::format("when listing {}", pname));
-    printseq(ls, std::cout);
-    ls.clear();
-  } while (next != R::Cursor::end());
-}
-
-void mkpool(R::RADOS& r, const std::vector<std::string>& p,
-	    ba::yield_context y) {
-  const auto& pname = p[0];
-  bs::error_code ec;
-  r.create_pool(pname, std::nullopt, y[ec]);
-  if (ec)
-    throw bs::system_error(ec, fmt::format("when creating pool '{}'", pname));
-}
-
-void rmpool(R::RADOS& r, const std::vector<std::string>& p,
-	    ba::yield_context y) {
-  const auto& pname = p[0];
-  bs::error_code ec;
-  r.delete_pool(pname, y[ec]);
-  if (ec)
-    throw bs::system_error(ec, fmt::format("when removing pool '{}'", pname));
-}
-
-void create(R::RADOS& r, const std::vector<std::string>& p,
-	    ba::yield_context y) {
-  const auto& pname = p[0];
-  const R::Object obj = p[1];
-  const auto pool = lookup_pool(r, pname, y);
-
-  bs::error_code ec;
-  R::WriteOp op;
-  op.create(true);
-  r.execute(obj, pool, std::move(op), y[ec]);
-  if (ec)
-    throw bs::system_error(ec,
-			   fmt::format(
-			     "when creating object '{}' in pool '{}'",
-			     obj, pname));
-}
-
-inline constexpr std::size_t io_size = 4 << 20;
-
-void write(R::RADOS& r, const std::vector<std::string>& p, ba::yield_context y) {
-  const auto& pname = p[0];
-  const R::Object obj(p[1]);
-  const auto pool = lookup_pool(r, pname, y);
-
-  bs::error_code ec;
-  std::unique_ptr<char[]> buf = std::make_unique<char[]>(io_size);
-  std::size_t off = 0;
-  boost::io::ios_exception_saver ies(std::cin);
-
-  cin.exceptions(std::istream::badbit);
-  std::cin.clear();
-
-  while (!std::cin.eof()) {
-    auto curoff = off;
-    std::cin.read(buf.get(), io_size);
-    auto len = std::cin.gcount();
-    off += len;
-    if (len == 0)
-      break; // Nothin' to do.
-
-    ceph::buffer::list bl;
-    bl.append(buffer::create_static(len, buf.get()));
-    R::WriteOp op;
-    op.write(curoff, std::move(bl));
-    r.execute(obj, pool, std::move(op), y[ec]);
-
-    if (ec)
-      throw bs::system_error(ec, fmt::format(
-			       "when writing object '{}' in pool '{}'",
-			       obj, pname));
-  }
-}
-
-void read(R::RADOS& r, const std::vector<std::string>& p, ba::yield_context y) {
-  const auto& pname = p[0];
-  const R::Object obj(p[1]);
-  const auto pool = lookup_pool(r, pname, y);
-
-  bs::error_code ec;
-  std::uint64_t len;
-  {
-    R::ReadOp op;
-    op.stat(&len, nullptr);
-    r.execute(obj, pool, std::move(op),
-	      nullptr, y[ec]);
-    if (ec)
-      throw bs::system_error(
-	ec,
-	fmt::format("when getting length of object '{}' in pool '{}'",
-		    obj, pname));
-  }
-
-  std::size_t off = 0;
-  ceph::buffer::list bl;
-  while (auto toread = std::max(len - off, io_size)) {
-    R::ReadOp op;
-    op.read(off, toread, &bl);
-    r.execute(obj, pool, std::move(op), nullptr, y[ec]);
-    if (ec)
-      throw bs::system_error(
-	ec,
-	fmt::format("when reading from object '{}' in pool '{}'",
-		    obj, pool));
-
-    off += bl.length();
-    bl.write_stream(std::cout);
-    bl.clear();
-  }
-}
-
-void rm(R::RADOS& r, const std::vector<std::string>& p, ba::yield_context y) {
-  const auto& pname = p[0];
-  const R::Object obj = p[1];
-  const auto pool = lookup_pool(r, pname, y);
-
-  bs::error_code ec;
-  R::WriteOp op;
-  op.remove();
-  r.execute(obj, pool, std::move(op), y[ec]);
-  if (ec)
-    throw bs::system_error(ec, fmt::format(
-			     "when removing object '{}' in pool '{}'",
-			     obj, pname));
-}
-
-static constexpr auto version = std::make_tuple(0ul, 0ul, 1ul);
-
-using cmdfunc = void (*)(R::RADOS& r, const std::vector<std::string>& p,
-			 ba::yield_context);
-
-struct cmdesc {
-  std::string_view name;
-  std::size_t arity;
-  cmdfunc f;
-  std::string_view usage;
-  std::string_view desc;
-};
-
-const std::array commands = {
-  // Pools operations ;)
-
-  cmdesc{ "lspools"sv,
-	  0, &lspools,
-	  ""sv,
-	  "List all pools"sv },
-
-  // Pool operations
-
-  cmdesc{ "ls"sv,
-	  1, &ls,
-	  "POOL"sv,
-	  "list all objects in POOL"sv },
-  cmdesc{ "mkpool"sv,
-	  1, &mkpool,
-	  "POOL"sv,
-	  "create POOL"sv },
-  cmdesc{ "rmpool"sv,
-	  1, &rmpool,
-	  "POOL"sv,
-	  "remove POOL"sv },
-
-  // Object operations
-
-  cmdesc{ "create"sv,
-	  2, &create,
-	  "POOL OBJECT"sv,
-	  "exclusively create OBJECT in POOL"sv },
-  cmdesc{ "write"sv,
-	  2, &write,
-	  "POOL OBJECT"sv,
-	  "write to OBJECT in POOL from standard input"sv },
-  cmdesc{ "read"sv,
-	  2, &read,
-	  "POOL OBJECT"sv,
-	  "read contents of OBJECT in POOL to standard out"sv },
-  cmdesc{ "rm"sv,
-	  2, &rm,
-	  "POOL OBJECT"sv,
-	  "remove OBJECT in POOL"sv }
-};
-
-int main(int argc, char* argv[])
-{
-  const std::string_view prog(argv[0]);
-  std::string command;
-  namespace po = boost::program_options;
-  try {
-    std::vector<std::string> parameters;
-
-    po::options_description desc(fmt::format("{} options", prog));
-    desc.add_options()
-      ("help", "show help")
-      ("version", "show version")
-      ("command", po::value<std::string>(&command), "the operation to perform")
-      ("parameters", po::value<std::vector<std::string>>(&parameters),
-       "parameters to the command");
-
-    po::positional_options_description p;
-    p.add("command", 1);
-    p.add("parameters", -1);
-
-    po::variables_map vm;
-
-    po::store(po::command_line_parser(argc, argv).
-	      options(desc).positional(p).run(), vm);
-
-    po::notify(vm);
-
-    if (vm.count("help")) {
-      fmt::print("{}", desc);
-      fmt::print("\nCommands:\n");
-      for (const auto& cmd : commands) {
-	fmt::print("\t{} {}\n\t\t{}\n",
-		   cmd.name, cmd.usage, cmd.desc);
-      }
-      return 0;
-    }
-
-    if (vm.count("version")) {
-      fmt::print(
-	"{}: RADOS command exerciser, {},\n"
-	"RADOS library version {}\n"
-	"Copyright (C) 2019 Red Hat <contact@redhat.com>\n"
-	"This is free software; you can redistribute it and/or\n"
-	"modify it under the terms of the GNU Lesser General Public\n"
-	"License version 2.1, as published by the Free Software\n"
-	"Foundation.  See file COPYING.\n", prog,
-	verstr(version), verstr(R::RADOS::version()));
-      return 0;
-    }
-
-    if (vm.find("command") == vm.end()) {
-      fmt::print(std::cerr, "{}: A command is required\n", prog);
-      return 1;
-    }
-
-    ba::io_context c;
-
-    if (auto ci = std::find_if(commands.begin(), commands.end(),
-			       [&command](const cmdesc& c) {
-				 return c.name == command;
-			       }); ci != commands.end()) {
-      if (parameters.size() < ci->arity) {
-	fmt::print(std::cerr, "{}: {}: too few arguments\n\t{} {}\n",
-		   prog, command, ci->name, ci->usage);
-	return 1;
-      }
-      if (parameters.size() > ci->arity) {
-	fmt::print(std::cerr, "{}: {}: too many arguments\n\t{} {}\n",
-		   prog, command, ci->name, ci->usage);
-	return 1;
-      }
-      ba::spawn(c, [&](ba::yield_context y) {
-		     auto r = R::RADOS::Builder{}.build(c, y);
-		     ci->f(r, parameters, y);
-		   });
-    } else {
-      fmt::print(std::cerr, "{}: {}: unknown command\n", prog, command);
-      return 1;
-    }
-    c.run();
-  } catch (const std::exception& e) {
-    fmt::print(std::cerr, "{}: {}: {}\n", prog, command, e.what());
-    return 1;
-  }
-
-  return 0;
-}