Merge pull request #47771 from linuxbox2/wip-librgw-frontends

librgw: add ability to conditionally export HTTP frontends/apis

19 files changed, 1659 insertions, 1404 deletions

diff --git a/src/common/options/rgw.yaml.in b/src/common/options/rgw.yaml.in
index ca2db88ae29..a8cf381fe19 100644
--- a/src/common/options/rgw.yaml.in
+++ b/src/common/options/rgw.yaml.in
@@ -1148,6 +1148,24 @@ options:
   services:
   - rgw
   with_legacy: true
+- name: rgw_nfs_frontends
+  type: str
+  level: basic
+  desc: RGW frontends configuration when running as librgw/nfs
+  long_desc: A comma-delimited list of frontends configuration. Each configuration
+    contains the type of the frontend followed by an optional space delimited set
+    of key=value config parameters.
+  fmt_desc: Configures the HTTP frontend(s). The configuration for multiple
+    frontends can be provided in a comma-delimited list. Each frontend
+    configuration may include a list of options separated by spaces,
+    where each option is in the form "key=value" or "key". See
+    `HTTP Frontends`_ for more on supported options.
+  default: rgw-nfs
+  services:
+  - rgw
+  with_legacy: true
+  see_also:
+  - rgw_frontends
 - name: rgw_rados_pool_autoscale_bias
   type: float
   level: advanced
diff --git a/src/global/signal_handler.h b/src/global/signal_handler.h
index c7cb84a1066..bfdf0464413 100644
--- a/src/global/signal_handler.h
+++ b/src/global/signal_handler.h
@@ -22,7 +22,7 @@
 
 typedef void (*signal_handler_t)(int);
 namespace ceph {
-  struct BackTrace;
+struct BackTrace;
 }
 
 #if defined(HAVE_SIGDESCR_NP)
diff --git a/src/rgw/CMakeLists.txt b/src/rgw/CMakeLists.txt
index 6206db6ff18..f4460802827 100644
--- a/src/rgw/CMakeLists.txt
+++ b/src/rgw/CMakeLists.txt
@@ -276,25 +276,41 @@ if(WITH_RADOSGW_MOTR)
 endif()
 
 set(rgw_a_srcs
+  rgw_appmain.cc
+  rgw_asio_client.cc
+  rgw_asio_frontend.cc
   rgw_auth_keystone.cc
   rgw_client_io.cc
+  rgw_file.cc
   rgw_frontend.cc
   rgw_http_client_curl.cc
+  rgw_kmip_client_impl.cc
+  rgw_lib.cc
   rgw_loadgen.cc
+  rgw_loadgen_process.cc
+  rgw_log.cc
+  rgw_lua_request.cc
+  rgw_opa.cc
+  rgw_os_lib.cc
   rgw_period_pusher.cc
+  rgw_process.cc
   rgw_realm_reloader.cc
   rgw_realm_watcher.cc
-  rgw_os_lib.cc
-  rgw_process.cc
+  rgw_rest_bucket.cc
+  rgw_rest_config.cc
+  rgw_rest_info.cc
+  rgw_rest_log.cc
+  rgw_rest_metadata.cc
+  rgw_rest_ratelimit.cc
+  rgw_rest_realm.cc
+  rgw_rest_sts.cc
   rgw_rest_swift.cc
   rgw_rest_usage.cc
-  rgw_rest_info.cc
   rgw_rest_user.cc
+  rgw_signal.cc
   rgw_swift_auth.cc
   rgw_usage.cc
-  rgw_opa.cc
-  rgw_sts.cc
-  rgw_rest_sts.cc)
+  rgw_sts.cc)
 
 gperf_generate(${CMAKE_SOURCE_DIR}/src/rgw/rgw_iam_policy_keywords.gperf
   rgw_iam_policy_keywords.frag.cc)
@@ -308,6 +324,7 @@ add_library(rgw_a STATIC
 
 target_compile_definitions(rgw_a PUBLIC "-DCLS_CLIENT_HIDE_IOCTX")
 target_include_directories(rgw_a PUBLIC "${CMAKE_SOURCE_DIR}/src/dmclock/support/src")
+target_include_directories(rgw_a PRIVATE "${CMAKE_SOURCE_DIR}/src/libkmip")
 target_include_directories(rgw_a SYSTEM PUBLIC "../rapidjson/include")
 target_include_directories(rgw_a PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw")
 
@@ -345,11 +362,7 @@ target_link_libraries(rgw_schedulers
   PUBLIC dmclock::dmclock spawn)
 
 set(radosgw_srcs
-  rgw_main.cc
-  rgw_loadgen_process.cc
-  rgw_asio_client.cc
-  rgw_asio_frontend.cc
-  rgw_kmip_client_impl.cc)
+  rgw_main.cc)
 
 add_executable(radosgw ${radosgw_srcs})
 
@@ -413,12 +426,23 @@ target_link_libraries(radosgw-object-expirer ${rgw_libs} librados
 install(TARGETS radosgw-object-expirer DESTINATION bin)
 
 set(librgw_srcs
-  librgw.cc
-  rgw_file.cc)
+  librgw.cc)
 add_library(rgw SHARED ${librgw_srcs})
+
+target_compile_definitions(rgw PUBLIC "-DCLS_CLIENT_HIDE_IOCTX")
+target_include_directories(rgw
+  PUBLIC "${CMAKE_SOURCE_DIR}/src/dmclock/support/src"
+  PRIVATE "${CMAKE_SOURCE_DIR}/src/libkmip"
+  PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw"
+  PRIVATE "${LUA_INCLUDE_DIR}")
+
+target_include_directories(rgw SYSTEM PUBLIC "../rapidjson/include")
+
 target_link_libraries(rgw
   PRIVATE
   ${rgw_libs}
+  rgw_schedulers
+  kmip
   librados
   cls_rgw_client
   cls_otp_client
@@ -429,7 +453,6 @@ target_link_libraries(rgw
   neorados_cls_fifo
   cls_version_client
   cls_user_client
-  global
   ${LIB_RESOLV}
   ${CURL_LIBRARIES}
   ${EXPAT_LIBRARIES}
diff --git a/src/rgw/librgw.cc b/src/rgw/librgw.cc
index d26cd49ec89..38e43a59da1 100644
--- a/src/rgw/librgw.cc
+++ b/src/rgw/librgw.cc
@@ -4,7 +4,7 @@
 /*
  * Ceph - scalable distributed file system
  *
- * Copyright (C) 2011 New Dream Network
+ * Copyright (C) 2022 New Dream Network
  *
  * This is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
@@ -13,53 +13,18 @@
  *
  */
 
-#include "include/compat.h"
 #include <sys/types.h>
 #include <string.h>
 #include <chrono>
 
-#include "include/types.h"
 #include "include/rados/librgw.h"
-#include "rgw/rgw_acl_s3.h"
-#include "rgw_acl.h"
 
 #include "include/str_list.h"
-#include "include/stringify.h"
-#include "global/signal_handler.h"
-#include "common/config.h"
-#include "common/errno.h"
-#include "common/Timer.h"
-#include "common/Throttle.h"
-#include "common/WorkQueue.h"
 #include "common/ceph_argparse.h"
 #include "common/ceph_context.h"
-#include "common/common_init.h"
 #include "common/dout.h"
 
-#include "rgw_resolve.h"
-#include "rgw_op.h"
-#include "rgw_rest.h"
-#include "rgw_frontend.h"
-#include "rgw_request.h"
-#include "rgw_process.h"
-#include "rgw_rest_user.h"
-#include "rgw_rest_s3.h"
-#include "rgw_os_lib.h"
-#include "rgw_auth.h"
-#include "rgw_auth_s3.h"
 #include "rgw_lib.h"
-#include "rgw_lib_frontend.h"
-#include "rgw_http_client.h"
-#include "rgw_http_client_curl.h"
-#include "rgw_perf_counters.h"
-#ifdef WITH_RADOSGW_AMQP_ENDPOINT
-#include "rgw_amqp.h"
-#endif
-#ifdef WITH_RADOSGW_KAFKA_ENDPOINT
-#include "rgw_kafka.h"
-#endif
-
-#include "services/svc_zone.h"
 
 #include <errno.h>
 #include <thread>
@@ -68,684 +33,14 @@
 
 #define dout_subsys ceph_subsys_rgw
 
-using namespace std;
-
-bool global_stop = false;
-
-static void handle_sigterm(int signum)
-{
-  dout(20) << __func__ << " SIGUSR1 ignored" << dendl;
-}
-
 namespace rgw {
 
-  using std::string;
-
-  static std::mutex librgw_mtx;
-
-  RGWLib rgwlib;
-
-  class C_InitTimeout : public Context {
-  public:
-    C_InitTimeout() {}
-    void finish(int r) override {
-      derr << "Initialization timeout, failed to initialize" << dendl;
-      exit(1);
-    }
-  };
-
-  void RGWLibProcess::checkpoint()
-  {
-    m_tp.drain(&req_wq);
-  }
-
-#define MIN_EXPIRE_S 120
-
-  void RGWLibProcess::run()
-  {
-    /* write completion interval */
-    RGWLibFS::write_completion_interval_s =
-      cct->_conf->rgw_nfs_write_completion_interval_s;
-
-    /* start write timer */
-    RGWLibFS::write_timer.resume();
-
-    /* gc loop */
-    while (! shutdown) {
-      lsubdout(cct, rgw, 5) << "RGWLibProcess GC" << dendl;
-
-      /* dirent invalidate timeout--basically, the upper-bound on
-       * inconsistency with the S3 namespace */
-      auto expire_s = cct->_conf->rgw_nfs_namespace_expire_secs;
-
-      /* delay between gc cycles */
-      auto delay_s = std::max(int64_t(1), std::min(int64_t(MIN_EXPIRE_S), expire_s/2));
-
-      unique_lock uniq(mtx);
-    restart:
-      int cur_gen = gen;
-      for (auto iter = mounted_fs.begin(); iter != mounted_fs.end();
-	   ++iter) {
-	RGWLibFS* fs = iter->first->ref();
-	uniq.unlock();
-	fs->gc();
-        const DoutPrefix dp(cct, dout_subsys, "librgw: ");
-	fs->update_user(&dp);
-	fs->rele();
-	uniq.lock();
-	if (cur_gen != gen)
-	  goto restart; /* invalidated */
-      }
-      cv.wait_for(uniq, std::chrono::seconds(delay_s));
-      uniq.unlock();
-    }
-  }
-
-  void RGWLibProcess::handle_request(const DoutPrefixProvider *dpp, RGWRequest* r)
-  {
-    /*
-     * invariant: valid requests are derived from RGWLibRequst
-     */
-    RGWLibRequest* req = static_cast<RGWLibRequest*>(r);
-
-    // XXX move RGWLibIO and timing setup into process_request
-
-#if 0 /* XXX */
-    utime_t tm = ceph_clock_now();
-#endif
-
-    RGWLibIO io_ctx;
-
-    int ret = process_request(req, &io_ctx);
-    if (ret < 0) {
-      /* we don't really care about return code */
-      dout(20) << "process_request() returned " << ret << dendl;
-
-    }
-    delete req;
-  } /* handle_request */
-
-  int RGWLibProcess::process_request(RGWLibRequest* req)
-  {
-    // XXX move RGWLibIO and timing setup into process_request
-
-#if 0 /* XXX */
-    utime_t tm = ceph_clock_now();
-#endif
-
-    RGWLibIO io_ctx;
-
-    int ret = process_request(req, &io_ctx);
-    if (ret < 0) {
-      /* we don't really care about return code */
-      dout(20) << "process_request() returned " << ret << dendl;
-    }
-    return ret;
-  } /* process_request */
-
-  static inline void abort_req(req_state *s, RGWOp *op, int err_no)
-  {
-    if (!s)
-      return;
-
-    /* XXX the dump_errno and dump_bucket_from_state behaviors in
-     * the abort_early (rgw_rest.cc) might be valuable, but aren't
-     * safe to call presently as they return HTTP data */
-
-    perfcounter->inc(l_rgw_failed_req);
-  } /* abort_req */
-
-  int RGWLibProcess::process_request(RGWLibRequest* req, RGWLibIO* io)
-  {
-    int ret = 0;
-    bool should_log = true; // XXX
-
-    dout(1) << "====== " << __func__
-	    << " starting new request req=" << hex << req << dec
-	    << " ======" << dendl;
-
-    /*
-     * invariant: valid requests are derived from RGWOp--well-formed
-     * requests should have assigned RGWRequest::op in their descendant
-     * constructor--if not, the compiler can find it, at the cost of
-     * a runtime check
-     */
-    RGWOp *op = (req->op) ? req->op : dynamic_cast<RGWOp*>(req);
-    if (! op) {
-      ldpp_dout(op, 1) << "failed to derive cognate RGWOp (invalid op?)" << dendl;
-      return -EINVAL;
-    }
-
-    io->init(req->cct);
-
-    perfcounter->inc(l_rgw_req);
-
-    RGWEnv& rgw_env = io->get_env();
-
-    /* XXX
-     * until major refactoring of req_state and req_info, we need
-     * to build their RGWEnv boilerplate from the RGWLibRequest,
-     * pre-staging any strings (HTTP_HOST) that provoke a crash when
-     * not found
-     */
-
-    /* XXX for now, use "";  could be a legit hostname, or, in future,
-     * perhaps a tenant (Yehuda) */
-    rgw_env.set("HTTP_HOST", "");
-
-    /* XXX and -then- bloat up req_state with string copies from it */
-    req_state rstate(req->cct, &rgw_env, req->id);
-    req_state *s = &rstate;
-
-    // XXX fix this
-    s->cio = io;
-
-    /* XXX and -then- stash req_state pointers everywhere they are needed */
-    ret = req->init(rgw_env, store, io, s);
-    if (ret < 0) {
-      ldpp_dout(op, 10) << "failed to initialize request" << dendl;
-      abort_req(s, op, ret);
-      goto done;
-    }
-
-    /* req is-a RGWOp, currently initialized separately */
-    ret = req->op_init();
-    if (ret < 0) {
-      dout(10) << "failed to initialize RGWOp" << dendl;
-      abort_req(s, op, ret);
-      goto done;
-    }
-
-    /* now expected by rgw_log_op() */
-    rgw_env.set("REQUEST_METHOD", s->info.method);
-    rgw_env.set("REQUEST_URI", s->info.request_uri);
-    rgw_env.set("QUERY_STRING", "");
-
-    try {
-      /* XXX authorize does less here then in the REST path, e.g.,
-       * the user's info is cached, but still incomplete */
-      ldpp_dout(s, 2) << "authorizing" << dendl;
-      ret = req->authorize(op, null_yield);
-      if (ret < 0) {
-	dout(10) << "failed to authorize request" << dendl;
-	abort_req(s, op, ret);
-	goto done;
-      }
-
-      /* FIXME: remove this after switching all handlers to the new
-       * authentication infrastructure. */
-      if (! s->auth.identity) {
-	s->auth.identity = rgw::auth::transform_old_authinfo(s);
-      }
-
-      ldpp_dout(s, 2) << "reading op permissions" << dendl;
-      ret = req->read_permissions(op, null_yield);
-      if (ret < 0) {
-	abort_req(s, op, ret);
-	goto done;
-      }
-
-      ldpp_dout(s, 2) << "init op" << dendl;
-      ret = op->init_processing(null_yield);
-      if (ret < 0) {
-	abort_req(s, op, ret);
-	goto done;
-      }
-
-      ldpp_dout(s, 2) << "verifying op mask" << dendl;
-      ret = op->verify_op_mask();
-      if (ret < 0) {
-	abort_req(s, op, ret);
-	goto done;
-      }
-
-      ldpp_dout(s, 2) << "verifying op permissions" << dendl;
-      ret = op->verify_permission(null_yield);
-      if (ret < 0) {
-	if (s->system_request) {
-	  ldpp_dout(op, 2) << "overriding permissions due to system operation" << dendl;
-	} else if (s->auth.identity->is_admin_of(s->user->get_id())) {
-	  ldpp_dout(op, 2) << "overriding permissions due to admin operation" << dendl;
-	} else {
-	  abort_req(s, op, ret);
-	  goto done;
-	}
-      }
-
-      ldpp_dout(s, 2) << "verifying op params" << dendl;
-      ret = op->verify_params();
-      if (ret < 0) {
-	abort_req(s, op, ret);
-	goto done;
-      }
-
-      ldpp_dout(s, 2) << "executing" << dendl;
-      op->pre_exec();
-      op->execute(null_yield);
-      op->complete();
-
-    } catch (const ceph::crypto::DigestException& e) {
-      dout(0) << "authentication failed" << e.what() << dendl;
-      abort_req(s, op, -ERR_INVALID_SECRET_KEY);
-    }
-
-  done:
-    try {
-      io->complete_request();
-    } catch (rgw::io::Exception& e) {
-      dout(0) << "ERROR: io->complete_request() returned "
-              << e.what() << dendl;
-    }
-    if (should_log) {
-      rgw_log_op(nullptr /* !rest */, s, (op ? op->name() : "unknown"), olog);
-    }
-
-    int http_ret = s->err.http_ret;
-
-    ldpp_dout(s, 2) << "http status=" << http_ret << dendl;
-
-    ldpp_dout(op, 1) << "====== " << __func__
-	    << " req done req=" << hex << req << dec << " http_status="
-	    << http_ret
-	    << " ======" << dendl;
-
-    return (ret < 0 ? ret : s->err.ret);
-  } /* process_request */
-
-  int RGWLibProcess::start_request(RGWLibContinuedReq* req)
-  {
-
-    dout(1) << "====== " << __func__
-	    << " starting new continued request req=" << hex << req << dec
-	    << " ======" << dendl;
-
-    /*
-     * invariant: valid requests are derived from RGWOp--well-formed
-     * requests should have assigned RGWRequest::op in their descendant
-     * constructor--if not, the compiler can find it, at the cost of
-     * a runtime check
-     */
-    RGWOp *op = (req->op) ? req->op : dynamic_cast<RGWOp*>(req);
-    if (! op) {
-      ldpp_dout(op, 1) << "failed to derive cognate RGWOp (invalid op?)" << dendl;
-      return -EINVAL;
-    }
-
-    req_state* s = req->get_state();
-    RGWLibIO& io_ctx = req->get_io();
-    RGWEnv& rgw_env = io_ctx.get_env();
-
-    rgw_env.set("HTTP_HOST", "");
-
-    int ret = req->init(rgw_env, store, &io_ctx, s);
-    if (ret < 0) {
-      ldpp_dout(op, 10) << "failed to initialize request" << dendl;
-      abort_req(s, op, ret);
-      goto done;
-    }
-
-    /* req is-a RGWOp, currently initialized separately */
-    ret = req->op_init();
-    if (ret < 0) {
-      dout(10) << "failed to initialize RGWOp" << dendl;
-      abort_req(s, op, ret);
-      goto done;
-    }
-
-    /* XXX authorize does less here then in the REST path, e.g.,
-     * the user's info is cached, but still incomplete */
-    ldpp_dout(s, 2) << "authorizing" << dendl;
-    ret = req->authorize(op, null_yield);
-    if (ret < 0) {
-      dout(10) << "failed to authorize request" << dendl;
-      abort_req(s, op, ret);
-      goto done;
-    }
-
-    /* FIXME: remove this after switching all handlers to the new authentication
-     * infrastructure. */
-    if (! s->auth.identity) {
-      s->auth.identity = rgw::auth::transform_old_authinfo(s);
-    }
-
-    ldpp_dout(s, 2) << "reading op permissions" << dendl;
-    ret = req->read_permissions(op, null_yield);
-    if (ret < 0) {
-      abort_req(s, op, ret);
-      goto done;
-    }
-
-    ldpp_dout(s, 2) << "init op" << dendl;
-    ret = op->init_processing(null_yield);
-    if (ret < 0) {
-      abort_req(s, op, ret);
-      goto done;
-    }
-
-    ldpp_dout(s, 2) << "verifying op mask" << dendl;
-    ret = op->verify_op_mask();
-    if (ret < 0) {
-      abort_req(s, op, ret);
-      goto done;
-    }
-
-    ldpp_dout(s, 2) << "verifying op permissions" << dendl;
-    ret = op->verify_permission(null_yield);
-    if (ret < 0) {
-      if (s->system_request) {
-	ldpp_dout(op, 2) << "overriding permissions due to system operation" << dendl;
-      } else if (s->auth.identity->is_admin_of(s->user->get_id())) {
-	ldpp_dout(op, 2) << "overriding permissions due to admin operation" << dendl;
-      } else {
-	abort_req(s, op, ret);
-	goto done;
-      }
-    }
-
-    ldpp_dout(s, 2) << "verifying op params" << dendl;
-    ret = op->verify_params();
-    if (ret < 0) {
-      abort_req(s, op, ret);
-      goto done;
-    }
-
-    op->pre_exec();
-    req->exec_start();
-
-  done:
-    return (ret < 0 ? ret : s->err.ret);
-  }
-
-  int RGWLibProcess::finish_request(RGWLibContinuedReq* req)
-  {
-    RGWOp *op = (req->op) ? req->op : dynamic_cast<RGWOp*>(req);
-    if (! op) {
-      ldpp_dout(op, 1) << "failed to derive cognate RGWOp (invalid op?)" << dendl;
-      return -EINVAL;
-    }
-
-    int ret = req->exec_finish();
-    int op_ret = op->get_ret();
-
-    ldpp_dout(op, 1) << "====== " << __func__
-	    << " finishing continued request req=" << hex << req << dec
-	    << " op status=" << op_ret
-	    << " ======" << dendl;
-
-    perfcounter->inc(l_rgw_req);
-
-    return ret;
-  }
-
-  int RGWLibFrontend::init()
-  {
-    pprocess = new RGWLibProcess(g_ceph_context, &env,
-				 g_conf()->rgw_thread_pool_size, conf);
-    return 0;
-  }
-
-  int RGWLib::init()
-  {
-    vector<const char*> args;
-    return init(args);
-  }
-
-  int RGWLib::init(vector<const char*>& args)
-  {
-    int r = 0;
-
-    /* alternative default for module */
-    map<string,string> defaults = {
-      { "debug_rgw", "1/5" },
-      { "keyring", "$rgw_data/keyring" },
-      { "log_file", "/var/log/radosgw/$cluster-$name.log" }
-    };
-
-    cct = rgw_global_init(&defaults, args,
-			  CEPH_ENTITY_TYPE_CLIENT,
-			  CODE_ENVIRONMENT_DAEMON,
-			  CINIT_FLAG_UNPRIVILEGED_DAEMON_DEFAULTS);
-
-    ceph::mutex mutex = ceph::make_mutex("main");
-    SafeTimer init_timer(g_ceph_context, mutex);
-    init_timer.init();
-    mutex.lock();
-    init_timer.add_event_after(g_conf()->rgw_init_timeout, new C_InitTimeout);
-    mutex.unlock();
-
-    common_init_finish(g_ceph_context);
-
-    rgw_tools_init(this, g_ceph_context);
-
-    rgw_init_resolver();
-    rgw::curl::setup_curl(boost::none);
-    rgw_http_client_init(g_ceph_context);
-
-    auto run_gc =
-      g_conf()->rgw_enable_gc_threads &&
-      g_conf()->rgw_nfs_run_gc_threads;
-
-    auto run_lc =
-      g_conf()->rgw_enable_lc_threads &&
-      g_conf()->rgw_nfs_run_lc_threads;
-
-    auto run_quota =
-      g_conf()->rgw_enable_quota_threads &&
-      g_conf()->rgw_nfs_run_quota_threads;
-
-    auto run_sync =
-      g_conf()->rgw_run_sync_thread &&
-      g_conf()->rgw_nfs_run_sync_thread;
-
-    StoreManager::Config cfg = StoreManager::get_config(false, g_ceph_context);
-    store = StoreManager::get_storage(this, g_ceph_context,
-					 cfg,
-					 run_gc,
-					 run_lc,
-					 run_quota,
-					 run_sync,
-					 g_conf().get_val<bool>("rgw_dynamic_resharding"));
-
-    if (!store) {
-      mutex.lock();
-      init_timer.cancel_all_events();
-      init_timer.shutdown();
-      mutex.unlock();
-
-      derr << "Couldn't init storage provider (RADOS)" << dendl;
-      return -EIO;
-    }
-
-    r = rgw_perf_start(g_ceph_context);
-
-    rgw_rest_init(g_ceph_context, store->get_zone()->get_zonegroup());
-
-    mutex.lock();
-    init_timer.cancel_all_events();
-    init_timer.shutdown();
-    mutex.unlock();
-
-    if (r)
-      return -EIO;
-
-    const string& ldap_uri = store->ctx()->_conf->rgw_ldap_uri;
-    const string& ldap_binddn = store->ctx()->_conf->rgw_ldap_binddn;
-    const string& ldap_searchdn = store->ctx()->_conf->rgw_ldap_searchdn;
-    const string& ldap_searchfilter = store->ctx()->_conf->rgw_ldap_searchfilter;
-    const string& ldap_dnattr =
-      store->ctx()->_conf->rgw_ldap_dnattr;
-    std::string ldap_bindpw = parse_rgw_ldap_bindpw(store->ctx());
-
-    ldh = new rgw::LDAPHelper(ldap_uri, ldap_binddn, ldap_bindpw.c_str(),
-			      ldap_searchdn, ldap_searchfilter, ldap_dnattr);
-    ldh->init();
-    ldh->bind();
-
-    rgw_log_usage_init(g_ceph_context, store);
-
-    // XXX ex-RGWRESTMgr_lib, mgr->set_logging(true)
-
-    OpsLogManifold* olog_manifold = new OpsLogManifold();
-    if (!g_conf()->rgw_ops_log_socket_path.empty()) {
-      OpsLogSocket* olog_socket = new OpsLogSocket(g_ceph_context, g_conf()->rgw_ops_log_data_backlog);
-      olog_socket->init(g_conf()->rgw_ops_log_socket_path);
-      olog_manifold->add_sink(olog_socket);
-    }
-    OpsLogFile* ops_log_file;
-    if (!g_conf()->rgw_ops_log_file_path.empty()) {
-      ops_log_file = new OpsLogFile(g_ceph_context, g_conf()->rgw_ops_log_file_path, g_conf()->rgw_ops_log_data_backlog);
-      ops_log_file->start();
-      olog_manifold->add_sink(ops_log_file);
-    }
-    olog_manifold->add_sink(new OpsLogRados(store));
-    olog = olog_manifold;
-
-    int port = 80;
-    RGWProcessEnv env = { store, &rest, olog, port };
-
-    string fe_count{"0"};
-    fec = new RGWFrontendConfig("rgwlib");
-    fe = new RGWLibFrontend(env, fec);
-
-    init_async_signal_handler();
-    register_async_signal_handler(SIGUSR1, handle_sigterm);
-
-    map<string, string> service_map_meta;
-    service_map_meta["pid"] = stringify(getpid());
-    service_map_meta["frontend_type#" + fe_count] = "rgw-nfs";
-    service_map_meta["frontend_config#" + fe_count] = fec->get_config();
-
-    fe->init();
-    if (r < 0) {
-      derr << "ERROR: failed initializing frontend" << dendl;
-      return r;
-    }
-
-    fe->run();
-
-    r = store->register_to_service_map(this, "rgw-nfs", service_map_meta);
-    if (r < 0) {
-      derr << "ERROR: failed to register to service map: " << cpp_strerror(-r) << dendl;
-      /* ignore error */
-    }
-
-#ifdef WITH_RADOSGW_AMQP_ENDPOINT
-    if (!rgw::amqp::init(cct.get())) {
-      derr << "ERROR: failed to initialize AMQP manager" << dendl;
-    }
-#endif
-#ifdef WITH_RADOSGW_KAFKA_ENDPOINT
-    if (!rgw::kafka::init(cct.get())) {
-      derr << "ERROR: failed to initialize Kafka manager" << dendl;
-    }
-#endif
-
-    return 0;
-  } /* RGWLib::init() */
-
-  int RGWLib::stop()
-  {
-    derr << "shutting down" << dendl;
-
-    fe->stop();
-
-    fe->join();
-
-    delete fe;
-    delete fec;
-    delete ldh;
-
-    unregister_async_signal_handler(SIGUSR1, handle_sigterm);
-    shutdown_async_signal_handler();
-
-    rgw_log_usage_finalize();
-    
-    delete olog;
-
-    StoreManager::close_storage(store);
-
-    rgw_tools_cleanup();
-    rgw_shutdown_resolver();
-    rgw_http_client_cleanup();
-    rgw::curl::cleanup_curl();
-#ifdef WITH_RADOSGW_AMQP_ENDPOINT
-    rgw::amqp::shutdown();
-#endif
-#ifdef WITH_RADOSGW_KAFKA_ENDPOINT
-    rgw::kafka::shutdown();
-#endif
-
-    rgw_perf_stop(g_ceph_context);
-
-    dout(1) << "final shutdown" << dendl;
-    cct.reset();
-
-    return 0;
-  } /* RGWLib::stop() */
-
-  int RGWLibIO::set_uid(rgw::sal::Store* store, const rgw_user& uid)
-  {
-    const DoutPrefix dp(store->ctx(), dout_subsys, "librgw: ");
-    std::unique_ptr<rgw::sal::User> user = store->get_user(uid);
-    /* object exists, but policy is broken */
-    int ret = user->load_user(&dp, null_yield);
-    if (ret < 0) {
-      derr << "ERROR: failed reading user info: uid=" << uid << " ret="
-	   << ret << dendl;
-    }
-    user_info = user->get_info();
-    return ret;
-  }
-
-  int RGWLibRequest::read_permissions(RGWOp* op, optional_yield y) {
-    /* bucket and object ops */
-    int ret =
-      rgw_build_bucket_policies(op, rgwlib.get_store(), get_state(), y);
-    if (ret < 0) {
-      ldpp_dout(op, 10) << "read_permissions (bucket policy) on "
-				  << get_state()->bucket << ":"
-				  << get_state()->object
-				  << " only_bucket=" << only_bucket()
-				  << " ret=" << ret << dendl;
-      if (ret == -ENODATA)
-	ret = -EACCES;
-    } else if (! only_bucket()) {
-      /* object ops */
-      ret = rgw_build_object_policies(op, rgwlib.get_store(), get_state(),
-				      op->prefetch_data(), y);
-      if (ret < 0) {
-	ldpp_dout(op, 10) << "read_permissions (object policy) on"
-				    << get_state()->bucket << ":"
-				    << get_state()->object
-				    << " ret=" << ret << dendl;
-	if (ret == -ENODATA)
-	  ret = -EACCES;
-      }
-    }
-    return ret;
-  } /* RGWLibRequest::read_permissions */
-
-  int RGWHandler_Lib::authorize(const DoutPrefixProvider *dpp, optional_yield y)
-  {
-    /* TODO: handle
-     *  1. subusers
-     *  2. anonymous access
-     *  3. system access
-     *  4. ?
-     *
-     *  Much or all of this depends on handling the cached authorization
-     *  correctly (e.g., dealing with keystone) at mount time.
-     */
-    s->perm_mask = RGW_PERM_FULL_CONTROL;
-
-    // populate the owner info
-    s->owner.set_id(s->user->get_id());
-    s->owner.set_name(s->user->get_display_name());
+bool global_stop = false;
 
-    return 0;
-  } /* RGWHandler_Lib::authorize */
+static std::mutex librgw_mtx;
+static RGWLib rgwlib;
 
-} /* namespace rgw */
+} // namespace rgw
 
 extern "C" {
 
@@ -755,6 +50,8 @@ int librgw_create(librgw_t* rgw, int argc, char **argv)
 
   int rc = -EINVAL;
 
+  g_rgwlib = &rgwlib;
+
   if (! g_ceph_context) {
     std::lock_guard<std::mutex> lg(librgw_mtx);
     if (! g_ceph_context) {
@@ -784,6 +81,9 @@ void librgw_shutdown(librgw_t rgw)
 
   CephContext* cct = static_cast<CephContext*>(rgw);
   rgwlib.stop();
+
+  dout(1) << "final shutdown" << dendl;
+
   cct->put();
 }
 
diff --git a/src/rgw/rgw_appmain.cc b/src/rgw/rgw_appmain.cc
new file mode 100644
index 00000000000..3f548e0e041
--- /dev/null
+++ b/src/rgw/rgw_appmain.cc
@@ -0,0 +1,608 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab ft=cpp
+
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2022 Red Hat, Inc
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include <boost/intrusive/list.hpp>
+#include "global/global_init.h"
+#include "global/signal_handler.h"
+#include "common/config.h"
+#include "common/errno.h"
+#include "common/Timer.h"
+#include "common/TracepointProvider.h"
+#include "common/openssl_opts_handler.h"
+#include "common/numa.h"
+#include "include/compat.h"
+#include "include/str_list.h"
+#include "include/stringify.h"
+#include "rgw_main.h"
+#include "rgw_common.h"
+#include "rgw_sal_rados.h"
+#include "rgw_period_pusher.h"
+#include "rgw_realm_reloader.h"
+#include "rgw_rest.h"
+#include "rgw_rest_s3.h"
+#include "rgw_rest_swift.h"
+#include "rgw_rest_admin.h"
+#include "rgw_rest_info.h"
+#include "rgw_rest_usage.h"
+#include "rgw_rest_user.h"
+#include "rgw_rest_bucket.h"
+#include "rgw_rest_metadata.h"
+#include "rgw_rest_log.h"
+#include "rgw_rest_config.h"
+#include "rgw_rest_realm.h"
+#include "rgw_rest_ratelimit.h"
+#include "rgw_swift_auth.h"
+#include "rgw_log.h"
+#include "rgw_lib.h"
+#include "rgw_frontend.h"
+#include "rgw_lib_frontend.h"
+#include "rgw_tools.h"
+#include "rgw_resolve.h"
+#include "rgw_process.h"
+#include "rgw_frontend.h"
+#include "rgw_http_client_curl.h"
+#include "rgw_kmip_client.h"
+#include "rgw_kmip_client_impl.h"
+#include "rgw_perf_counters.h"
+#include "rgw_signal.h"
+#ifdef WITH_RADOSGW_AMQP_ENDPOINT
+#include "rgw_amqp.h"
+#endif
+#ifdef WITH_RADOSGW_KAFKA_ENDPOINT
+#include "rgw_kafka.h"
+#endif
+#include "rgw_asio_frontend.h"
+#include "rgw_dmclock_scheduler_ctx.h"
+#include "rgw_lua.h"
+#ifdef WITH_RADOSGW_DBSTORE
+#include "rgw_sal_dbstore.h"
+#endif
+#include "rgw_lua_background.h"
+#include "services/svc_zone.h"
+
+#ifdef HAVE_SYS_PRCTL_H
+#include <sys/prctl.h>
+#endif
+
+#define dout_subsys ceph_subsys_rgw
+
+using namespace std;
+
+namespace {
+  TracepointProvider::Traits rgw_op_tracepoint_traits(
+    "librgw_op_tp.so", "rgw_op_tracing");
+  TracepointProvider::Traits rgw_rados_tracepoint_traits(
+    "librgw_rados_tp.so", "rgw_rados_tracing");
+}
+
+OpsLogFile* rgw::AppMain::ops_log_file;
+
+void rgw::AppMain::init_frontends1(bool nfs) 
+{
+  this->nfs = nfs;
+  std::string fe_key = (nfs) ? "rgw_nfs_frontends" : "rgw_frontends";
+  std::vector<std::string> frontends;
+  std::string rgw_frontends_str = g_conf().get_val<string>(fe_key);
+  g_conf().early_expand_meta(rgw_frontends_str, &cerr);
+  get_str_vec(rgw_frontends_str, ",", frontends);
+
+  /* default frontends */
+  if (nfs) {
+    const auto is_rgw_nfs = [](const auto& s){return s == "rgw-nfs";};
+    if (std::find_if(frontends.begin(), frontends.end(), is_rgw_nfs) == frontends.end()) {
+      frontends.push_back("rgw-nfs");
+    }
+  } else {
+    if (frontends.empty()) {
+      frontends.push_back("beast");
+    }
+  }
+
+  for (auto &f : frontends) {
+    if (f.find("beast") != string::npos) {
+      have_http_frontend = true;
+      if (f.find("port") != string::npos) {
+        // check for the most common ws problems
+        if ((f.find("port=") == string::npos) ||
+            (f.find("port= ") != string::npos)) {
+          derr <<
+    R"(WARNING: radosgw frontend config found unexpected spacing around 'port'
+    (ensure frontend port parameter has the form 'port=80' with no spaces
+    before or after '='))"
+               << dendl;
+        }
+      }
+    } else {
+      if (f.find("civetweb") != string::npos) {
+        have_http_frontend = true;
+      }
+    } /* fe !beast */
+
+    RGWFrontendConfig *config = new RGWFrontendConfig(f);
+    int r = config->init();
+    if (r < 0) {
+      delete config;
+      cerr << "ERROR: failed to init config: " << f << std::endl;
+      continue;
+    }
+
+    fe_configs.push_back(config);
+    fe_map.insert(
+        pair<string, RGWFrontendConfig *>(config->get_framework(), config));
+  } /* for each frontend */
+
+  // maintain existing region root pool for new multisite objects
+  if (!g_conf()->rgw_region_root_pool.empty()) {
+    const char *root_pool = g_conf()->rgw_region_root_pool.c_str();
+    if (g_conf()->rgw_zonegroup_root_pool.empty()) {
+      g_conf().set_val_or_die("rgw_zonegroup_root_pool", root_pool);
+    }
+    if (g_conf()->rgw_period_root_pool.empty()) {
+      g_conf().set_val_or_die("rgw_period_root_pool", root_pool);
+    }
+    if (g_conf()->rgw_realm_root_pool.empty()) {
+      g_conf().set_val_or_die("rgw_realm_root_pool", root_pool);
+    }
+  }
+
+  // for region -> zonegroup conversion (must happen before
+  // common_init_finish())
+  if (!g_conf()->rgw_region.empty() && g_conf()->rgw_zonegroup.empty()) {
+    g_conf().set_val_or_die("rgw_zonegroup", g_conf()->rgw_region.c_str());
+  }
+
+  ceph::crypto::init_openssl_engine_once();
+} /* init_frontends1 */
+
+void rgw::AppMain::init_numa()
+{
+  if (nfs) {
+    return;
+  }
+
+  int numa_node = g_conf().get_val<int64_t>("rgw_numa_node");
+  size_t numa_cpu_set_size = 0;
+  cpu_set_t numa_cpu_set;
+
+  if (numa_node >= 0) {
+    int r = get_numa_node_cpu_set(numa_node, &numa_cpu_set_size, &numa_cpu_set);
+    if (r < 0) {
+      dout(1) << __func__ << " unable to determine rgw numa node " << numa_node
+              << " CPUs" << dendl;
+      numa_node = -1;
+    } else {
+      r = set_cpu_affinity_all_threads(numa_cpu_set_size, &numa_cpu_set);
+      if (r < 0) {
+        derr << __func__ << " failed to set numa affinity: " << cpp_strerror(r)
+        << dendl;
+      }
+    }
+  } else {
+    dout(1) << __func__ << " not setting numa affinity" << dendl;
+  }
+} /* init_numa */
+
+void rgw::AppMain::init_storage()
+{
+    auto run_gc =
+    (g_conf()->rgw_enable_gc_threads &&
+      ((!nfs) || (nfs && g_conf()->rgw_nfs_run_gc_threads)));
+
+  auto run_lc =
+    (g_conf()->rgw_enable_lc_threads &&
+      ((!nfs) || (nfs && g_conf()->rgw_nfs_run_lc_threads)));
+
+  auto run_quota =
+    (g_conf()->rgw_enable_quota_threads &&
+      ((!nfs) || (nfs && g_conf()->rgw_nfs_run_quota_threads)));
+
+  auto run_sync =
+    (g_conf()->rgw_run_sync_thread &&
+      ((!nfs) || (nfs && g_conf()->rgw_nfs_run_sync_thread)));
+
+  StoreManager::Config cfg = StoreManager::get_config(false, g_ceph_context);
+  store = StoreManager::get_storage(dpp, dpp->get_cct(),
+          cfg,
+          run_gc,
+          run_lc,
+          run_quota,
+          run_sync,
+          g_conf().get_val<bool>("rgw_dynamic_resharding"),
+          g_conf()->rgw_cache_enabled);
+
+} /* init_storage */
+
+void rgw::AppMain::init_perfcounters()
+{
+  (void) rgw_perf_start(dpp->get_cct());
+} /* init_perfcounters */
+
+void rgw::AppMain::init_http_clients()
+{
+  rgw_init_resolver();
+  rgw::curl::setup_curl(fe_map);
+  rgw_http_client_init(dpp->get_cct());
+  rgw_kmip_client_init(*new RGWKMIPManagerImpl(dpp->get_cct()));
+} /* init_http_clients */
+
+void rgw::AppMain::cond_init_apis() 
+{
+   rgw_rest_init(g_ceph_context, store->get_zone()->get_zonegroup());
+
+  if (have_http_frontend) {
+    std::vector<std::string> apis;
+    get_str_vec(g_conf()->rgw_enable_apis, apis);
+
+    std::map<std::string, bool> apis_map;
+    for (auto &api : apis) {
+      apis_map[api] = true;
+    }
+
+    /* warn about insecure keystone secret config options */
+    if (!(g_ceph_context->_conf->rgw_keystone_admin_token.empty() ||
+          g_ceph_context->_conf->rgw_keystone_admin_password.empty())) {
+      dout(0)
+          << "WARNING: rgw_keystone_admin_token and "
+             "rgw_keystone_admin_password should be avoided as they can "
+             "expose secrets.  Prefer the new rgw_keystone_admin_token_path "
+             "and rgw_keystone_admin_password_path options, which read their "
+             "secrets from files."
+          << dendl;
+    }
+
+    // S3 website mode is a specialization of S3
+    const bool s3website_enabled = apis_map.count("s3website") > 0;
+    const bool sts_enabled = apis_map.count("sts") > 0;
+    const bool iam_enabled = apis_map.count("iam") > 0;
+    const bool pubsub_enabled =
+        apis_map.count("pubsub") > 0 || apis_map.count("notifications") > 0;
+    // Swift API entrypoint could placed in the root instead of S3
+    const bool swift_at_root = g_conf()->rgw_swift_url_prefix == "/";
+    if (apis_map.count("s3") > 0 || s3website_enabled) {
+      if (!swift_at_root) {
+        rest.register_default_mgr(set_logging(
+            rest_filter(store, RGW_REST_S3,
+                        new RGWRESTMgr_S3(s3website_enabled, sts_enabled,
+                                          iam_enabled, pubsub_enabled))));
+      } else {
+        derr << "Cannot have the S3 or S3 Website enabled together with "
+             << "Swift API placed in the root of hierarchy" << dendl;
+      }
+    }
+
+    if (apis_map.count("swift") > 0) {
+      RGWRESTMgr_SWIFT* const swift_resource = new RGWRESTMgr_SWIFT;
+
+      if (! g_conf()->rgw_cross_domain_policy.empty()) {
+        swift_resource->register_resource("crossdomain.xml",
+                            set_logging(new RGWRESTMgr_SWIFT_CrossDomain));
+      }
+
+      swift_resource->register_resource("healthcheck",
+                            set_logging(new RGWRESTMgr_SWIFT_HealthCheck));
+
+      swift_resource->register_resource("info",
+                            set_logging(new RGWRESTMgr_SWIFT_Info));
+
+      if (! swift_at_root) {
+        rest.register_resource(g_conf()->rgw_swift_url_prefix,
+                            set_logging(rest_filter(store, RGW_REST_SWIFT,
+                                                    swift_resource)));
+      } else {
+        if (store->get_zone()->get_zonegroup().get_zone_count() > 1) {
+          derr << "Placing Swift API in the root of URL hierarchy while running"
+              << " multi-site configuration requires another instance of RadosGW"
+              << " with S3 API enabled!" << dendl;
+        }
+
+        rest.register_default_mgr(set_logging(swift_resource));
+      }
+    }
+
+    if (apis_map.count("swift_auth") > 0) {
+      rest.register_resource(g_conf()->rgw_swift_auth_entry,
+                set_logging(new RGWRESTMgr_SWIFT_Auth));
+    }
+
+    if (apis_map.count("admin") > 0) {
+      RGWRESTMgr_Admin *admin_resource = new RGWRESTMgr_Admin;
+      admin_resource->register_resource("info", new RGWRESTMgr_Info);
+      admin_resource->register_resource("usage", new RGWRESTMgr_Usage);
+      admin_resource->register_resource("user", new RGWRESTMgr_User);
+      /* XXX dang part of this is RADOS specific */
+      admin_resource->register_resource("bucket", new RGWRESTMgr_Bucket);
+    
+      /*Registering resource for /admin/metadata */
+      admin_resource->register_resource("metadata", new RGWRESTMgr_Metadata);
+      /* XXX dang ifdef these RADOS ? */
+      admin_resource->register_resource("log", new RGWRESTMgr_Log);
+      admin_resource->register_resource("config", new RGWRESTMgr_Config);
+      admin_resource->register_resource("realm", new RGWRESTMgr_Realm);
+      admin_resource->register_resource("ratelimit", new RGWRESTMgr_Ratelimit);
+      rest.register_resource(g_conf()->rgw_admin_entry, admin_resource);
+    }
+  } /* have_http_frontend */
+} /* init_apis */
+
+void rgw::AppMain::init_ldap()
+{
+  const string &ldap_uri = store->ctx()->_conf->rgw_ldap_uri;
+  const string &ldap_binddn = store->ctx()->_conf->rgw_ldap_binddn;
+  const string &ldap_searchdn = store->ctx()->_conf->rgw_ldap_searchdn;
+  const string &ldap_searchfilter = store->ctx()->_conf->rgw_ldap_searchfilter;
+  const string &ldap_dnattr = store->ctx()->_conf->rgw_ldap_dnattr;
+  std::string ldap_bindpw = parse_rgw_ldap_bindpw(store->ctx());
+
+  ldh.reset(new rgw::LDAPHelper(ldap_uri, ldap_binddn,
+            ldap_bindpw.c_str(), ldap_searchdn, ldap_searchfilter, ldap_dnattr));
+  ldh->init();
+  ldh->bind();
+} /* init_ldap */
+
+void rgw::AppMain::init_opslog()
+{
+  rgw_log_usage_init(dpp->get_cct(), store);
+
+  OpsLogManifold *olog_manifold = new OpsLogManifold();
+  if (!g_conf()->rgw_ops_log_socket_path.empty()) {
+    OpsLogSocket *olog_socket =
+        new OpsLogSocket(g_ceph_context, g_conf()->rgw_ops_log_data_backlog);
+    olog_socket->init(g_conf()->rgw_ops_log_socket_path);
+    olog_manifold->add_sink(olog_socket);
+  }
+  if (!g_conf()->rgw_ops_log_file_path.empty()) {
+    ops_log_file =
+        new OpsLogFile(g_ceph_context, g_conf()->rgw_ops_log_file_path,
+                       g_conf()->rgw_ops_log_data_backlog);
+    ops_log_file->start();
+    olog_manifold->add_sink(ops_log_file);
+  }
+  olog_manifold->add_sink(new OpsLogRados(store));
+  olog = olog_manifold;
+} /* init_opslog */
+
+int rgw::AppMain::init_frontends2(RGWLib* rgwlib)
+{
+  int r{0};
+  vector<string> frontends_def;
+  std::string frontend_defs_str =
+    g_conf().get_val<string>("rgw_frontend_defaults");
+  get_str_vec(frontend_defs_str, ",", frontends_def);
+
+  service_map_meta["pid"] = stringify(getpid());
+
+  std::map<std::string, std::unique_ptr<RGWFrontendConfig> > fe_def_map;
+  for (auto& f : frontends_def) {
+    RGWFrontendConfig *config = new RGWFrontendConfig(f);
+    int r = config->init();
+    if (r < 0) {
+      delete config;
+      cerr << "ERROR: failed to init default config: " << f << std::endl;
+      continue;
+    }
+    fe_def_map[config->get_framework()].reset(config);
+  }
+
+  /* Initialize the registry of auth strategies which will coordinate
+   * the dynamic reconfiguration. */
+  implicit_tenant_context.reset(new rgw::auth::ImplicitTenants{g_conf()});
+  g_conf().add_observer(implicit_tenant_context.get());
+  auto auth_registry =
+    rgw::auth::StrategyRegistry::create(dpp->get_cct(), *(implicit_tenant_context.get()), store);
+
+  /* allocate a mime table (you'd never guess that from the name) */
+  rgw_tools_init(dpp, dpp->get_cct());
+
+  /* Header custom behavior */
+  rest.register_x_headers(g_conf()->rgw_log_http_headers);
+
+  sched_ctx.reset(new rgw::dmclock::SchedulerCtx{dpp->get_cct()});
+  ratelimiter.reset(new ActiveRateLimiter{dpp->get_cct()});
+  ratelimiter->start();
+
+  int fe_count = 0;
+  for (multimap<string, RGWFrontendConfig *>::iterator fiter = fe_map.begin();
+       fiter != fe_map.end(); ++fiter, ++fe_count) {
+    RGWFrontendConfig *config = fiter->second;
+    string framework = config->get_framework();
+
+    auto def_iter = fe_def_map.find(framework);
+    if (def_iter != fe_def_map.end()) {
+      config->set_default_config(*def_iter->second);
+    }
+
+    RGWFrontend* fe = nullptr;
+
+    if (framework == "loadgen") {
+      int port;
+      config->get_val("port", 80, &port);
+      std::string uri_prefix;
+      config->get_val("prefix", "", &uri_prefix);
+
+      RGWProcessEnv env = {store, &rest, olog, port, uri_prefix,
+	    auth_registry, ratelimiter.get(), lua_background.get()};
+
+      fe = new RGWLoadGenFrontend(env, config);
+    }
+    else if (framework == "beast") {
+      int port;
+      config->get_val("port", 80, &port);
+      std::string uri_prefix;
+      config->get_val("prefix", "", &uri_prefix);
+      RGWProcessEnv env{store, &rest, olog, port, uri_prefix,
+	    auth_registry, ratelimiter.get(), lua_background.get()};
+      fe = new RGWAsioFrontend(env, config, *(sched_ctx.get()));
+    }
+    else if (framework == "rgw-nfs") {
+      int port = 80;
+      RGWProcessEnv env = { store, &rest, olog, port };
+      fe = new RGWLibFrontend(env, config);
+      if (rgwlib) {
+        rgwlib->set_fe(static_cast<RGWLibFrontend*>(fe));
+      }
+    }
+
+    service_map_meta["frontend_type#" + stringify(fe_count)] = framework;
+    service_map_meta["frontend_config#" + stringify(fe_count)] = config->get_config();
+
+    if (! fe) {
+      dout(0) << "WARNING: skipping unknown framework: " << framework << dendl;
+      continue;
+    }
+
+    dout(0) << "starting handler: " << fiter->first << dendl;
+    int r = fe->init();
+    if (r < 0) {
+      derr << "ERROR: failed initializing frontend" << dendl;
+      return -r;
+    }
+    r = fe->run();
+    if (r < 0) {
+      derr << "ERROR: failed run" << dendl;
+      return -r;
+    }
+
+    fes.push_back(fe);
+  }
+
+  std::string daemon_type = (nfs) ? "rgw-nfs" : "rgw";
+  r = store->register_to_service_map(dpp, daemon_type, service_map_meta);
+  if (r < 0) {
+    derr << "ERROR: failed to register to service map: " << cpp_strerror(-r) << dendl;
+    /* ignore error */
+  }
+
+  if (store->get_name() == "rados") {
+    // add a watcher to respond to realm configuration changes
+    pusher = std::make_unique<RGWPeriodPusher>(dpp, store, null_yield);
+    fe_pauser = std::make_unique<RGWFrontendPauser>(fes, *(implicit_tenant_context.get()), pusher.get());
+    rgw_pauser = std::make_unique<RGWPauser>();
+    rgw_pauser->add_pauser(fe_pauser.get());
+    if (lua_background) {
+      rgw_pauser->add_pauser(lua_background.get());
+    }
+    reloader = std::make_unique<RGWRealmReloader>(store, service_map_meta, rgw_pauser.get());
+    realm_watcher = std::make_unique<RGWRealmWatcher>(dpp, g_ceph_context,
+				  static_cast<rgw::sal::RadosStore*>(store)->svc()->zone->get_realm());
+    realm_watcher->add_watcher(RGWRealmNotify::Reload, *reloader);
+    realm_watcher->add_watcher(RGWRealmNotify::ZonesNeedPeriod, *pusher.get());
+  }
+
+  return r;
+} /* init_frontends2 */
+
+void rgw::AppMain::init_tracepoints()
+{
+  TracepointProvider::initialize<rgw_rados_tracepoint_traits>(dpp->get_cct());
+  TracepointProvider::initialize<rgw_op_tracepoint_traits>(dpp->get_cct());
+  tracing::rgw::tracer.init("rgw");
+} /* init_tracepoints() */
+
+void rgw::AppMain::init_notification_endpoints()
+{
+#ifdef WITH_RADOSGW_AMQP_ENDPOINT
+  if (!rgw::amqp::init(dpp->get_cct())) {
+    derr << "ERROR: failed to initialize AMQP manager" << dendl;
+  }
+#endif
+#ifdef WITH_RADOSGW_KAFKA_ENDPOINT
+  if (!rgw::kafka::init(dpp->get_cct())) {
+    derr << "ERROR: failed to initialize Kafka manager" << dendl;
+  }
+#endif
+} /* init_notification_endpoints */
+
+void rgw::AppMain::init_lua()
+{
+  int r{0};
+  const auto &luarocks_path =
+      g_conf().get_val<std::string>("rgw_luarocks_location");
+  if (luarocks_path.empty()) {
+    store->set_luarocks_path("");
+  } else {
+    store->set_luarocks_path(luarocks_path + "/" + g_conf()->name.to_str());
+  }
+#ifdef WITH_RADOSGW_LUA_PACKAGES
+  rgw::lua::packages_t failed_packages;
+  std::string output;
+  r = rgw::lua::install_packages(dpp, store, null_yield, failed_packages,
+                                 output);
+  if (r < 0) {
+    dout(1) << "WARNING: failed to install lua packages from allowlist"
+            << dendl;
+  }
+  if (!output.empty()) {
+    dout(10) << "INFO: lua packages installation output: \n" << output << dendl;
+  }
+  for (const auto &p : failed_packages) {
+    dout(5) << "WARNING: failed to install lua package: " << p
+            << " from allowlist" << dendl;
+  }
+#endif
+
+  if (store->get_name() == "rados") { /* Supported for only RadosStore */
+    lua_background = std::make_unique<
+      rgw::lua::Background>(store, dpp->get_cct(), store->get_luarocks_path());
+    lua_background->start();
+  }
+} /* init_lua */
+
+void rgw::AppMain::shutdown(std::function<void(void)> finalize_async_signals)
+{
+  if (store->get_name() == "rados") {
+    reloader.reset(); // stop the realm reloader
+  }
+
+  for (auto& fe : fes) {
+    fe->stop();
+  }
+
+  for (auto& fe : fes) {
+    fe->join();
+    delete fe;
+  }
+
+  for (auto& fec : fe_configs) {
+    delete fec;
+  }
+
+  ldh.reset(nullptr); // deletes
+  finalize_async_signals(); // callback
+  rgw_log_usage_finalize();
+  
+  delete olog;
+
+  if (lua_background) {
+    lua_background->shutdown();
+  }
+
+  StoreManager::close_storage(store);
+
+  rgw_tools_cleanup();
+  rgw_shutdown_resolver();
+  rgw_http_client_cleanup();
+  rgw_kmip_client_cleanup();
+  rgw::curl::cleanup_curl();
+  g_conf().remove_observer(implicit_tenant_context.get());
+  implicit_tenant_context.reset(); // deletes
+#ifdef WITH_RADOSGW_AMQP_ENDPOINT
+  rgw::amqp::shutdown();
+#endif
+#ifdef WITH_RADOSGW_KAFKA_ENDPOINT
+  rgw::kafka::shutdown();
+#endif
+  rgw_perf_stop(g_ceph_context);
+  ratelimiter.reset(); // deletes--ensure this happens before we destruct
+} /* AppMain::shutdown */
diff --git a/src/rgw/rgw_file.cc b/src/rgw/rgw_file.cc
index 500776bff1e..953cf98a199 100644
--- a/src/rgw/rgw_file.cc
+++ b/src/rgw/rgw_file.cc
@@ -39,8 +39,6 @@ using namespace rgw;
 
 namespace rgw {
 
-  extern RGWLib rgwlib;
-
   const string RGWFileHandle::root_name = "/";
 
   std::atomic<uint32_t> RGWLibFS::fs_inst_counter;
@@ -105,7 +103,7 @@ namespace rgw {
     std::string bucket_name{path};
     RGWStatBucketRequest req(cct, user->clone(), bucket_name, bs);
 
-    int rc = rgwlib.get_fe()->execute_req(&req);
+    int rc = g_rgwlib->get_fe()->execute_req(&req);
     if ((rc == 0) &&
 	(req.get_ret() == 0) &&
 	(req.matched())) {
@@ -201,7 +199,7 @@ namespace rgw {
 	RGWStatObjRequest req(cct, user->clone(),
 			      parent->bucket_name(), obj_path,
 			      RGWStatObjRequest::FLAG_NONE);
-	int rc = rgwlib.get_fe()->execute_req(&req);
+	int rc = g_rgwlib->get_fe()->execute_req(&req);
 	if ((rc == 0) &&
 	    (req.get_ret() == 0)) {
 	  fhr = lookup_fh(parent, path, RGWFileHandle::FLAG_CREATE);
@@ -238,7 +236,7 @@ namespace rgw {
 	RGWStatObjRequest req(cct, user->clone(),
 			      parent->bucket_name(), obj_path,
 			      RGWStatObjRequest::FLAG_NONE);
-	int rc = rgwlib.get_fe()->execute_req(&req);
+	int rc = g_rgwlib->get_fe()->execute_req(&req);
 	if ((rc == 0) &&
 	    (req.get_ret() == 0)) {
 	  fhr = lookup_fh(parent, path, RGWFileHandle::FLAG_DIRECTORY);
@@ -269,7 +267,7 @@ namespace rgw {
 	std::string object_name{path};
 	RGWStatLeafRequest req(cct, user->clone(),
 			       parent, object_name);
-	int rc = rgwlib.get_fe()->execute_req(&req);
+	int rc = g_rgwlib->get_fe()->execute_req(&req);
 	if ((rc == 0) &&
 	    (req.get_ret() == 0)) {
 	  if (req.matched) {
@@ -322,7 +320,7 @@ namespace rgw {
 
     RGWReadRequest req(get_context(), user->clone(), rgw_fh, offset, length, buffer);
 
-    int rc = rgwlib.get_fe()->execute_req(&req);
+    int rc = g_rgwlib->get_fe()->execute_req(&req);
     if ((rc == 0) &&
         ((rc = req.get_ret()) == 0)) {
       lock_guard guard(rgw_fh->mtx);
@@ -344,7 +342,7 @@ namespace rgw {
 
     RGWReadRequest req(get_context(), user->clone(), rgw_fh, offset, length, buffer);
 
-    int rc = rgwlib.get_fe()->execute_req(&req);
+    int rc = g_rgwlib->get_fe()->execute_req(&req);
     if ((rc == 0) &&
         ((rc = req.get_ret()) == 0)) {
       lock_guard(rgw_fh->mtx);
@@ -400,14 +398,14 @@ namespace rgw {
 	/* delete object w/key "<bucket>/" (uxattrs), if any */
 	string oname{"/"};
 	RGWDeleteObjRequest req(cct, user->clone(), bkt_fh->bucket_name(), oname);
-	rc = rgwlib.get_fe()->execute_req(&req);
+	rc = g_rgwlib->get_fe()->execute_req(&req);
 	/* don't care if ENOENT */
 	unref(bkt_fh);
       }
 
       string bname{name};
       RGWDeleteBucketRequest req(cct, user->clone(), bname);
-      rc = rgwlib.get_fe()->execute_req(&req);
+      rc = g_rgwlib->get_fe()->execute_req(&req);
       if (! rc) {
 	rc = req.get_ret();
       }
@@ -443,7 +441,7 @@ namespace rgw {
 	oname += "/";
       }
       RGWDeleteObjRequest req(cct, user->clone(), parent->bucket_name(), oname);
-      rc = rgwlib.get_fe()->execute_req(&req);
+      rc = g_rgwlib->get_fe()->execute_req(&req);
       if (! rc) {
 	rc = req.get_ret();
       }
@@ -520,7 +518,7 @@ namespace rgw {
       case 0:
       {
 	RGWCopyObjRequest req(cct, user->clone(), src_fh, dst_fh, src_name, dst_name);
-	int rc = rgwlib.get_fe()->execute_req(&req);
+	int rc = g_rgwlib->get_fe()->execute_req(&req);
 	if ((rc != 0) ||
 	    ((rc = req.get_ret()) != 0)) {
 	  ldout(get_context(), 1)
@@ -646,7 +644,7 @@ namespace rgw {
       req.emplace_attr(RGW_ATTR_UNIX_KEY1, std::move(ux_key));
       req.emplace_attr(RGW_ATTR_UNIX1, std::move(ux_attrs));
 
-      rc = rgwlib.get_fe()->execute_req(&req);
+      rc = g_rgwlib->get_fe()->execute_req(&req);
       rc2 = req.get_ret();
     } else {
       /* create an object representing the directory */
@@ -672,7 +670,7 @@ namespace rgw {
       req.emplace_attr(RGW_ATTR_UNIX_KEY1, std::move(ux_key));
       req.emplace_attr(RGW_ATTR_UNIX1, std::move(ux_attrs));
 
-      rc = rgwlib.get_fe()->execute_req(&req);
+      rc = g_rgwlib->get_fe()->execute_req(&req);
       rc2 = req.get_ret();
     }
 
@@ -728,7 +726,7 @@ namespace rgw {
     RGWPutObjRequest req(cct, user->clone(), parent->bucket_name(), obj_name, bl);
     MkObjResult mkr{nullptr, -EINVAL};
 
-    rc = rgwlib.get_fe()->execute_req(&req);
+    rc = g_rgwlib->get_fe()->execute_req(&req);
     rc2 = req.get_ret();
 
     if ((rc == 0) &&
@@ -843,7 +841,7 @@ namespace rgw {
     req.emplace_attr(RGW_ATTR_UNIX_KEY1, std::move(ux_key));
     req.emplace_attr(RGW_ATTR_UNIX1, std::move(ux_attrs));
 
-    rc = rgwlib.get_fe()->execute_req(&req);
+    rc = g_rgwlib->get_fe()->execute_req(&req);
     rc2 = req.get_ret();
     if (! ((rc == 0) &&
         (rc2 == 0))) {
@@ -922,7 +920,7 @@ namespace rgw {
     req.emplace_attr(RGW_ATTR_ETAG, std::move(etag));
     req.emplace_attr(RGW_ATTR_ACL, std::move(acls));
 
-    rc = rgwlib.get_fe()->execute_req(&req);
+    rc = g_rgwlib->get_fe()->execute_req(&req);
     rc2 = req.get_ret();
 
     if (rc == -ENOENT) {
@@ -936,7 +934,7 @@ namespace rgw {
       req.emplace_attr(RGW_ATTR_UNIX_KEY1, std::move(ux_key));
       req.emplace_attr(RGW_ATTR_UNIX1, std::move(ux_attrs));
 
-      rc = rgwlib.get_fe()->execute_req(&req);
+      rc = g_rgwlib->get_fe()->execute_req(&req);
       rc2 = req.get_ret();
     }
 
@@ -1008,7 +1006,7 @@ namespace rgw {
       }
     }
 
-    rc = rgwlib.get_fe()->execute_req(&req);
+    rc = g_rgwlib->get_fe()->execute_req(&req);
     rc2 = req.get_ret();
     rc3 = ((rc == 0) && (rc2 == 0)) ? 0 : -EIO;
 
@@ -1067,7 +1065,7 @@ namespace rgw {
 
     RGWGetAttrsRequest req(cct, user->clone(), rgw_fh->bucket_name(), obj_name);
 
-    rc = rgwlib.get_fe()->execute_req(&req);
+    rc = g_rgwlib->get_fe()->execute_req(&req);
     rc2 = req.get_ret();
     rc3 = ((rc == 0) && (rc2 == 0)) ? 0 : -EIO;
 
@@ -1137,7 +1135,7 @@ namespace rgw {
       return -EINVAL;
     }
 
-    rc = rgwlib.get_fe()->execute_req(&req);
+    rc = g_rgwlib->get_fe()->execute_req(&req);
     rc2 = req.get_ret();
 
     return (((rc == 0) && (rc2 == 0)) ? 0 : -EIO);
@@ -1173,7 +1171,7 @@ namespace rgw {
       return -EINVAL;
     }
 
-    rc = rgwlib.get_fe()->execute_req(&req);
+    rc = g_rgwlib->get_fe()->execute_req(&req);
     rc2 = req.get_ret();
 
     return (((rc == 0) && (rc2 == 0)) ? 0 : -EIO);
@@ -1204,7 +1202,7 @@ namespace rgw {
     req.emplace_attr(RGW_ATTR_UNIX_KEY1, std::move(ux_key));
     req.emplace_attr(RGW_ATTR_UNIX1, std::move(ux_attrs));
 
-    rc = rgwlib.get_fe()->execute_req(&req);
+    rc = g_rgwlib->get_fe()->execute_req(&req);
     rc2 = req.get_ret();
 
     if ((rc != 0) || (rc2 != 0)) {
@@ -1237,7 +1235,7 @@ namespace rgw {
     /* force cache drain, forces objects to evict */
     fh_cache.drain(ObjUnref(this),
 		  RGWFileHandle::FHCache::FLAG_LOCK);
-    rgwlib.get_fe()->get_process()->unregister_fs(this);
+    g_rgwlib->get_fe()->get_process()->unregister_fs(this);
     rele();
   } /* RGWLibFS::close */
 
@@ -1526,9 +1524,9 @@ namespace rgw {
       return false;
 
     RGWRMdirCheck req(fs->get_context(),
-		      rgwlib.get_store()->get_user(fs->get_user()->user_id),
+		      g_rgwlib->get_store()->get_user(fs->get_user()->user_id),
 		      this);
-    int rc = rgwlib.get_fe()->execute_req(&req);
+    int rc = g_rgwlib->get_fe()->execute_req(&req);
     if (! rc) {
       return req.valid && req.has_children;
     }
@@ -1582,9 +1580,9 @@ namespace rgw {
     }
 
     if (is_root()) {
-      RGWListBucketsRequest req(cct, rgwlib.get_store()->get_user(fs->get_user()->user_id),
+      RGWListBucketsRequest req(cct, g_rgwlib->get_store()->get_user(fs->get_user()->user_id),
 				this, rcb, cb_arg, offset);
-      rc = rgwlib.get_fe()->execute_req(&req);
+      rc = g_rgwlib->get_fe()->execute_req(&req);
       if (! rc) {
 	(void) clock_gettime(CLOCK_MONOTONIC_COARSE, &now); /* !LOCKED */
 	lock_guard guard(mtx);
@@ -1595,9 +1593,9 @@ namespace rgw {
 	*eof = req.eof();
       }
     } else {
-      RGWReaddirRequest req(cct, rgwlib.get_store()->get_user(fs->get_user()->user_id),
+      RGWReaddirRequest req(cct, g_rgwlib->get_store()->get_user(fs->get_user()->user_id),
 			    this, rcb, cb_arg, offset);
-      rc = rgwlib.get_fe()->execute_req(&req);
+      rc = g_rgwlib->get_fe()->execute_req(&req);
       if (! rc) {
 	(void) clock_gettime(CLOCK_MONOTONIC_COARSE, &now); /* !LOCKED */
 	lock_guard guard(mtx);
@@ -1664,10 +1662,10 @@ namespace rgw {
       /* start */
       std::string object_name = relative_object_name();
       f->write_req =
-	new RGWWriteRequest(rgwlib.get_store(),
-			    rgwlib.get_store()->get_user(fs->get_user()->user_id),
+	new RGWWriteRequest(g_rgwlib->get_store(),
+			    g_rgwlib->get_store()->get_user(fs->get_user()->user_id),
 			    this, bucket_name(), object_name);
-      rc = rgwlib.get_fe()->start_req(f->write_req);
+      rc = g_rgwlib->get_fe()->start_req(f->write_req);
       if (rc < 0) {
 	lsubdout(fs->get_context(), rgw, 5)
 	  << __func__
@@ -1754,7 +1752,7 @@ namespace rgw {
 	<< __func__
 	<< " finishing write trans on " << object_name()
 	<< dendl;
-      rc = rgwlib.get_fe()->finish_req(f->write_req);
+      rc = g_rgwlib->get_fe()->finish_req(f->write_req);
       if (! rc) {
 	rc = f->write_req->get_ret();
       }
@@ -2054,15 +2052,15 @@ void rgwfile_version(int *major, int *minor, int *extra)
 				  sec_key, "/");
   ceph_assert(new_fs);
 
-  const DoutPrefix dp(rgwlib.get_store()->ctx(), dout_subsys, "rgw mount: ");
-  rc = new_fs->authorize(&dp, rgwlib.get_store());
+  const DoutPrefix dp(g_rgwlib->get_store()->ctx(), dout_subsys, "rgw mount: ");
+  rc = new_fs->authorize(&dp, g_rgwlib->get_store());
   if (rc != 0) {
     delete new_fs;
     return -EINVAL;
   }
 
   /* register fs for shared gc */
-  rgwlib.get_fe()->get_process()->register_fs(new_fs);
+  g_rgwlib->get_fe()->get_process()->register_fs(new_fs);
 
   struct rgw_fs *fs = new_fs->get_fs();
   fs->rgw = rgw;
@@ -2097,15 +2095,15 @@ int rgw_mount2(librgw_t rgw, const char *uid, const char *acc_key,
 
   ceph_assert(new_fs); /* should we be using ceph_assert? */
 
-  const DoutPrefix dp(rgwlib.get_store()->ctx(), dout_subsys, "rgw mount2: ");
-  rc = new_fs->authorize(&dp, rgwlib.get_store());
+  const DoutPrefix dp(g_rgwlib->get_store()->ctx(), dout_subsys, "rgw mount2: ");
+  rc = new_fs->authorize(&dp, g_rgwlib->get_store());
   if (rc != 0) {
     delete new_fs;
     return -EINVAL;
   }
 
   /* register fs for shared gc */
-  rgwlib.get_fe()->get_process()->register_fs(new_fs);
+  g_rgwlib->get_fe()->get_process()->register_fs(new_fs);
 
   struct rgw_fs *fs = new_fs->get_fs();
   fs->rgw = rgw;
@@ -2150,9 +2148,9 @@ int rgw_statfs(struct rgw_fs *rgw_fs,
   struct rados_cluster_stat_t stats;
 
   RGWGetClusterStatReq req(fs->get_context(),
-			   rgwlib.get_store()->get_user(fs->get_user()->user_id),
+			   g_rgwlib->get_store()->get_user(fs->get_user()->user_id),
 			   stats);
-  int rc = rgwlib.get_fe()->execute_req(&req);
+  int rc = g_rgwlib->get_fe()->execute_req(&req);
   if (rc < 0) {
     lderr(fs->get_context()) << "ERROR: getting total cluster usage"
                              << cpp_strerror(-rc) << dendl;
@@ -2655,7 +2653,7 @@ int rgw_readv(struct rgw_fs *rgw_fs,
 		      bl);
   req.do_hexdump = false;
 
-  rc = rgwlib.get_fe()->execute_req(&req);
+  rc = g_rgwlib->get_fe()->execute_req(&req);
 
   if (! rc) {
     RGWReadV* rdv = static_cast<RGWReadV*>(
@@ -2715,10 +2713,10 @@ int rgw_writev(struct rgw_fs *rgw_fs, struct rgw_file_handle *fh,
   }
 
   std::string oname = rgw_fh->relative_object_name();
-  RGWPutObjRequest req(cct, rgwlib.get_store()->get_user(fs->get_user()->user_id),
+  RGWPutObjRequest req(cct, g_rgwlib->get_store()->get_user(fs->get_user()->user_id),
 		       rgw_fh->bucket_name(), oname, bl);
 
-  int rc = rgwlib.get_fe()->execute_req(&req);
+  int rc = g_rgwlib->get_fe()->execute_req(&req);
 
   /* XXX update size (in request) */
 
diff --git a/src/rgw/rgw_file.h b/src/rgw/rgw_file.h
index 81e7ac0a346..f217b415cd5 100644
--- a/src/rgw/rgw_file.h
+++ b/src/rgw/rgw_file.h
@@ -1010,7 +1010,7 @@ namespace rgw {
 	  return -ERR_USER_SUSPENDED;
       } else {
 	/* try external authenticators (ldap for now) */
-	rgw::LDAPHelper* ldh = rgwlib.get_ldh(); /* !nullptr */
+	rgw::LDAPHelper* ldh = g_rgwlib->get_ldh(); /* !nullptr */
 	RGWToken token;
 	/* boost filters and/or string_ref may throw on invalid input */
 	try {
@@ -1314,7 +1314,7 @@ namespace rgw {
     RGWUserInfo* get_user() { return &user->get_info(); }
 
     void update_user(const DoutPrefixProvider *dpp) {
-      (void) rgwlib.get_store()->get_user_by_access_key(dpp, key.id, null_yield, &user);
+      (void) g_rgwlib->get_store()->get_user_by_access_key(dpp, key.id, null_yield, &user);
     }
 
     void close();
diff --git a/src/rgw/rgw_frontend.h b/src/rgw/rgw_frontend.h
index a648489ac91..12f38602b3c 100644
--- a/src/rgw/rgw_frontend.h
+++ b/src/rgw/rgw_frontend.h
@@ -6,6 +6,7 @@
 
 #include <map>
 #include <string>
+#include <vector>
 
 #include "common/RWLock.h"
 
@@ -188,12 +189,12 @@ public:
 
 // FrontendPauser implementation for RGWRealmReloader
 class RGWFrontendPauser : public RGWRealmReloader::Pauser {
-  std::list<RGWFrontend*> &frontends;
+  std::vector<RGWFrontend*> &frontends;
   RGWRealmReloader::Pauser* pauser;
   rgw::auth::ImplicitTenants& implicit_tenants;
 
  public:
-  RGWFrontendPauser(std::list<RGWFrontend*> &frontends,
+  RGWFrontendPauser(std::vector<RGWFrontend*> &frontends,
                     rgw::auth::ImplicitTenants& implicit_tenants,
                     RGWRealmReloader::Pauser* pauser = nullptr)
     : frontends(frontends),
diff --git a/src/rgw/rgw_lib.cc b/src/rgw/rgw_lib.cc
new file mode 100644
index 00000000000..ebb69a26d14
--- /dev/null
+++ b/src/rgw/rgw_lib.cc
@@ -0,0 +1,609 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab ft=cpp
+
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2011 New Dream Network
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include <sys/types.h>
+#include <string.h>
+#include <chrono>
+
+#include "include/rados/librgw.h"
+#include "rgw_acl.h"
+
+#include "include/str_list.h"
+#include "global/signal_handler.h"
+#include "common/Timer.h"
+#include "common/WorkQueue.h"
+#include "common/ceph_argparse.h"
+#include "common/ceph_context.h"
+#include "common/common_init.h"
+#include "common/dout.h"
+
+#include "rgw_op.h"
+#include "rgw_rest.h"
+#include "rgw_log.h"
+#include "rgw_frontend.h"
+#include "rgw_request.h"
+#include "rgw_process.h"
+#include "rgw_auth.h"
+#include "rgw_lib.h"
+#include "rgw_lib_frontend.h"
+#include "rgw_perf_counters.h"
+#include "rgw_signal.h"
+#include "rgw_main.h"
+
+#include <errno.h>
+#include <thread>
+#include <string>
+#include <mutex>
+
+#define dout_subsys ceph_subsys_rgw
+
+using namespace std;
+
+namespace rgw {
+
+  RGWLib* g_rgwlib = nullptr;
+
+  class C_InitTimeout : public Context {
+  public:
+    C_InitTimeout() {}
+    void finish(int r) override {
+      derr << "Initialization timeout, failed to initialize" << dendl;
+      exit(1);
+    }
+  };
+
+  void RGWLibProcess::checkpoint()
+  {
+    m_tp.drain(&req_wq);
+  }
+
+#define MIN_EXPIRE_S 120
+
+  void RGWLibProcess::run()
+  {
+    /* write completion interval */
+    RGWLibFS::write_completion_interval_s =
+      cct->_conf->rgw_nfs_write_completion_interval_s;
+
+    /* start write timer */
+    RGWLibFS::write_timer.resume();
+
+    /* gc loop */
+    while (! shutdown) {
+      lsubdout(cct, rgw, 5) << "RGWLibProcess GC" << dendl;
+
+      /* dirent invalidate timeout--basically, the upper-bound on
+       * inconsistency with the S3 namespace */
+      auto expire_s = cct->_conf->rgw_nfs_namespace_expire_secs;
+
+      /* delay between gc cycles */
+      auto delay_s = std::max(int64_t(1), std::min(int64_t(MIN_EXPIRE_S), expire_s/2));
+
+      unique_lock uniq(mtx);
+    restart:
+      int cur_gen = gen;
+      for (auto iter = mounted_fs.begin(); iter != mounted_fs.end();
+	   ++iter) {
+	RGWLibFS* fs = iter->first->ref();
+	uniq.unlock();
+	fs->gc();
+        const DoutPrefix dp(cct, dout_subsys, "librgw: ");
+	fs->update_user(&dp);
+	fs->rele();
+	uniq.lock();
+	if (cur_gen != gen)
+	  goto restart; /* invalidated */
+      }
+      cv.wait_for(uniq, std::chrono::seconds(delay_s));
+      uniq.unlock();
+    }
+  }
+
+  void RGWLibProcess::handle_request(const DoutPrefixProvider *dpp, RGWRequest* r)
+  {
+    /*
+     * invariant: valid requests are derived from RGWLibRequst
+     */
+    RGWLibRequest* req = static_cast<RGWLibRequest*>(r);
+
+    // XXX move RGWLibIO and timing setup into process_request
+
+#if 0 /* XXX */
+    utime_t tm = ceph_clock_now();
+#endif
+
+    RGWLibIO io_ctx;
+
+    int ret = process_request(req, &io_ctx);
+    if (ret < 0) {
+      /* we don't really care about return code */
+      dout(20) << "process_request() returned " << ret << dendl;
+
+    }
+    delete req;
+  } /* handle_request */
+
+  int RGWLibProcess::process_request(RGWLibRequest* req)
+  {
+    // XXX move RGWLibIO and timing setup into process_request
+
+#if 0 /* XXX */
+    utime_t tm = ceph_clock_now();
+#endif
+
+    RGWLibIO io_ctx;
+
+    int ret = process_request(req, &io_ctx);
+    if (ret < 0) {
+      /* we don't really care about return code */
+      dout(20) << "process_request() returned " << ret << dendl;
+    }
+    return ret;
+  } /* process_request */
+
+  static inline void abort_req(req_state *s, RGWOp *op, int err_no)
+  {
+    if (!s)
+      return;
+
+    /* XXX the dump_errno and dump_bucket_from_state behaviors in
+     * the abort_early (rgw_rest.cc) might be valuable, but aren't
+     * safe to call presently as they return HTTP data */
+
+    perfcounter->inc(l_rgw_failed_req);
+  } /* abort_req */
+
+  int RGWLibProcess::process_request(RGWLibRequest* req, RGWLibIO* io)
+  {
+    int ret = 0;
+    bool should_log = true; // XXX
+
+    dout(1) << "====== " << __func__
+	    << " starting new request req=" << hex << req << dec
+	    << " ======" << dendl;
+
+    /*
+     * invariant: valid requests are derived from RGWOp--well-formed
+     * requests should have assigned RGWRequest::op in their descendant
+     * constructor--if not, the compiler can find it, at the cost of
+     * a runtime check
+     */
+    RGWOp *op = (req->op) ? req->op : dynamic_cast<RGWOp*>(req);
+    if (! op) {
+      ldpp_dout(op, 1) << "failed to derive cognate RGWOp (invalid op?)" << dendl;
+      return -EINVAL;
+    }
+
+    io->init(req->cct);
+
+    perfcounter->inc(l_rgw_req);
+
+    RGWEnv& rgw_env = io->get_env();
+
+    /* XXX
+     * until major refactoring of req_state and req_info, we need
+     * to build their RGWEnv boilerplate from the RGWLibRequest,
+     * pre-staging any strings (HTTP_HOST) that provoke a crash when
+     * not found
+     */
+
+    /* XXX for now, use "";  could be a legit hostname, or, in future,
+     * perhaps a tenant (Yehuda) */
+    rgw_env.set("HTTP_HOST", "");
+
+    /* XXX and -then- bloat up req_state with string copies from it */
+    req_state rstate(req->cct, &rgw_env, req->id);
+    req_state *s = &rstate;
+
+    // XXX fix this
+    s->cio = io;
+
+    /* XXX and -then- stash req_state pointers everywhere they are needed */
+    ret = req->init(rgw_env, store, io, s);
+    if (ret < 0) {
+      ldpp_dout(op, 10) << "failed to initialize request" << dendl;
+      abort_req(s, op, ret);
+      goto done;
+    }
+
+    /* req is-a RGWOp, currently initialized separately */
+    ret = req->op_init();
+    if (ret < 0) {
+      dout(10) << "failed to initialize RGWOp" << dendl;
+      abort_req(s, op, ret);
+      goto done;
+    }
+
+    /* now expected by rgw_log_op() */
+    rgw_env.set("REQUEST_METHOD", s->info.method);
+    rgw_env.set("REQUEST_URI", s->info.request_uri);
+    rgw_env.set("QUERY_STRING", "");
+
+    try {
+      /* XXX authorize does less here then in the REST path, e.g.,
+       * the user's info is cached, but still incomplete */
+      ldpp_dout(s, 2) << "authorizing" << dendl;
+      ret = req->authorize(op, null_yield);
+      if (ret < 0) {
+	dout(10) << "failed to authorize request" << dendl;
+	abort_req(s, op, ret);
+	goto done;
+      }
+
+      /* FIXME: remove this after switching all handlers to the new
+       * authentication infrastructure. */
+      if (! s->auth.identity) {
+	s->auth.identity = rgw::auth::transform_old_authinfo(s);
+      }
+
+      ldpp_dout(s, 2) << "reading op permissions" << dendl;
+      ret = req->read_permissions(op, null_yield);
+      if (ret < 0) {
+	abort_req(s, op, ret);
+	goto done;
+      }
+
+      ldpp_dout(s, 2) << "init op" << dendl;
+      ret = op->init_processing(null_yield);
+      if (ret < 0) {
+	abort_req(s, op, ret);
+	goto done;
+      }
+
+      ldpp_dout(s, 2) << "verifying op mask" << dendl;
+      ret = op->verify_op_mask();
+      if (ret < 0) {
+	abort_req(s, op, ret);
+	goto done;
+      }
+
+      ldpp_dout(s, 2) << "verifying op permissions" << dendl;
+      ret = op->verify_permission(null_yield);
+      if (ret < 0) {
+	if (s->system_request) {
+	  ldpp_dout(op, 2) << "overriding permissions due to system operation" << dendl;
+	} else if (s->auth.identity->is_admin_of(s->user->get_id())) {
+	  ldpp_dout(op, 2) << "overriding permissions due to admin operation" << dendl;
+	} else {
+	  abort_req(s, op, ret);
+	  goto done;
+	}
+      }
+
+      ldpp_dout(s, 2) << "verifying op params" << dendl;
+      ret = op->verify_params();
+      if (ret < 0) {
+	abort_req(s, op, ret);
+	goto done;
+      }
+
+      ldpp_dout(s, 2) << "executing" << dendl;
+      op->pre_exec();
+      op->execute(null_yield);
+      op->complete();
+
+    } catch (const ceph::crypto::DigestException& e) {
+      dout(0) << "authentication failed" << e.what() << dendl;
+      abort_req(s, op, -ERR_INVALID_SECRET_KEY);
+    }
+
+  done:
+    try {
+      io->complete_request();
+    } catch (rgw::io::Exception& e) {
+      dout(0) << "ERROR: io->complete_request() returned "
+              << e.what() << dendl;
+    }
+    if (should_log) {
+      rgw_log_op(nullptr /* !rest */, s, (op ? op->name() : "unknown"), olog);
+    }
+
+    int http_ret = s->err.http_ret;
+
+    ldpp_dout(s, 2) << "http status=" << http_ret << dendl;
+
+    ldpp_dout(op, 1) << "====== " << __func__
+	    << " req done req=" << hex << req << dec << " http_status="
+	    << http_ret
+	    << " ======" << dendl;
+
+    return (ret < 0 ? ret : s->err.ret);
+  } /* process_request */
+
+  int RGWLibProcess::start_request(RGWLibContinuedReq* req)
+  {
+
+    dout(1) << "====== " << __func__
+	    << " starting new continued request req=" << hex << req << dec
+	    << " ======" << dendl;
+
+    /*
+     * invariant: valid requests are derived from RGWOp--well-formed
+     * requests should have assigned RGWRequest::op in their descendant
+     * constructor--if not, the compiler can find it, at the cost of
+     * a runtime check
+     */
+    RGWOp *op = (req->op) ? req->op : dynamic_cast<RGWOp*>(req);
+    if (! op) {
+      ldpp_dout(op, 1) << "failed to derive cognate RGWOp (invalid op?)" << dendl;
+      return -EINVAL;
+    }
+
+    req_state* s = req->get_state();
+    RGWLibIO& io_ctx = req->get_io();
+    RGWEnv& rgw_env = io_ctx.get_env();
+
+    rgw_env.set("HTTP_HOST", "");
+
+    int ret = req->init(rgw_env, store, &io_ctx, s);
+    if (ret < 0) {
+      ldpp_dout(op, 10) << "failed to initialize request" << dendl;
+      abort_req(s, op, ret);
+      goto done;
+    }
+
+    /* req is-a RGWOp, currently initialized separately */
+    ret = req->op_init();
+    if (ret < 0) {
+      dout(10) << "failed to initialize RGWOp" << dendl;
+      abort_req(s, op, ret);
+      goto done;
+    }
+
+    /* XXX authorize does less here then in the REST path, e.g.,
+     * the user's info is cached, but still incomplete */
+    ldpp_dout(s, 2) << "authorizing" << dendl;
+    ret = req->authorize(op, null_yield);
+    if (ret < 0) {
+      dout(10) << "failed to authorize request" << dendl;
+      abort_req(s, op, ret);
+      goto done;
+    }
+
+    /* FIXME: remove this after switching all handlers to the new authentication
+     * infrastructure. */
+    if (! s->auth.identity) {
+      s->auth.identity = rgw::auth::transform_old_authinfo(s);
+    }
+
+    ldpp_dout(s, 2) << "reading op permissions" << dendl;
+    ret = req->read_permissions(op, null_yield);
+    if (ret < 0) {
+      abort_req(s, op, ret);
+      goto done;
+    }
+
+    ldpp_dout(s, 2) << "init op" << dendl;
+    ret = op->init_processing(null_yield);
+    if (ret < 0) {
+      abort_req(s, op, ret);
+      goto done;
+    }
+
+    ldpp_dout(s, 2) << "verifying op mask" << dendl;
+    ret = op->verify_op_mask();
+    if (ret < 0) {
+      abort_req(s, op, ret);
+      goto done;
+    }
+
+    ldpp_dout(s, 2) << "verifying op permissions" << dendl;
+    ret = op->verify_permission(null_yield);
+    if (ret < 0) {
+      if (s->system_request) {
+	ldpp_dout(op, 2) << "overriding permissions due to system operation" << dendl;
+      } else if (s->auth.identity->is_admin_of(s->user->get_id())) {
+	ldpp_dout(op, 2) << "overriding permissions due to admin operation" << dendl;
+      } else {
+	abort_req(s, op, ret);
+	goto done;
+      }
+    }
+
+    ldpp_dout(s, 2) << "verifying op params" << dendl;
+    ret = op->verify_params();
+    if (ret < 0) {
+      abort_req(s, op, ret);
+      goto done;
+    }
+
+    op->pre_exec();
+    req->exec_start();
+
+  done:
+    return (ret < 0 ? ret : s->err.ret);
+  }
+
+  int RGWLibProcess::finish_request(RGWLibContinuedReq* req)
+  {
+    RGWOp *op = (req->op) ? req->op : dynamic_cast<RGWOp*>(req);
+    if (! op) {
+      ldpp_dout(op, 1) << "failed to derive cognate RGWOp (invalid op?)" << dendl;
+      return -EINVAL;
+    }
+
+    int ret = req->exec_finish();
+    int op_ret = op->get_ret();
+
+    ldpp_dout(op, 1) << "====== " << __func__
+	    << " finishing continued request req=" << hex << req << dec
+	    << " op status=" << op_ret
+	    << " ======" << dendl;
+
+    perfcounter->inc(l_rgw_req);
+
+    return ret;
+  }
+
+  int RGWLibFrontend::init()
+  {
+    pprocess = new RGWLibProcess(g_ceph_context, &env,
+				 g_conf()->rgw_thread_pool_size, conf);
+    return 0;
+  }
+
+  void RGWLib::set_fe(rgw::RGWLibFrontend* fe)
+  {
+    this->fe = fe;
+  }
+
+  int RGWLib::init()
+  {
+    vector<const char*> args;
+    return init(args);
+  }
+
+  int RGWLib::init(vector<const char*>& args)
+  {
+    /* alternative default for module */
+    map<std::string,std::string> defaults = {
+      { "debug_rgw", "1/5" },
+      { "keyring", "$rgw_data/keyring" },
+      { "log_file", "/var/log/radosgw/$cluster-$name.log" },
+      { "objecter_inflight_ops", "24576" },
+      // require a secure mon connection by default
+      { "ms_mon_client_mode", "secure" },
+      { "auth_client_required", "cephx" },
+    };
+
+    cct = rgw_global_init(&defaults, args,
+			  CEPH_ENTITY_TYPE_CLIENT,
+			  CODE_ENVIRONMENT_DAEMON,
+			  CINIT_FLAG_UNPRIVILEGED_DAEMON_DEFAULTS);
+
+    ceph::mutex mutex = ceph::make_mutex("main");
+    SafeTimer init_timer(g_ceph_context, mutex);
+    init_timer.init();
+    mutex.lock();
+    init_timer.add_event_after(g_conf()->rgw_init_timeout, new C_InitTimeout);
+    mutex.unlock();
+
+    /* stage all front-ends (before common-init-finish) */
+    main.init_frontends1(true /* nfs */);
+
+    common_init_finish(g_ceph_context);
+
+    main.init_perfcounters();
+    main.init_http_clients();
+
+    main.init_storage();
+    if (! main.get_store()) {
+      mutex.lock();
+      init_timer.cancel_all_events();
+      init_timer.shutdown();
+      mutex.unlock();
+
+      derr << "Couldn't init storage provider (RADOS)" << dendl;
+      return -EIO;
+    }
+
+    main.cond_init_apis();
+
+    mutex.lock();
+    init_timer.cancel_all_events();
+    init_timer.shutdown();
+    mutex.unlock();
+
+    main.init_ldap();
+    main.init_opslog();
+
+    init_async_signal_handler();
+    register_async_signal_handler(SIGUSR1, rgw::signal::handle_sigterm);
+
+    main.init_tracepoints();
+    main.init_frontends2(this /* rgwlib */);
+    main.init_notification_endpoints();
+    main.init_lua();
+
+    return 0;
+  } /* RGWLib::init() */
+
+  int RGWLib::stop()
+  {
+    derr << "shutting down" << dendl;
+
+    const auto finalize_async_signals = []() {
+      unregister_async_signal_handler(SIGUSR1, rgw::signal::handle_sigterm);
+      shutdown_async_signal_handler();
+    };
+
+    main.shutdown(finalize_async_signals);
+
+    return 0;
+  } /* RGWLib::stop() */
+
+  int RGWLibIO::set_uid(rgw::sal::Store* store, const rgw_user& uid)
+  {
+    const DoutPrefix dp(store->ctx(), dout_subsys, "librgw: ");
+    std::unique_ptr<rgw::sal::User> user = store->get_user(uid);
+    /* object exists, but policy is broken */
+    int ret = user->load_user(&dp, null_yield);
+    if (ret < 0) {
+      derr << "ERROR: failed reading user info: uid=" << uid << " ret="
+	   << ret << dendl;
+    }
+    user_info = user->get_info();
+    return ret;
+  }
+
+  int RGWLibRequest::read_permissions(RGWOp* op, optional_yield y) {
+    /* bucket and object ops */
+    int ret =
+      rgw_build_bucket_policies(op, g_rgwlib->get_store(), get_state(), y);
+    if (ret < 0) {
+      ldpp_dout(op, 10) << "read_permissions (bucket policy) on "
+				  << get_state()->bucket << ":"
+				  << get_state()->object
+				  << " only_bucket=" << only_bucket()
+				  << " ret=" << ret << dendl;
+      if (ret == -ENODATA)
+	ret = -EACCES;
+    } else if (! only_bucket()) {
+      /* object ops */
+      ret = rgw_build_object_policies(op, g_rgwlib->get_store(), get_state(),
+				      op->prefetch_data(), y);
+      if (ret < 0) {
+	ldpp_dout(op, 10) << "read_permissions (object policy) on"
+				    << get_state()->bucket << ":"
+				    << get_state()->object
+				    << " ret=" << ret << dendl;
+	if (ret == -ENODATA)
+	  ret = -EACCES;
+      }
+    }
+    return ret;
+  } /* RGWLibRequest::read_permissions */
+
+  int RGWHandler_Lib::authorize(const DoutPrefixProvider *dpp, optional_yield y)
+  {
+    /* TODO: handle
+     *  1. subusers
+     *  2. anonymous access
+     *  3. system access
+     *  4. ?
+     *
+     *  Much or all of this depends on handling the cached authorization
+     *  correctly (e.g., dealing with keystone) at mount time.
+     */
+    s->perm_mask = RGW_PERM_FULL_CONTROL;
+
+    // populate the owner info
+    s->owner.set_id(s->user->get_id());
+    s->owner.set_name(s->user->get_display_name());
+
+    return 0;
+  } /* RGWHandler_Lib::authorize */
+
+} /* namespace rgw */
diff --git a/src/rgw/rgw_lib.h b/src/rgw/rgw_lib.h
index 7336aaf4c8f..06a69025588 100644
--- a/src/rgw/rgw_lib.h
+++ b/src/rgw/rgw_lib.h
@@ -5,18 +5,13 @@
 #define RGW_LIB_H
 
 #include <mutex>
-#include "include/unordered_map.h"
-#include "global/global_init.h"
 #include "rgw_common.h"
 #include "rgw_client_io.h"
 #include "rgw_rest.h"
 #include "rgw_request.h"
-#include "rgw_frontend.h"
-#include "rgw_process.h"
-#include "rgw_rest_s3.h" // RGW_Auth_S3
 #include "rgw_ldap.h"
-#include "services/svc_zone_utils.h"
 #include "include/ceph_assert.h"
+#include "rgw_main.h"
 
 class OpsLogSink;
 
@@ -25,35 +20,32 @@ namespace rgw {
   class RGWLibFrontend;
 
   class RGWLib : public DoutPrefixProvider {
-    RGWFrontendConfig* fec;
-    RGWLibFrontend* fe;
-    OpsLogSink* olog;
-    rgw::LDAPHelper* ldh{nullptr};
-    RGWREST rest; // XXX needed for RGWProcessEnv
-    rgw::sal::Store* store;
     boost::intrusive_ptr<CephContext> cct;
+    AppMain main;
+    RGWLibFrontend* fe;
 
   public:
-    RGWLib() : fec(nullptr), fe(nullptr), olog(nullptr), store(nullptr)
+    RGWLib() : main(this), fe(nullptr)
       {}
     ~RGWLib() {}
 
-    rgw::sal::Store* get_store() { return store; }
+    rgw::sal::Store* get_store() { return main.get_store(); }
 
     RGWLibFrontend* get_fe() { return fe; }
 
-    rgw::LDAPHelper* get_ldh() { return ldh; }
-
+    rgw::LDAPHelper* get_ldh() { return main.get_ldh(); }
     CephContext *get_cct() const override { return cct.get(); }
     unsigned get_subsys() const { return ceph_subsys_rgw; }
     std::ostream& gen_prefix(std::ostream& out) const { return out << "lib rgw: "; }
 
+    void set_fe(RGWLibFrontend* fe);
+
     int init();
     int init(std::vector<const char *>& args);
     int stop();
   };
 
-  extern RGWLib rgwlib;
+  extern RGWLib* g_rgwlib;
 
 /* request interface */
 
@@ -109,14 +101,12 @@ namespace rgw {
 
   }; /* RGWLibIO */
 
-/* XXX */
   class RGWRESTMgr_Lib : public RGWRESTMgr {
   public:
     RGWRESTMgr_Lib() {}
     ~RGWRESTMgr_Lib() override {}
   }; /* RGWRESTMgr_Lib */
 
-/* XXX */
   class RGWHandler_Lib : public RGWHandler {
     friend class RGWRESTMgr_Lib;
   public:
@@ -140,7 +130,7 @@ namespace rgw {
     inline req_state* get_state() { return this->RGWRequest::s; }
 
     RGWLibRequest(CephContext* _cct, std::unique_ptr<rgw::sal::User> _user)
-      :  RGWRequest(rgwlib.get_store()->get_new_req_id()),
+      :  RGWRequest(g_rgwlib->get_store()->get_new_req_id()),
 	 tuser(std::move(_user)), cct(_cct)
       {}
 
@@ -198,7 +188,7 @@ namespace rgw {
 	io_ctx.init(_cct);
 
 	RGWRequest::init_state(&rstate);
-	RGWHandler::init(rgwlib.get_store(), &rstate, &io_ctx);
+	RGWHandler::init(g_rgwlib->get_store(), &rstate, &io_ctx);
 
 	get_state()->req_id = store->zone_unique_id(id);
 	get_state()->trans_id = store->zone_unique_trans_id(id);
diff --git a/src/rgw/rgw_loadgen_process.cc b/src/rgw/rgw_loadgen_process.cc
index e5a966b4fd7..524d664c7cc 100644
--- a/src/rgw/rgw_loadgen_process.cc
+++ b/src/rgw/rgw_loadgen_process.cc
@@ -11,6 +11,7 @@
 #include "rgw_process.h"
 #include "rgw_loadgen.h"
 #include "rgw_client_io.h"
+#include "rgw_signal.h"
 
 #include <atomic>
 
@@ -18,8 +19,6 @@
 
 using namespace std;
 
-extern void signal_shutdown();
-
 void RGWLoadGenProcess::checkpoint()
 {
   m_tp.drain(&req_wq);
@@ -100,7 +99,7 @@ done:
 
   delete[] objs;
 
-  signal_shutdown();
+  rgw::signal::signal_shutdown();
 } /* RGWLoadGenProcess::run() */
 
 void RGWLoadGenProcess::gen_request(const string& method,
diff --git a/src/rgw/rgw_main.cc b/src/rgw/rgw_main.cc
index 2b6e66e3806..b09902948d4 100644
--- a/src/rgw/rgw_main.cc
+++ b/src/rgw/rgw_main.cc
@@ -8,51 +8,12 @@
 #include "common/config.h"
 #include "common/errno.h"
 #include "common/Timer.h"
-#include "common/safe_io.h"
 #include "common/TracepointProvider.h"
-#include "common/openssl_opts_handler.h"
-#include "common/numa.h"
-#include "include/compat.h"
-#include "include/str_list.h"
-#include "include/stringify.h"
+#include "rgw_main.h"
+#include "rgw_signal.h"
 #include "rgw_common.h"
-#include "rgw_sal_rados.h"
-#include "rgw_period_pusher.h"
-#include "rgw_realm_reloader.h"
-#include "rgw_rest.h"
-#include "rgw_rest_s3.h"
-#include "rgw_rest_swift.h"
-#include "rgw_rest_admin.h"
-#include "rgw_rest_info.h"
-#include "rgw_rest_usage.h"
-#include "rgw_rest_user.h"
-#include "rgw_rest_sts.h"
-#include "rgw_swift_auth.h"
+#include "rgw_lib.h"
 #include "rgw_log.h"
-#include "rgw_tools.h"
-#include "rgw_resolve.h"
-#include "rgw_request.h"
-#include "rgw_process.h"
-#include "rgw_frontend.h"
-#include "rgw_http_client_curl.h"
-#include "rgw_kmip_client.h"
-#include "rgw_kmip_client_impl.h"
-#include "rgw_perf_counters.h"
-#ifdef WITH_RADOSGW_AMQP_ENDPOINT
-#include "rgw_amqp.h"
-#endif
-#ifdef WITH_RADOSGW_KAFKA_ENDPOINT
-#include "rgw_kafka.h"
-#endif
-#include "rgw_asio_frontend.h"
-#include "rgw_dmclock_scheduler_ctx.h"
-#include "rgw_lua.h"
-#ifdef WITH_RADOSGW_DBSTORE
-#include "rgw_sal_dbstore.h"
-#endif
-#include "rgw_lua_background.h"
-
-#include "services/svc_zone.h"
 
 #ifdef HAVE_SYS_PRCTL_H
 #include <sys/prctl.h>
@@ -62,83 +23,13 @@ using namespace std;
 
 static constexpr auto dout_subsys = ceph_subsys_rgw;
 
-namespace {
-TracepointProvider::Traits rgw_op_tracepoint_traits("librgw_op_tp.so",
-                                                 "rgw_op_tracing");
-TracepointProvider::Traits rgw_rados_tracepoint_traits("librgw_rados_tp.so",
-                                                 "rgw_rados_tracing");
-}
-
 static sig_t sighandler_alrm;
 
-class RGWProcess;
-
-static int signal_fd[2] = {0, 0};
-
-void signal_shutdown()
-{
-  int val = 0;
-  int ret = write(signal_fd[0], (char *)&val, sizeof(val));
-  if (ret < 0) {
-    derr << "ERROR: " << __func__ << ": write() returned "
-         << cpp_strerror(errno) << dendl;
-  }
-}
-
-static void wait_shutdown()
-{
-  int val;
-  int r = safe_read_exact(signal_fd[1], &val, sizeof(val));
-  if (r < 0) {
-    derr << "safe_read_exact returned with error" << dendl;
-  }
-}
-
-static int signal_fd_init()
-{
-  return socketpair(AF_UNIX, SOCK_STREAM, 0, signal_fd);
-}
-
-static void signal_fd_finalize()
-{
-  close(signal_fd[0]);
-  close(signal_fd[1]);
-}
-
-static void handle_sigterm(int signum)
-{
-  dout(1) << __func__ << dendl;
-
-  // send a signal to make fcgi's accept(2) wake up.  unfortunately the
-  // initial signal often isn't sufficient because we race with accept's
-  // check of the flag wet by ShutdownPending() above.
-  if (signum != SIGUSR1) {
-    signal_shutdown();
-
-    // safety net in case we get stuck doing an orderly shutdown.
-    uint64_t secs = g_ceph_context->_conf->rgw_exit_timeout_secs;
-    if (secs)
-      alarm(secs);
-    dout(1) << __func__ << " set alarm for " << secs << dendl;
-  }
-
-}
-
-static OpsLogFile* ops_log_file = nullptr;
-
-static void rgw_sighup_handler(int signum) {
-    if (ops_log_file != nullptr) {
-        ops_log_file->reopen();
-    }
-    sighup_handler(signum);
-}
-
 static void godown_alarm(int signum)
 {
   _exit(0);
 }
 
-
 class C_InitTimeout : public Context {
 public:
   C_InitTimeout() {}
@@ -164,46 +55,13 @@ static int usage()
   return 0;
 }
 
-static RGWRESTMgr *set_logging(RGWRESTMgr *mgr)
-{
-  mgr->set_logging(true);
-  return mgr;
-}
-
-static RGWRESTMgr *rest_filter(rgw::sal::Store* store, int dialect, RGWRESTMgr *orig)
-{
-  RGWSyncModuleInstanceRef sync_module = store->get_sync_module();
-  if (sync_module) {
-    return sync_module->get_rest_filter(dialect, orig);
-  } else {
-    return orig;
-  }
-}
-
-class RGWPauser : public RGWRealmReloader::Pauser {
-  std::vector<Pauser*> pausers;
-
-public:
-  ~RGWPauser() override = default;
-  
-  void add_pauser(Pauser* pauser) {
-    pausers.push_back(pauser);
-  }
-
-  void pause() override {
-    std::for_each(pausers.begin(), pausers.end(), [](Pauser* p){p->pause();});
-  }
-  void resume(rgw::sal::Store* store) override {
-    std::for_each(pausers.begin(), pausers.end(), [store](Pauser* p){p->resume(store);});
-  }
-
-};
-
 /*
  * start up the RADOS connection and then handle HTTP messages as they come in
  */
-int main(int argc, const char **argv)
-{
+int main(int argc, char *argv[])
+{ 
+  int r{0};
+
   // dout() messages will be sent to stderr, but FCGX wants messages on stdout
   // Redirect stderr to stdout.
   TEMP_FAILURE_RETRY(close(STDERR_FILENO));
@@ -215,7 +73,7 @@ int main(int argc, const char **argv)
   }
 
   /* alternative default for module */
-  map<string,string> defaults = {
+  map<std::string,std::string> defaults = {
     { "debug_rgw", "1/5" },
     { "keyring", "$rgw_data/keyring" },
     { "objecter_inflight_ops", "24576" },
@@ -242,84 +100,11 @@ int main(int argc, const char **argv)
   auto cct = rgw_global_init(&defaults, args, CEPH_ENTITY_TYPE_CLIENT,
 			     CODE_ENVIRONMENT_DAEMON, flags);
 
-  // First, let's determine which frontends are configured.
-  list<string> frontends;
-  string rgw_frontends_str = g_conf().get_val<string>("rgw_frontends");
-  g_conf().early_expand_meta(rgw_frontends_str, &cerr);
-  get_str_list(rgw_frontends_str, ",", frontends);
-  multimap<string, RGWFrontendConfig *> fe_map;
-  list<RGWFrontendConfig *> configs;
-  if (frontends.empty()) {
-    frontends.push_back("beast");
-  }
-  for (list<string>::iterator iter = frontends.begin(); iter != frontends.end(); ++iter) {
-    string& f = *iter;
-
-    if (f.find("beast") != string::npos) {
-      if (f.find("port") != string::npos) {
-        // check for the most common ws problems
-        if ((f.find("port=") == string::npos) ||
-            (f.find("port= ") != string::npos)) {
-          derr << "WARNING: radosgw frontend config found unexpected spacing around 'port' "
-               << "(ensure frontend port parameter has the form 'port=80' with no spaces "
-               << "before or after '=')" << dendl;
-        }
-      }
-    }
-
-    RGWFrontendConfig *config = new RGWFrontendConfig(f);
-    int r = config->init();
-    if (r < 0) {
-      delete config;
-      cerr << "ERROR: failed to init config: " << f << std::endl;
-      return EINVAL;
-    }
-
-    configs.push_back(config);
-
-    string framework = config->get_framework();
-    fe_map.insert(pair<string, RGWFrontendConfig*>(framework, config));
-  }
-
-  int numa_node = g_conf().get_val<int64_t>("rgw_numa_node");
-  size_t numa_cpu_set_size = 0;
-  cpu_set_t numa_cpu_set;
-
-  if (numa_node >= 0) {
-    int r = get_numa_node_cpu_set(numa_node, &numa_cpu_set_size, &numa_cpu_set);
-    if (r < 0) {
-      dout(1) << __func__ << " unable to determine rgw numa node " << numa_node
-              << " CPUs" << dendl;
-      numa_node = -1;
-    } else {
-      r = set_cpu_affinity_all_threads(numa_cpu_set_size, &numa_cpu_set);
-      if (r < 0) {
-        derr << __func__ << " failed to set numa affinity: " << cpp_strerror(r)
-        << dendl;
-      }
-    }
-  } else {
-    dout(1) << __func__ << " not setting numa affinity" << dendl;
-  }
-
-  // maintain existing region root pool for new multisite objects
-  if (!g_conf()->rgw_region_root_pool.empty()) {
-    const char *root_pool = g_conf()->rgw_region_root_pool.c_str();
-    if (g_conf()->rgw_zonegroup_root_pool.empty()) {
-      g_conf().set_val_or_die("rgw_zonegroup_root_pool", root_pool);
-    }
-    if (g_conf()->rgw_period_root_pool.empty()) {
-      g_conf().set_val_or_die("rgw_period_root_pool", root_pool);
-    }
-    if (g_conf()->rgw_realm_root_pool.empty()) {
-      g_conf().set_val_or_die("rgw_realm_root_pool", root_pool);
-    }
-  }
+  DoutPrefix dp(cct.get(), dout_subsys, "rgw main: ");
+  rgw::AppMain main(&dp);
 
-  // for region -> zonegroup conversion (must happen before common_init_finish())
-  if (!g_conf()->rgw_region.empty() && g_conf()->rgw_zonegroup.empty()) {
-    g_conf().set_val_or_die("rgw_zonegroup", g_conf()->rgw_region.c_str());
-  }
+  main.init_frontends1(false /* nfs */);
+  main.init_numa();
 
   if (g_conf()->daemonize) {
     global_init_daemonize(g_ceph_context);
@@ -331,39 +116,27 @@ int main(int argc, const char **argv)
   init_timer.add_event_after(g_conf()->rgw_init_timeout, new C_InitTimeout);
   mutex.unlock();
 
-  ceph::crypto::init_openssl_engine_once();
-
   common_init_finish(g_ceph_context);
-
   init_async_signal_handler();
 
-  TracepointProvider::initialize<rgw_rados_tracepoint_traits>(g_ceph_context);
-  TracepointProvider::initialize<rgw_op_tracepoint_traits>(g_ceph_context);
-
-  const DoutPrefix dp(cct.get(), dout_subsys, "rgw main: ");
-  int r = rgw_tools_init(&dp, g_ceph_context);
+  /* XXXX check locations thru sighandler_alrm */
+  register_async_signal_handler(SIGHUP, rgw::signal::sighup_handler);
+  r = rgw::signal::signal_fd_init();
   if (r < 0) {
-    derr << "ERROR: unable to initialize rgw tools" << dendl;
-    return -r;
+    derr << "ERROR: unable to initialize signal fds" << dendl;
+  exit(1);
   }
-  tracing::rgw::tracer.init("rgw");
-  rgw_init_resolver();
-  rgw::curl::setup_curl(fe_map);
-  rgw_http_client_init(g_ceph_context);
-  rgw_kmip_client_init(*new RGWKMIPManagerImpl(g_ceph_context));
-  
-  StoreManager::Config cfg = StoreManager::get_config(false, g_ceph_context);
-
-  rgw::sal::Store* store =
-    StoreManager::get_storage(&dp, g_ceph_context,
-				 cfg,
-				 g_conf()->rgw_enable_gc_threads,
-				 g_conf()->rgw_enable_lc_threads,
-				 g_conf()->rgw_enable_quota_threads,
-				 g_conf()->rgw_run_sync_thread,
-				 g_conf().get_val<bool>("rgw_dynamic_resharding"),
-				 g_conf()->rgw_cache_enabled);
-  if (!store) {
+
+  register_async_signal_handler(SIGTERM, rgw::signal::handle_sigterm);
+  register_async_signal_handler(SIGINT, rgw::signal::handle_sigterm);
+  register_async_signal_handler(SIGUSR1, rgw::signal::handle_sigterm);
+  sighandler_alrm = signal(SIGALRM, godown_alarm);
+
+  main.init_perfcounters();
+  main.init_http_clients();
+
+  main.init_storage();
+  if (! main.get_store()) {
     mutex.lock();
     init_timer.cancel_all_events();
     init_timer.shutdown();
@@ -372,299 +145,20 @@ int main(int argc, const char **argv)
     derr << "Couldn't init storage provider (RADOS)" << dendl;
     return EIO;
   }
-  r = rgw_perf_start(g_ceph_context);
-  if (r < 0) {
-    derr << "ERROR: failed starting rgw perf" << dendl;
-    return -r;
-  }
 
-  rgw_rest_init(g_ceph_context, store->get_zone()->get_zonegroup());
+  main.cond_init_apis();
 
   mutex.lock();
   init_timer.cancel_all_events();
   init_timer.shutdown();
   mutex.unlock();
 
-  rgw_log_usage_init(g_ceph_context, store);
-
-  RGWREST rest;
-
-  list<string> apis;
-
-  get_str_list(g_conf()->rgw_enable_apis, apis);
-
-  map<string, bool> apis_map;
-  for (list<string>::iterator li = apis.begin(); li != apis.end(); ++li) {
-    apis_map[*li] = true;
-  }
-
-  /* warn about insecure keystone secret config options */
-  if (!(g_ceph_context->_conf->rgw_keystone_admin_token.empty() ||
-	g_ceph_context->_conf->rgw_keystone_admin_password.empty())) {
-    dout(0) << "WARNING: rgw_keystone_admin_token and rgw_keystone_admin_password should be avoided as they can expose secrets.  Prefer the new rgw_keystone_admin_token_path and rgw_keystone_admin_password_path options, which read their secrets from files." << dendl;
-  }
-
-  // S3 website mode is a specialization of S3
-  const bool s3website_enabled = apis_map.count("s3website") > 0;
-  const bool sts_enabled = apis_map.count("sts") > 0;
-  const bool iam_enabled = apis_map.count("iam") > 0;
-  const bool pubsub_enabled = apis_map.count("pubsub") > 0 || apis_map.count("notifications") > 0;
-  // Swift API entrypoint could placed in the root instead of S3
-  const bool swift_at_root = g_conf()->rgw_swift_url_prefix == "/";
-  if (apis_map.count("s3") > 0 || s3website_enabled) {
-    if (! swift_at_root) {
-      rest.register_default_mgr(set_logging(rest_filter(store, RGW_REST_S3,
-                                                        new RGWRESTMgr_S3(s3website_enabled, sts_enabled, iam_enabled, pubsub_enabled))));
-    } else {
-      derr << "Cannot have the S3 or S3 Website enabled together with "
-           << "Swift API placed in the root of hierarchy" << dendl;
-      return EINVAL;
-    }
-  }
-
-  if (pubsub_enabled) {
-#ifdef WITH_RADOSGW_AMQP_ENDPOINT
-    if (!rgw::amqp::init(cct.get())) {
-        dout(1) << "ERROR: failed to initialize AMQP manager" << dendl;
-    }
-#endif
-#ifdef WITH_RADOSGW_KAFKA_ENDPOINT
-    if (!rgw::kafka::init(cct.get())) {
-        dout(1) << "ERROR: failed to initialize Kafka manager" << dendl;
-    }
-#endif
-  }
-
-  const auto& luarocks_path = g_conf().get_val<std::string>("rgw_luarocks_location");
-  if (luarocks_path.empty()) {
-    store->set_luarocks_path("");
-  } else {
-    store->set_luarocks_path(luarocks_path+"/"+g_conf()->name.to_str());
-  }
-#ifdef WITH_RADOSGW_LUA_PACKAGES
-  rgw::lua::packages_t failed_packages;
-  std::string output;
-  r = rgw::lua::install_packages(&dp, store, null_yield, failed_packages, output);
-  if (r < 0) {
-    dout(1) << "WARNING: failed to install lua packages from allowlist" << dendl;
-  }
-  if (!output.empty()) {
-    dout(10) << "INFO: lua packages installation output: \n" << output << dendl; 
-  }
-  for (const auto& p : failed_packages) {
-    dout(5) << "WARNING: failed to install lua package: " << p << " from allowlist" << dendl;
-  }
-#endif
-
-  if (apis_map.count("swift") > 0) {
-    RGWRESTMgr_SWIFT* const swift_resource = new RGWRESTMgr_SWIFT;
-
-    if (! g_conf()->rgw_cross_domain_policy.empty()) {
-      swift_resource->register_resource("crossdomain.xml",
-                          set_logging(new RGWRESTMgr_SWIFT_CrossDomain));
-    }
-
-    swift_resource->register_resource("healthcheck",
-                          set_logging(new RGWRESTMgr_SWIFT_HealthCheck));
-
-    swift_resource->register_resource("info",
-                          set_logging(new RGWRESTMgr_SWIFT_Info));
-
-    if (! swift_at_root) {
-      rest.register_resource(g_conf()->rgw_swift_url_prefix,
-                          set_logging(rest_filter(store, RGW_REST_SWIFT,
-                                                  swift_resource)));
-    } else {
-      if (store->get_zone()->get_zonegroup().get_zone_count() > 1) {
-        derr << "Placing Swift API in the root of URL hierarchy while running"
-             << " multi-site configuration requires another instance of RadosGW"
-             << " with S3 API enabled!" << dendl;
-      }
-
-      rest.register_default_mgr(set_logging(swift_resource));
-    }
-  }
-
-  if (apis_map.count("swift_auth") > 0) {
-    rest.register_resource(g_conf()->rgw_swift_auth_entry,
-               set_logging(new RGWRESTMgr_SWIFT_Auth));
-  }
-
-  if (apis_map.count("admin") > 0) {
-    RGWRESTMgr_Admin *admin_resource = new RGWRESTMgr_Admin;
-    admin_resource->register_resource("info", new RGWRESTMgr_Info);
-    admin_resource->register_resource("usage", new RGWRESTMgr_Usage);
-    admin_resource->register_resource("user", new RGWRESTMgr_User);
-  
-    /* Register store-specific admin APIs */
-    store->register_admin_apis(admin_resource);
-    rest.register_resource(g_conf()->rgw_admin_entry, admin_resource);
-  }
-
-  /* Initialize the registry of auth strategies which will coordinate
-   * the dynamic reconfiguration. */
-  rgw::auth::ImplicitTenants implicit_tenant_context{g_conf()};
-  g_conf().add_observer(&implicit_tenant_context);
-  auto auth_registry = \
-    rgw::auth::StrategyRegistry::create(g_ceph_context, implicit_tenant_context, store);
-
-  /* Header custom behavior */
-  rest.register_x_headers(g_conf()->rgw_log_http_headers);
-
-  if (cct->_conf.get_val<std::string>("rgw_scheduler_type") == "dmclock" &&
-      !cct->check_experimental_feature_enabled("dmclock")){
-    derr << "dmclock scheduler type is experimental and needs to be"
-	 << "set in the option enable experimental data corrupting features"
-	 << dendl;
-    return EINVAL;
-  }
-
-  rgw::dmclock::SchedulerCtx sched_ctx{cct.get()};
-
-  OpsLogManifold *olog = new OpsLogManifold();
-  ActiveRateLimiter ratelimiting{cct.get()};
-  ratelimiting.start();
-
-  if (!g_conf()->rgw_ops_log_socket_path.empty()) {
-    OpsLogSocket* olog_socket = new OpsLogSocket(g_ceph_context, g_conf()->rgw_ops_log_data_backlog);
-    olog_socket->init(g_conf()->rgw_ops_log_socket_path);
-    olog->add_sink(olog_socket);
-  }
-  if (!g_conf()->rgw_ops_log_file_path.empty()) {
-    ops_log_file = new OpsLogFile(g_ceph_context, g_conf()->rgw_ops_log_file_path, g_conf()->rgw_ops_log_data_backlog);
-    ops_log_file->start();
-    olog->add_sink(ops_log_file);
-  }
-  register_async_signal_handler(SIGHUP, rgw_sighup_handler);
-  olog->add_sink(new OpsLogRados(store));
-
-  r = signal_fd_init();
-  if (r < 0) {
-    derr << "ERROR: unable to initialize signal fds" << dendl;
-    exit(1);
-  }
-
-  register_async_signal_handler(SIGTERM, handle_sigterm);
-  register_async_signal_handler(SIGINT, handle_sigterm);
-  register_async_signal_handler(SIGUSR1, handle_sigterm);
-  sighandler_alrm = signal(SIGALRM, godown_alarm);
-
-  map<string, string> service_map_meta;
-  service_map_meta["pid"] = stringify(getpid());
-
-  list<RGWFrontend *> fes;
-
-  string frontend_defs_str = g_conf().get_val<string>("rgw_frontend_defaults");
-
-  list<string> frontends_def;
-  get_str_list(frontend_defs_str, ",", frontends_def);
-
-  map<string, std::unique_ptr<RGWFrontendConfig> > fe_def_map;
-  for (auto& f : frontends_def) {
-    RGWFrontendConfig *config = new RGWFrontendConfig(f);
-    int r = config->init();
-    if (r < 0) {
-      delete config;
-      cerr << "ERROR: failed to init default config: " << f << std::endl;
-      return EINVAL;
-    }
-
-    fe_def_map[config->get_framework()].reset(config);
-  }
-
-  int fe_count = 0;
-
-  std::unique_ptr<rgw::lua::Background> lua_background;
-  if (store->get_name() == "rados") { /* Supported for only RadosStore */
-    lua_background = std::make_unique<rgw::lua::Background>(store, cct.get(), store->get_luarocks_path());
-    lua_background->start();
-  }
-
-  for (multimap<string, RGWFrontendConfig *>::iterator fiter = fe_map.begin();
-       fiter != fe_map.end(); ++fiter, ++fe_count) {
-    RGWFrontendConfig *config = fiter->second;
-    string framework = config->get_framework();
-
-    auto def_iter = fe_def_map.find(framework);
-    if (def_iter != fe_def_map.end()) {
-      config->set_default_config(*def_iter->second);
-    }
-
-    RGWFrontend *fe = NULL;
-
-    if (framework == "loadgen") {
-      int port;
-      config->get_val("port", 80, &port);
-      std::string uri_prefix;
-      config->get_val("prefix", "", &uri_prefix);
-
-      RGWProcessEnv env = { store, &rest, olog, port, uri_prefix, 
-                            auth_registry, &ratelimiting, lua_background.get()};
-
-      fe = new RGWLoadGenFrontend(env, config);
-    }
-    else if (framework == "beast") {
-      int port;
-      config->get_val("port", 80, &port);
-      std::string uri_prefix;
-      config->get_val("prefix", "", &uri_prefix);
-      RGWProcessEnv env{ store, &rest, olog, port, uri_prefix, 
-                         auth_registry, &ratelimiting, lua_background.get()};
-      fe = new RGWAsioFrontend(env, config, sched_ctx);
-    }
-
-    service_map_meta["frontend_type#" + stringify(fe_count)] = framework;
-    service_map_meta["frontend_config#" + stringify(fe_count)] = config->get_config();
-
-    if (fe == NULL) {
-      dout(0) << "WARNING: skipping unknown framework: " << framework << dendl;
-      continue;
-    }
-
-    dout(0) << "starting handler: " << fiter->first << dendl;
-    int r = fe->init();
-    if (r < 0) {
-      derr << "ERROR: failed initializing frontend" << dendl;
-      return -r;
-    }
-    r = fe->run();
-    if (r < 0) {
-      derr << "ERROR: failed run" << dendl;
-      return -r;
-    }
-
-    fes.push_back(fe);
-  }
-
-  r = store->register_to_service_map(&dp, "rgw", service_map_meta);
-  if (r < 0) {
-    derr << "ERROR: failed to register to service map: " << cpp_strerror(-r) << dendl;
-
-    /* ignore error */
-  }
-
-
-  std::unique_ptr<RGWRealmReloader> reloader;
-  std::unique_ptr<RGWPeriodPusher> pusher;
-  std::unique_ptr<RGWFrontendPauser> fe_pauser;
-  std::unique_ptr<RGWRealmWatcher> realm_watcher;
-  std::unique_ptr<RGWPauser> rgw_pauser;
-  if (store->get_name() == "rados") {
-    // add a watcher to respond to realm configuration changes
-    pusher = std::make_unique<RGWPeriodPusher>(&dp, store, null_yield);
-    fe_pauser = std::make_unique<RGWFrontendPauser>(fes, implicit_tenant_context, pusher.get());
-    rgw_pauser = std::make_unique<RGWPauser>();
-    rgw_pauser->add_pauser(fe_pauser.get());
-    if (lua_background) {
-      rgw_pauser->add_pauser(lua_background.get());
-    }
-    reloader = std::make_unique<RGWRealmReloader>(store, service_map_meta, rgw_pauser.get());
-
-    realm_watcher = std::make_unique<RGWRealmWatcher>(&dp, g_ceph_context,
-				  static_cast<rgw::sal::RadosStore*>(store)->svc()->zone->get_realm());
-    realm_watcher->add_watcher(RGWRealmNotify::Reload, *reloader);
-    realm_watcher->add_watcher(RGWRealmNotify::ZonesNeedPeriod, *pusher.get());
-  }
+  main.init_ldap();
+  main.init_opslog();
+  main.init_tracepoints();
+  main.init_frontends2(nullptr /* RGWLib */);
+  main.init_notification_endpoints();
+  main.init_lua();
 
 #if defined(HAVE_SYS_PRCTL_H)
   if (prctl(PR_SET_DUMPABLE, 1) == -1) {
@@ -672,67 +166,23 @@ int main(int argc, const char **argv)
   }
 #endif
 
-  wait_shutdown();
+  rgw::signal::wait_shutdown();
 
   derr << "shutting down" << dendl;
 
-  if (store->get_name() == "rados") {
-    reloader.reset(); // stop the realm reloader
-  }
-
-  for (list<RGWFrontend *>::iterator liter = fes.begin(); liter != fes.end();
-       ++liter) {
-    RGWFrontend *fe = *liter;
-    fe->stop();
-  }
-
-  for (list<RGWFrontend *>::iterator liter = fes.begin(); liter != fes.end();
-       ++liter) {
-    RGWFrontend *fe = *liter;
-    fe->join();
-    delete fe;
-  }
-
-  for (list<RGWFrontendConfig *>::iterator liter = configs.begin();
-       liter != configs.end(); ++liter) {
-    RGWFrontendConfig *fec = *liter;
-    delete fec;
-  }
-
-  unregister_async_signal_handler(SIGHUP, rgw_sighup_handler);
-  unregister_async_signal_handler(SIGTERM, handle_sigterm);
-  unregister_async_signal_handler(SIGINT, handle_sigterm);
-  unregister_async_signal_handler(SIGUSR1, handle_sigterm);
-  shutdown_async_signal_handler();
-
-  rgw_log_usage_finalize();
-  delete olog;
-
-  if (lua_background) {
-    lua_background->shutdown();
-  }
-
-  StoreManager::close_storage(store);
-  rgw::auth::s3::LDAPEngine::shutdown();
-  rgw_tools_cleanup();
-  rgw_shutdown_resolver();
-  rgw_http_client_cleanup();
-  rgw_kmip_client_cleanup();
-  rgw::curl::cleanup_curl();
-  g_conf().remove_observer(&implicit_tenant_context);
-#ifdef WITH_RADOSGW_AMQP_ENDPOINT
-  rgw::amqp::shutdown();
-#endif
-#ifdef WITH_RADOSGW_KAFKA_ENDPOINT
-  rgw::kafka::shutdown();
-#endif
-
+  const auto finalize_async_signals = []() {
+    unregister_async_signal_handler(SIGHUP, rgw::signal::sighup_handler);
+    unregister_async_signal_handler(SIGTERM, rgw::signal::handle_sigterm);
+    unregister_async_signal_handler(SIGINT, rgw::signal::handle_sigterm);
+    unregister_async_signal_handler(SIGUSR1, rgw::signal::handle_sigterm);
+    shutdown_async_signal_handler();
+  };
 
-  rgw_perf_stop(g_ceph_context);
+  main.shutdown(finalize_async_signals);
 
   dout(1) << "final shutdown" << dendl;
 
-  signal_fd_finalize();
+  rgw::signal::signal_fd_finalize();
 
   return 0;
-}
+} /* main(int argc, char* argv[]) */
diff --git a/src/rgw/rgw_main.h b/src/rgw/rgw_main.h
new file mode 100644
index 00000000000..bcd45badf66
--- /dev/null
+++ b/src/rgw/rgw_main.h
@@ -0,0 +1,133 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab ft=cpp
+
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2022 Red Hat, Inc
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#pragma once
+
+#include <vector>
+#include <map>
+#include <string>
+#include "rgw_common.h"
+#include "rgw_rest.h"
+#include "rgw_frontend.h"
+#include "rgw_period_pusher.h"
+#include "rgw_realm_reloader.h"
+#include "rgw_ldap.h"
+#include "rgw_lua.h"
+#include "rgw_lua_background.h"
+#include "rgw_dmclock_scheduler_ctx.h"
+#include "rgw_ratelimit.h"
+
+
+class RGWPauser : public RGWRealmReloader::Pauser {
+  std::vector<Pauser*> pausers;
+
+public:
+  ~RGWPauser() override = default;
+  
+  void add_pauser(Pauser* pauser) {
+    pausers.push_back(pauser);
+  }
+
+  void pause() override {
+    std::for_each(pausers.begin(), pausers.end(), [](Pauser* p){p->pause();});
+  }
+  void resume(rgw::sal::Store* store) override {
+    std::for_each(pausers.begin(), pausers.end(), [store](Pauser* p){p->resume(store);});
+  }
+
+};
+
+namespace rgw {
+
+class RGWLib;
+class AppMain {
+  /* several components should be initalized only if librgw is
+    * also serving HTTP */
+  bool have_http_frontend{false};
+  bool nfs{false};
+
+  std::vector<RGWFrontend*> fes;
+  std::vector<RGWFrontendConfig*> fe_configs;
+  std::multimap<string, RGWFrontendConfig*> fe_map;
+  std::unique_ptr<rgw::LDAPHelper> ldh;
+  OpsLogSink* olog;
+  RGWREST rest;
+  std::unique_ptr<rgw::lua::Background> lua_background;
+  std::unique_ptr<rgw::auth::ImplicitTenants> implicit_tenant_context;
+  std::unique_ptr<rgw::dmclock::SchedulerCtx> sched_ctx;
+  std::unique_ptr<ActiveRateLimiter> ratelimiter;
+  std::map<std::string, std::string> service_map_meta;
+  // wow, realm reloader has a lot of parts
+  std::unique_ptr<RGWRealmReloader> reloader;
+  std::unique_ptr<RGWPeriodPusher> pusher;
+  std::unique_ptr<RGWFrontendPauser> fe_pauser;
+  std::unique_ptr<RGWRealmWatcher> realm_watcher;
+  std::unique_ptr<RGWPauser> rgw_pauser;
+  rgw::sal::Store* store;
+  DoutPrefixProvider* dpp;
+
+public:
+  AppMain(DoutPrefixProvider* dpp)
+    : dpp(dpp)
+    {}
+
+  void shutdown(std::function<void(void)> finalize_async_signals
+	       = []() { /* nada */});
+
+  rgw::sal::Store* get_store() {
+    return store;
+  }
+
+  rgw::LDAPHelper* get_ldh() {
+    return ldh.get();
+  }
+
+  void init_frontends1(bool nfs = false);
+  void init_numa();
+  void init_storage();
+  void init_perfcounters();
+  void init_http_clients();
+  void cond_init_apis();
+  void init_ldap();
+  void init_opslog();
+  int init_frontends2(RGWLib* rgwlib = nullptr);
+  void init_tracepoints();
+  void init_notification_endpoints();
+  void init_lua();
+
+  bool have_http() {
+    return have_http_frontend;
+  }
+
+  static OpsLogFile* ops_log_file;
+}; /* AppMain */
+} // namespace rgw
+
+static inline RGWRESTMgr *set_logging(RGWRESTMgr* mgr)
+{
+  mgr->set_logging(true);
+  return mgr;
+}
+
+static inline RGWRESTMgr *rest_filter(rgw::sal::Store* store, int dialect, RGWRESTMgr* orig)
+{
+  RGWSyncModuleInstanceRef sync_module = store->get_sync_module();
+  if (sync_module) {
+    return sync_module->get_rest_filter(dialect, orig);
+  } else {
+    return orig;
+  }
+}
+
diff --git a/src/rgw/rgw_process.h b/src/rgw/rgw_process.h
index 9fb1aaded37..85734447be5 100644
--- a/src/rgw/rgw_process.h
+++ b/src/rgw/rgw_process.h
@@ -20,7 +20,6 @@
 
 #define dout_context g_ceph_context
 
-extern void signal_shutdown();
 
 namespace rgw::dmclock {
   class Scheduler;
diff --git a/src/rgw/rgw_rest.h b/src/rgw/rgw_rest.h
index 926756402ea..0b7defa2f11 100644
--- a/src/rgw/rgw_rest.h
+++ b/src/rgw/rgw_rest.h
@@ -13,6 +13,7 @@
 #include "rgw_op.h"
 #include "rgw_formats.h"
 #include "rgw_client_io.h"
+#include "rgw_lua_background.h"
 
 extern std::map<std::string, std::string> rgw_to_http_attrs;
 
diff --git a/src/rgw/rgw_signal.cc b/src/rgw/rgw_signal.cc
new file mode 100644
index 00000000000..4bb29d0df68
--- /dev/null
+++ b/src/rgw/rgw_signal.cc
@@ -0,0 +1,91 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab ft=cpp
+
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2022 Red Hat, Inc
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "rgw_signal.h"
+#include "global/signal_handler.h"
+#include "common/safe_io.h"
+#include "common/errno.h"
+#include "rgw_main.h"
+#include "rgw_log.h"
+
+#ifdef HAVE_SYS_PRCTL_H
+#include <sys/prctl.h>
+#endif
+
+#define dout_subsys ceph_subsys_rgw
+#define dout_context g_ceph_context
+
+
+static int signal_fd[2] = {0, 0};
+
+namespace rgw {
+namespace signal {
+
+void sighup_handler(int signum) {
+    if (rgw::AppMain::ops_log_file != nullptr) {
+        rgw::AppMain::ops_log_file->reopen();
+    }
+    g_ceph_context->reopen_logs();
+} /* sighup_handler */
+
+void signal_shutdown()
+{
+  int val = 0;
+  int ret = write(signal_fd[0], (char *)&val, sizeof(val));
+  if (ret < 0) {
+    derr << "ERROR: " << __func__ << ": write() returned "
+         << cpp_strerror(errno) << dendl;
+  }
+} /* signal_shutdown */
+
+void wait_shutdown()
+{
+  int val;
+  int r = safe_read_exact(signal_fd[1], &val, sizeof(val));
+  if (r < 0) {
+    derr << "safe_read_exact returned with error" << dendl;
+  }
+} /* wait_shutdown */
+
+int signal_fd_init()
+{
+  return socketpair(AF_UNIX, SOCK_STREAM, 0, signal_fd);
+} /* signal_fd_init */
+
+void signal_fd_finalize()
+{
+  close(signal_fd[0]);
+  close(signal_fd[1]);
+} /* signal_fd_finalize */
+
+void handle_sigterm(int signum)
+{
+  dout(1) << __func__ << dendl;
+
+  // send a signal to make fcgi's accept(2) wake up.  unfortunately the
+  // initial signal often isn't sufficient because we race with accept's
+  // check of the flag wet by ShutdownPending() above.
+  if (signum != SIGUSR1) {
+    signal_shutdown();
+
+    // safety net in case we get stuck doing an orderly shutdown.
+    uint64_t secs = g_ceph_context->_conf->rgw_exit_timeout_secs;
+    if (secs)
+      alarm(secs);
+    dout(1) << __func__ << " set alarm for " << secs << dendl;
+  }
+} /* handle_sigterm */
+
+}} /* namespace rgw::signal */
diff --git a/src/rgw/rgw_signal.h b/src/rgw/rgw_signal.h
new file mode 100644
index 00000000000..68fc4f614a3
--- /dev/null
+++ b/src/rgw/rgw_signal.h
@@ -0,0 +1,31 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab ft=cpp
+
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2022 Red Hat, Inc
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#pragma once
+
+
+namespace rgw {
+namespace signal {
+
+void signal_shutdown();
+void wait_shutdown();
+int signal_fd_init();
+void signal_fd_finalize();
+void handle_sigterm(int signum);
+void handle_sigterm(int signum);
+void sighup_handler(int signum);
+
+} // namespace signal
+} // namespace rgw
diff --git a/src/test/librgw_file.cc b/src/test/librgw_file.cc
index 9dc9541acaa..328c42e6f07 100644
--- a/src/test/librgw_file.cc
+++ b/src/test/librgw_file.cc
@@ -29,7 +29,7 @@
 using namespace std;
 
 namespace {
-  librgw_t rgw = nullptr;
+  librgw_t h_rgw = nullptr;
   string uid("testuser");
   string access_key("");
   string secret_key("");
@@ -49,13 +49,13 @@ namespace {
 }
 
 TEST(LibRGW, INIT) {
-  int ret = librgw_create(&rgw, saved_args.argc, saved_args.argv);
+  int ret = librgw_create(&h_rgw, saved_args.argc, saved_args.argv);
   ASSERT_EQ(ret, 0);
-  ASSERT_NE(rgw, nullptr);
+  ASSERT_NE(h_rgw, nullptr);
 }
 
 TEST(LibRGW, MOUNT) {
-  int ret = rgw_mount2(rgw, uid.c_str(), access_key.c_str(), secret_key.c_str(),
+  int ret = rgw_mount2(h_rgw, uid.c_str(), access_key.c_str(), secret_key.c_str(),
                        "/", &fs, RGW_MOUNT_FLAG_NONE);
   ASSERT_EQ(ret, 0);
   ASSERT_NE(fs, nullptr);
@@ -173,7 +173,9 @@ TEST(LibRGW, LIST_OBJECTS) {
   }
 }
 
-extern bool global_stop;
+namespace rgw {
+  extern bool global_stop;
+}
 
 TEST(LibRGW, GETATTR_OBJECTS) {
   if (do_getattr) {
@@ -181,7 +183,7 @@ TEST(LibRGW, GETATTR_OBJECTS) {
     struct stat st;
     int ret;
 
-    global_stop = true;
+    rgw::global_stop = true;
 
     for (auto& fid_row : bucket_matrix) {
       auto& fid = get<0>(fid_row); // bucket
@@ -237,7 +239,7 @@ TEST(LibRGW, UMOUNT) {
 }
 
 TEST(LibRGW, SHUTDOWN) {
-  librgw_shutdown(rgw);
+  librgw_shutdown(h_rgw);
 }
 
 int main(int argc, char *argv[])
diff --git a/src/test/librgw_file_nfsns.cc b/src/test/librgw_file_nfsns.cc
index 3adcc2d3ec6..2ce65519c01 100644
--- a/src/test/librgw_file_nfsns.cc
+++ b/src/test/librgw_file_nfsns.cc
@@ -21,6 +21,7 @@
 #include "include/rados/librgw.h"
 #include "include/rados/rgw_file.h"
 #include "rgw/rgw_file.h"
+#include "rgw_lib.h"
 #include "rgw/rgw_lib_frontend.h" // direct requests
 
 #include "gtest/gtest.h"
@@ -31,6 +32,7 @@
 #define dout_subsys ceph_subsys_rgw
 
 using namespace std;
+using namespace rgw; // g_rgwlib
 
 namespace {
 
@@ -259,9 +261,9 @@ TEST(LibRGW, SETUP_HIER1)
 		    << std::endl;
 	}
 	RGWPutObjRequest req(cct,
-			     rgwlib.get_store()->get_user(fs_private->get_user()->user_id),
+			     g_rgwlib->get_store()->get_user(fs_private->get_user()->user_id),
 			     bucket_name, obj_name, bl);
-	int rc = rgwlib.get_fe()->execute_req(&req);
+	int rc = g_rgwlib->get_fe()->execute_req(&req);
 	int rc2 = req.get_ret();
 	ASSERT_EQ(rc, 0);
 	ASSERT_EQ(rc2, 0);