rgw/lua/doc: support reloading lua packages on all RGWs

without requiring a restart of the RGWs
test instructions:
https://gist.github.com/yuvalif/95b8ed9ea73ab4591c59644a050e01e2
also use capitalized "Lua" in logs/doc

Signed-off-by: Yuval Lifshitz <ylifshit@ibm.com>

6 files changed, 243 insertions, 46 deletions

diff --git a/src/rgw/driver/daos/rgw_sal_daos.cc b/src/rgw/driver/daos/rgw_sal_daos.cc
index ebd6957d884..46db3dd654c 100644
--- a/src/rgw/driver/daos/rgw_sal_daos.cc
+++ b/src/rgw/driver/daos/rgw_sal_daos.cc
@@ -880,8 +880,8 @@ const std::string& DaosZone::get_current_period_id() {
   return current_period->get_id();
 }
 
-std::unique_ptr<LuaManager> DaosStore::get_lua_manager() {
-  return std::make_unique<DaosLuaManager>(this);
+std::unique_ptr<LuaManager> DaosStore::get_lua_manager(const DoutPrefixProvider *dpp, const std::string& luarocks_path) {
+  return std::make_unique<DaosLuaManager>(this, dpp, luarocks_path);
 }
 
 int DaosObject::get_obj_state(const DoutPrefixProvider* dpp,
diff --git a/src/rgw/driver/daos/rgw_sal_daos.h b/src/rgw/driver/daos/rgw_sal_daos.h
index d8f1d276da2..0eaf495d2e2 100644
--- a/src/rgw/driver/daos/rgw_sal_daos.h
+++ b/src/rgw/driver/daos/rgw_sal_daos.h
@@ -997,7 +997,7 @@ class DaosStore : public StoreDriver {
   }
   virtual std::string get_host_id() { return ""; }
 
-  virtual std::unique_ptr<LuaManager> get_lua_manager() override;
+  std::unique_ptr<LuaManager> get_lua_manager(const DoutPrefixProvider *dpp = nullptr, const std::string& luarocks_path = "") override;
   virtual std::unique_ptr<RGWRole> get_role(
       std::string name, std::string tenant, std::string path = "",
       std::string trust_policy = "", std::string max_session_duration_str = "",
diff --git a/src/rgw/driver/motr/rgw_sal_motr.cc b/src/rgw/driver/motr/rgw_sal_motr.cc
index c0dff1c1f9f..06df127594e 100644
--- a/src/rgw/driver/motr/rgw_sal_motr.cc
+++ b/src/rgw/driver/motr/rgw_sal_motr.cc
@@ -1151,9 +1151,9 @@ const std::string& MotrZone::get_current_period_id()
   return current_period->get_id();
 }
 
-std::unique_ptr<LuaManager> MotrStore::get_lua_manager()
+std::unique_ptr<LuaManager> MotrStore::get_lua_manager(const DoutPrefixProvider *dpp, const std::string& luarocks_path)
 {
-  return std::make_unique<MotrLuaManager>(this);
+  return std::make_unique<MotrLuaManager>(this, dpp, luarocks_path);
 }
 
 int MotrObject::get_obj_state(const DoutPrefixProvider* dpp, RGWObjState **_state, optional_yield y, bool follow_olh)
@@ -3901,6 +3901,11 @@ int MotrStore::init_metadata_cache(const DoutPrefixProvider *dpp,
   {
     return -ENOENT;
   }
+  
+  int MotrLuaManager::reload_packages(const DoutPrefixProvider* dpp, optional_yield y)
+  {
+    return -ENOENT;
+  }
 } // namespace rgw::sal
 
 extern "C" {
diff --git a/src/rgw/driver/motr/rgw_sal_motr.h b/src/rgw/driver/motr/rgw_sal_motr.h
index c9e6fab6a0f..eee843d7eff 100644
--- a/src/rgw/driver/motr/rgw_sal_motr.h
+++ b/src/rgw/driver/motr/rgw_sal_motr.h
@@ -559,6 +559,8 @@ class MotrLuaManager : public StoreLuaManager {
   virtual int remove_package(const DoutPrefixProvider* dpp, optional_yield y, const std::string& package_name) override;
   /** List lua packages */
   virtual int list_packages(const DoutPrefixProvider* dpp, optional_yield y, rgw::lua::packages_t& packages) override;
+  /** Reload lua packages */
+  virtual int reload_packages(const DoutPrefixProvider* dpp, optional_yield y) override;
 };
 
 class MotrOIDCProvider : public RGWOIDCProvider {
@@ -1047,7 +1049,7 @@ class MotrStore : public StoreDriver {
     virtual const RGWSyncModuleInstanceRef& get_sync_module() { return sync_module; }
     virtual std::string get_host_id() { return ""; }
 
-    virtual std::unique_ptr<LuaManager> get_lua_manager() override;
+    std::unique_ptr<LuaManager> get_lua_manager(const DoutPrefixProvider *dpp = nullptr, const std::string& luarocks_path = "") override;
     virtual std::unique_ptr<RGWRole> get_role(std::string name,
         std::string tenant,
         std::string path="",
diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc
index ffbfb65dacd..0c24a36a0a5 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.cc
+++ b/src/rgw/driver/rados/rgw_sal_rados.cc
@@ -853,7 +853,7 @@ int RadosBucket::set_acl(const DoutPrefixProvider* dpp, RGWAccessControlPolicy &
     cerr << "ERROR: failed to set bucket owner: " << cpp_strerror(-r) << std::endl;
     return r;
   }
-  
+
   return 0;
 }
 
@@ -1003,8 +1003,8 @@ std::string RadosBucket::topics_oid() const {
   return pubsub_oid_prefix + get_tenant() + ".bucket." + get_name() + "/" + get_marker();
 }
 
-int RadosBucket::read_topics(rgw_pubsub_bucket_topics& notifications, 
-    RGWObjVersionTracker* objv_tracker, optional_yield y, const DoutPrefixProvider *dpp) 
+int RadosBucket::read_topics(rgw_pubsub_bucket_topics& notifications,
+    RGWObjVersionTracker* objv_tracker, optional_yield y, const DoutPrefixProvider *dpp)
 {
   // read from cache
   auto cache = store->getRados()->get_topic_cache();
@@ -1030,7 +1030,7 @@ int RadosBucket::read_topics(rgw_pubsub_bucket_topics& notifications,
   try {
     decode(notifications, iter);
   } catch (buffer::error& err) {
-    ldpp_dout(dpp, 20) << " failed to decode bucket notifications from oid: " << topics_oid() << ". for bucket: " 
+    ldpp_dout(dpp, 20) << " failed to decode bucket notifications from oid: " << topics_oid() << ". for bucket: "
       << get_name() << ". error: " << err.what() << dendl;
     return -EIO;
   }
@@ -1049,14 +1049,14 @@ int RadosBucket::write_topics(const rgw_pubsub_bucket_topics& notifications,
   encode(notifications, bl);
 
   return rgw_put_system_obj(dpp, store->svc()->sysobj,
-      store->svc()->zone->get_zone_params().log_pool, 
+      store->svc()->zone->get_zone_params().log_pool,
       topics_oid(),
       bl, false, objv_tracker, real_time(), y);
 }
 
-int RadosBucket::remove_topics(RGWObjVersionTracker* objv_tracker, 
+int RadosBucket::remove_topics(RGWObjVersionTracker* objv_tracker,
     optional_yield y, const DoutPrefixProvider *dpp) {
-  return rgw_delete_system_obj(dpp, store->svc()->sysobj, 
+  return rgw_delete_system_obj(dpp, store->svc()->sysobj,
       store->svc()->zone->get_zone_params().log_pool,
       topics_oid(),
       objv_tracker, y);
@@ -1337,7 +1337,7 @@ int RadosStore::read_topics(const std::string& tenant, rgw_pubsub_topics& topics
   try {
     decode(topics, iter);
   } catch (buffer::error& err) {
-    ldpp_dout(dpp, 20) << " failed to decode topics from oid: " << topics_oid(tenant) << 
+    ldpp_dout(dpp, 20) << " failed to decode topics from oid: " << topics_oid(tenant) <<
       ". error: " << err.what() << dendl;
     return -EIO;
   }
@@ -1351,14 +1351,14 @@ int RadosStore::write_topics(const std::string& tenant, const rgw_pubsub_topics&
   encode(topics, bl);
 
   return rgw_put_system_obj(dpp, svc()->sysobj,
-      svc()->zone->get_zone_params().log_pool, 
+      svc()->zone->get_zone_params().log_pool,
       topics_oid(tenant),
       bl, false, objv_tracker, real_time(), y);
 }
 
 int RadosStore::remove_topics(const std::string& tenant, RGWObjVersionTracker* objv_tracker,
         optional_yield y, const DoutPrefixProvider *dpp) {
-  return rgw_delete_system_obj(dpp, svc()->sysobj, 
+  return rgw_delete_system_obj(dpp, svc()->sysobj,
       svc()->zone->get_zone_params().log_pool,
       topics_oid(tenant),
       objv_tracker, y);
@@ -1515,9 +1515,9 @@ void RadosStore::register_admin_apis(RGWRESTMgr* mgr)
   mgr->register_resource("ratelimit", new RGWRESTMgr_Ratelimit);
 }
 
-std::unique_ptr<LuaManager> RadosStore::get_lua_manager()
+std::unique_ptr<LuaManager> RadosStore::get_lua_manager(const std::string& luarocks_path)
 {
-  return std::make_unique<RadosLuaManager>(this);
+  return std::make_unique<RadosLuaManager>(this, luarocks_path);
 }
 
 std::unique_ptr<RGWRole> RadosStore::get_role(std::string name,
@@ -2479,7 +2479,7 @@ int RadosMultipartUpload::abort(const DoutPrefixProvider *dpp, CephContext *cct,
   if (!remove_objs.empty()) {
     del_op->params.remove_objs = &remove_objs;
   }
-  
+
   del_op->params.abortmp = true;
   del_op->params.parts_accounted_size = parts_accounted_size;
 
@@ -2748,9 +2748,9 @@ int RadosMultipartUpload::complete(const DoutPrefixProvider *dpp,
           ldpp_dout(dpp, 0) << "ERROR: compression type was changed during multipart upload ("
                            << cs_info.compression_type << ">>" << obj_part.cs_info.compression_type << ")" << dendl;
           ret = -ERR_INVALID_PART;
-          return ret; 
+          return ret;
       }
-      
+
       if (part_compressed) {
         int64_t new_ofs; // offset in compression data for new part
         if (cs_info.blocks.size() > 0)
@@ -2764,7 +2764,7 @@ int RadosMultipartUpload::complete(const DoutPrefixProvider *dpp,
           cb.len = block.len;
           cs_info.blocks.push_back(cb);
           new_ofs = cb.new_ofs + cb.len;
-        } 
+        }
         if (!compressed)
           cs_info.compression_type = obj_part.cs_info.compression_type;
         cs_info.orig_size += obj_part.cs_info.orig_size;
@@ -3318,15 +3318,18 @@ RGWBucketSyncPolicyHandlerRef RadosZone::get_sync_policy_handler()
   return store->svc()->zone->get_sync_policy_handler(get_id());
 }
 
-RadosLuaManager::RadosLuaManager(RadosStore* _s) : 
+RadosLuaManager::RadosLuaManager(RadosStore* _s, const std::string& _luarocks_path) :
+  StoreLuaManager(_luarocks_path),
   store(_s),
-  pool((store->svc() && store->svc()->zone) ? store->svc()->zone->get_zone_params().log_pool : rgw_pool())
+  pool((store->svc() && store->svc()->zone) ? store->svc()->zone->get_zone_params().log_pool : rgw_pool()),
+  ioctx(*store->getRados()->get_lc_pool_ctx()),
+  packages_watcher(this)
 { }
 
 int RadosLuaManager::get_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, std::string& script)
 {
   if (pool.empty()) {
-    ldpp_dout(dpp, 10) << "WARNING: missing pool when reading lua script " << dendl;
+    ldpp_dout(dpp, 10) << "WARNING: missing pool when reading Lua script " << dendl;
     return 0;
   }
   bufferlist bl;
@@ -3349,7 +3352,7 @@ int RadosLuaManager::get_script(const DoutPrefixProvider* dpp, optional_yield y,
 int RadosLuaManager::put_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, const std::string& script)
 {
   if (pool.empty()) {
-    ldpp_dout(dpp, 10) << "WARNING: missing pool when writing lua script " << dendl;
+    ldpp_dout(dpp, 10) << "WARNING: missing pool when writing Lua script " << dendl;
     return 0;
   }
   bufferlist bl;
@@ -3366,7 +3369,7 @@ int RadosLuaManager::put_script(const DoutPrefixProvider* dpp, optional_yield y,
 int RadosLuaManager::del_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key)
 {
   if (pool.empty()) {
-    ldpp_dout(dpp, 10) << "WARNING: missing pool when deleting lua script " << dendl;
+    ldpp_dout(dpp, 10) << "WARNING: missing pool when deleting Lua script " << dendl;
     return 0;
   }
   int r = rgw_delete_system_obj(dpp, store->svc()->sysobj, pool, key, nullptr, y);
@@ -3381,28 +3384,31 @@ const std::string PACKAGE_LIST_OBJECT_NAME = "lua_package_allowlist";
 
 int RadosLuaManager::add_package(const DoutPrefixProvider *dpp, optional_yield y, const std::string& package_name)
 {
+  if (!ioctx.is_valid()) {
+    ldpp_dout(dpp, 10) << "WARNING: missing pool when adding Lua package" << dendl;
+    return 0;
+  }
   // add package to list
   const bufferlist empty_bl;
   std::map<std::string, bufferlist> new_package{{package_name, empty_bl}};
   librados::ObjectWriteOperation op;
   op.omap_set(new_package);
-  auto ret = rgw_rados_operate(dpp, *(store->getRados()->get_lc_pool_ctx()),
+  return rgw_rados_operate(dpp, ioctx,
       PACKAGE_LIST_OBJECT_NAME, &op, y);
-
-  if (ret < 0) {
-    return ret;
-  }
-  return 0;
 }
 
 int RadosLuaManager::remove_package(const DoutPrefixProvider *dpp, optional_yield y, const std::string& package_name)
 {
+  if (!ioctx.is_valid()) {
+    ldpp_dout(dpp, 10) << "WARNING: missing pool when removing Lua package" << dendl;
+    return -ENOENT;
+  }
   librados::ObjectWriteOperation op;
   size_t pos = package_name.find(" ");
   if (pos != package_name.npos) {
     // remove specfic version of the the package
     op.omap_rm_keys(std::set<std::string>({package_name}));
-    auto ret = rgw_rados_operate(dpp, *(store->getRados()->get_lc_pool_ctx()),
+    auto ret = rgw_rados_operate(dpp, ioctx,
         PACKAGE_LIST_OBJECT_NAME, &op, y);
     if (ret < 0) {
         return ret;
@@ -3419,7 +3425,7 @@ int RadosLuaManager::remove_package(const DoutPrefixProvider *dpp, optional_yiel
     const std::string package_no_version = package.substr(0, package.find(" "));
     if (package_no_version.compare(package_name) == 0) {
         op.omap_rm_keys(std::set<std::string>({package}));
-        ret = rgw_rados_operate(dpp, *(store->getRados()->get_lc_pool_ctx()),
+        ret = rgw_rados_operate(dpp, ioctx,
             PACKAGE_LIST_OBJECT_NAME, &op, y);
         if (ret < 0) {
             return ret;
@@ -3431,6 +3437,10 @@ int RadosLuaManager::remove_package(const DoutPrefixProvider *dpp, optional_yiel
 
 int RadosLuaManager::list_packages(const DoutPrefixProvider *dpp, optional_yield y, rgw::lua::packages_t& packages)
 {
+  if (!ioctx.is_valid()) {
+    ldpp_dout(dpp, 10) << "WARNING: missing pool when listing Lua packages" << dendl;
+    return -ENOENT;
+  }
   constexpr auto max_chunk = 1024U;
   std::string start_after;
   bool more = true;
@@ -3439,7 +3449,7 @@ int RadosLuaManager::list_packages(const DoutPrefixProvider *dpp, optional_yield
     librados::ObjectReadOperation op;
     rgw::lua::packages_t packages_chunk;
     op.omap_get_keys2(start_after, max_chunk, &packages_chunk, &more, &rval);
-    const auto ret = rgw_rados_operate(dpp, *(store->getRados()->get_lc_pool_ctx()),
+    const auto ret = rgw_rados_operate(dpp, ioctx,
       PACKAGE_LIST_OBJECT_NAME, &op, nullptr, y);
 
     if (ret < 0) {
@@ -3452,6 +3462,162 @@ int RadosLuaManager::list_packages(const DoutPrefixProvider *dpp, optional_yield
   return 0;
 }
 
+int RadosLuaManager::watch_reload(const DoutPrefixProvider* dpp)
+{
+  if (!ioctx.is_valid()) {
+    ldpp_dout(dpp, 10) << "WARNING: missing pool when watching reloads of Lua packages" << dendl;
+    return -ENOENT;
+  }
+  // create the object to watch (object may already exist)
+  librados::ObjectWriteOperation op;
+  op.create(false);
+  auto r = rgw_rados_operate(dpp, ioctx,
+      PACKAGE_LIST_OBJECT_NAME, &op, null_yield);
+  if (r < 0) {
+    ldpp_dout(dpp, 1) << "ERROR: failed to watch " << PACKAGE_LIST_OBJECT_NAME
+        << ". cannot create object. error: " << cpp_strerror(r) << dendl;
+    return r;
+  }
+  r = ioctx.watch2(PACKAGE_LIST_OBJECT_NAME, &watch_handle, &packages_watcher);
+  if (r < 0) {
+    ldpp_dout(dpp, 1) << "ERROR: failed to watch " << PACKAGE_LIST_OBJECT_NAME
+        << ". error: " << cpp_strerror(r) << dendl;
+    return r;
+  }
+  ldpp_dout(dpp, 20) << "Started watching for reloads of  " << PACKAGE_LIST_OBJECT_NAME
+    << " with handle: " << watch_handle << dendl;
+
+  return 0;
+}
+
+int RadosLuaManager::unwatch_reload(const DoutPrefixProvider* dpp)
+{
+  if (watch_handle == 0) {
+    // nothing to unwatch
+    return 0;
+  }
+
+  if (!ioctx.is_valid()) {
+    ldpp_dout(dpp, 10) << "WARNING: missing pool when unwatching reloads of Lua packages" << dendl;
+    return -ENOENT;
+  }
+  const auto r = ioctx.unwatch2(watch_handle);
+  if (r < 0) {
+    ldpp_dout(dpp, 1) << "ERROR: failed to unwatch " << PACKAGE_LIST_OBJECT_NAME
+        << ". error: " << cpp_strerror(r) << dendl;
+    return r;
+  }
+  ldpp_dout(dpp, 20) << "Stopped watching for reloads of " << PACKAGE_LIST_OBJECT_NAME
+    << " with handle: " << watch_handle << dendl;
+
+  return 0;
+}
+
+void RadosLuaManager::ack_reload(const DoutPrefixProvider* dpp, uint64_t notify_id, uint64_t cookie, int reload_status) {
+  if (!ioctx.is_valid()) {
+    ldpp_dout(dpp, 10) << "WARNING: missing pool when acking reload of Lua packages" << dendl;
+    return;
+  }
+  bufferlist reply;
+  ceph::encode(reload_status, reply);
+  ioctx.notify_ack(PACKAGE_LIST_OBJECT_NAME, notify_id, cookie, reply);
+}
+
+void RadosLuaManager::handle_reload_notify(const DoutPrefixProvider* dpp, optional_yield y, uint64_t notify_id, uint64_t cookie) {
+  if (cookie != watch_handle) {
+    return;
+  }
+
+  rgw::lua::packages_t failed_packages;
+  std::string install_dir;
+  auto r = rgw::lua::install_packages(dpp, store, 
+      y, store->ctx()->_conf.get_val<std::string>("rgw_luarocks_location"), 
+      failed_packages, install_dir);
+  if (r < 0) {
+    ldpp_dout(dpp, 1) << "WARNING: failed to install Lua packages from allowlist. error code: " << r
+            << dendl;
+  }
+  set_luarocks_path(install_dir);
+  for (const auto &p : failed_packages) {
+    ldpp_dout(dpp, 5) << "WARNING: failed to install Lua package: " << p
+            << " from allowlist" << dendl;
+  }
+  
+  ack_reload(dpp, notify_id, cookie, r);
+}
+
+int RadosLuaManager::reload_packages(const DoutPrefixProvider *dpp, optional_yield y)
+{
+  if (!ioctx.is_valid()) {
+    ldpp_dout(dpp, 10) << "WARNING: missing pool trying to notify reload of Lua packages" << dendl;
+    return -ENOENT;
+  }
+  bufferlist empty_bl;
+  bufferlist reply_bl;
+  const uint64_t timeout_ms = 0;
+  auto r = rgw_rados_notify(dpp,
+      ioctx,
+      PACKAGE_LIST_OBJECT_NAME,
+      empty_bl, timeout_ms, &reply_bl, y);
+  if (r < 0) {
+    ldpp_dout(dpp, 1) << "ERROR: failed to notify reload on " << PACKAGE_LIST_OBJECT_NAME
+        << ". error: " << cpp_strerror(r) << dendl;
+    return r;
+  }
+ 
+  std::vector<librados::notify_ack_t> acks;
+  std::vector<librados::notify_timeout_t> timeouts;
+  ioctx.decode_notify_response(reply_bl, &acks, &timeouts);
+  if (timeouts.size() > 0) {
+    ldpp_dout(dpp, 1) << "ERROR: failed to notify reload on " << PACKAGE_LIST_OBJECT_NAME
+      << ". error: timeout" << dendl;
+    return -EAGAIN;
+  }
+  for (auto& ack : acks) {
+    try {
+      auto iter = ack.payload_bl.cbegin();
+      ceph::decode(r, iter);
+    } catch (buffer::error& err) {
+      ldpp_dout(dpp, 1) << "ERROR: couldn't decode Lua packages reload status. error: " << 
+        err.what() << dendl;
+      return -EINVAL;
+    }
+    if (r < 0) {
+      return r;
+    }
+  }
+
+  return 0;
+}
+
+void RadosLuaManager::PackagesWatcher::handle_notify(uint64_t notify_id, uint64_t cookie, uint64_t notifier_id, bufferlist &bl)
+{
+  parent->handle_reload_notify(this, null_yield, notify_id, cookie);
+}
+
+void RadosLuaManager::PackagesWatcher::handle_error(uint64_t cookie, int err)
+{
+  if (parent->watch_handle != cookie) {
+    return;
+  }
+  ldpp_dout(this, 5) << "WARNING: restarting reload watch handler. error: " << err << dendl;
+
+  parent->unwatch_reload(this);
+  parent->watch_reload(this);
+}
+
+CephContext* RadosLuaManager::PackagesWatcher::get_cct() const {
+  return parent->store->ctx();
+}
+
+unsigned RadosLuaManager::PackagesWatcher::get_subsys() const {
+  return dout_subsys;
+}
+
+std::ostream& RadosLuaManager::PackagesWatcher::gen_prefix(std::ostream& out) const {
+  return out << "rgw lua package reloader: ";
+}
+
 int RadosOIDCProvider::store_url(const DoutPrefixProvider *dpp, const std::string& url, bool exclusive, optional_yield y)
 {
   auto sysobj = store->svc()->sysobj;
diff --git a/src/rgw/driver/rados/rgw_sal_rados.h b/src/rgw/driver/rados/rgw_sal_rados.h
index 75d4be843de..228ba532869 100644
--- a/src/rgw/driver/rados/rgw_sal_rados.h
+++ b/src/rgw/driver/rados/rgw_sal_rados.h
@@ -198,7 +198,7 @@ class RadosStore : public StoreDriver {
     virtual int meta_remove(const DoutPrefixProvider* dpp, std::string& metadata_key, optional_yield y) override;
     virtual const RGWSyncModuleInstanceRef& get_sync_module() { return rados->get_sync_module(); }
     virtual std::string get_host_id() { return rados->host_id; }
-    virtual std::unique_ptr<LuaManager> get_lua_manager() override;
+    std::unique_ptr<LuaManager> get_lua_manager(const std::string& luarocks_path) override;
     virtual std::unique_ptr<RGWRole> get_role(std::string name,
 					      std::string tenant,
 					      std::string path="",
@@ -889,19 +889,43 @@ public:
 };
 
 class RadosLuaManager : public StoreLuaManager {
+  class PackagesWatcher : public librados::WatchCtx2, public DoutPrefixProvider {
+    RadosLuaManager* const parent;
+  public:
+    PackagesWatcher(RadosLuaManager* _parent) : 
+      parent(_parent) {}
+    ~PackagesWatcher() override = default;
+    void handle_notify(uint64_t notify_id, uint64_t cookie,
+                   uint64_t notifier_id, bufferlist& bl) override;
+    void handle_error(uint64_t cookie, int err) override;
+   
+    // DoutPrefixProvider iterface
+    CephContext* get_cct() const override;
+    unsigned get_subsys() const override;
+    std::ostream& gen_prefix(std::ostream& out) const override;
+  };
+
   RadosStore* const store;
   rgw_pool pool;
+  librados::IoCtx& ioctx;
+  PackagesWatcher packages_watcher;
+  void ack_reload(const DoutPrefixProvider* dpp, uint64_t notify_id, uint64_t cookie, int reload_status);
+  void handle_reload_notify(const DoutPrefixProvider* dpp, optional_yield y, uint64_t notify_id, uint64_t cookie);
+  uint64_t watch_handle = 0;
 
 public:
-  RadosLuaManager(RadosStore* _s);
-  virtual ~RadosLuaManager() = default;
-
-  virtual int get_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, std::string& script);
-  virtual int put_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, const std::string& script);
-  virtual int del_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key);
-  virtual int add_package(const DoutPrefixProvider* dpp, optional_yield y, const std::string& package_name);
-  virtual int remove_package(const DoutPrefixProvider* dpp, optional_yield y, const std::string& package_name);
-  virtual int list_packages(const DoutPrefixProvider* dpp, optional_yield y, rgw::lua::packages_t& packages);
+  RadosLuaManager(RadosStore* _s, const std::string& _luarocks_path);
+  ~RadosLuaManager() override = default;
+
+  int get_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, std::string& script) override;
+  int put_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, const std::string& script) override;
+  int del_script(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key) override;
+  int add_package(const DoutPrefixProvider* dpp, optional_yield y, const std::string& package_name) override;
+  int remove_package(const DoutPrefixProvider* dpp, optional_yield y, const std::string& package_name) override;
+  int list_packages(const DoutPrefixProvider* dpp, optional_yield y, rgw::lua::packages_t& packages) override;
+  int reload_packages(const DoutPrefixProvider* dpp, optional_yield y) override;
+  int watch_reload(const DoutPrefixProvider* dpp);
+  int unwatch_reload(const DoutPrefixProvider* dpp);
 };
 
 class RadosOIDCProvider : public RGWOIDCProvider {