diff options
author | Daniel Gryniewicz <dang@redhat.com> | 2019-09-12 16:57:06 +0200 |
---|---|---|
committer | Daniel Gryniewicz <dang@redhat.com> | 2019-09-26 15:37:57 +0200 |
commit | 54b470facffe8251dc37b2020e771d4d061cb2ae (patch) | |
tree | fc4f979a3637328ac64f41864e92feccbf637329 /src/rgw | |
parent | mgr/dashboard: Display the "destroyed" state in OSD list (#30514) (diff) | |
download | ceph-54b470facffe8251dc37b2020e771d4d061cb2ae.tar.xz ceph-54b470facffe8251dc37b2020e771d4d061cb2ae.zip |
Project Zipper - Bucketlist
Implement and do the initial conversion to the BucketList API for
Zipper. This invloves doing portions of the User and Bucket API as
well, so enable BucketList to be used.
This is not the final version; the API needs to be pushed further down
into RGWRados code, but this gets the API layer complete.
Signed-off-by: Daniel Gryniewicz <dang@redhat.com>
Diffstat (limited to 'src/rgw')
-rw-r--r-- | src/rgw/rgw_admin.cc | 5 | ||||
-rw-r--r-- | src/rgw/rgw_bucket.cc | 106 | ||||
-rw-r--r-- | src/rgw/rgw_bucket.h | 12 | ||||
-rw-r--r-- | src/rgw/rgw_cr_tools.cc | 7 | ||||
-rw-r--r-- | src/rgw/rgw_file.h | 24 | ||||
-rw-r--r-- | src/rgw/rgw_lc.cc | 3 | ||||
-rw-r--r-- | src/rgw/rgw_object_expirer_core.cc | 4 | ||||
-rw-r--r-- | src/rgw/rgw_op.cc | 108 | ||||
-rw-r--r-- | src/rgw/rgw_op.h | 23 | ||||
-rw-r--r-- | src/rgw/rgw_orphan.cc | 2 | ||||
-rw-r--r-- | src/rgw/rgw_pubsub.cc | 6 | ||||
-rw-r--r-- | src/rgw/rgw_rados.cc | 26 | ||||
-rw-r--r-- | src/rgw/rgw_rados.h | 6 | ||||
-rw-r--r-- | src/rgw/rgw_reshard.cc | 3 | ||||
-rw-r--r-- | src/rgw/rgw_rest_bucket.cc | 3 | ||||
-rw-r--r-- | src/rgw/rgw_rest_log.cc | 6 | ||||
-rw-r--r-- | src/rgw/rgw_rest_pubsub_common.cc | 6 | ||||
-rw-r--r-- | src/rgw/rgw_rest_s3.cc | 26 | ||||
-rw-r--r-- | src/rgw/rgw_rest_s3.h | 3 | ||||
-rw-r--r-- | src/rgw/rgw_rest_swift.cc | 37 | ||||
-rw-r--r-- | src/rgw/rgw_rest_swift.h | 10 | ||||
-rw-r--r-- | src/rgw/rgw_sal.cc | 182 | ||||
-rw-r--r-- | src/rgw/rgw_sal.h | 128 | ||||
-rw-r--r-- | src/rgw/rgw_tools.cc | 2 | ||||
-rw-r--r-- | src/rgw/rgw_user.cc | 139 |
25 files changed, 526 insertions, 351 deletions
diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc index 6e7fa46e5a1..bf47739161f 100644 --- a/src/rgw/rgw_admin.cc +++ b/src/rgw/rgw_admin.cc @@ -1207,7 +1207,7 @@ static int init_bucket(const string& tenant_name, const string& bucket_name, con auto obj_ctx = store->svc()->sysobj->init_obj_ctx(); int r; if (bucket_id.empty()) { - r = store->getRados()->get_bucket_info(obj_ctx, tenant_name, bucket_name, bucket_info, nullptr, null_yield, pattrs); + r = store->getRados()->get_bucket_info(store->svc(), tenant_name, bucket_name, bucket_info, nullptr, null_yield, pattrs); } else { string bucket_instance_id = bucket_name + ":" + bucket_id; r = store->getRados()->get_bucket_instance_info(obj_ctx, bucket_instance_id, bucket_info, NULL, pattrs, null_yield); @@ -1390,8 +1390,7 @@ int set_bucket_quota(rgw::sal::RGWRadosStore *store, int opt_cmd, { RGWBucketInfo bucket_info; map<string, bufferlist> attrs; - auto obj_ctx = store->svc()->sysobj->init_obj_ctx(); - int r = store->getRados()->get_bucket_info(obj_ctx, tenant_name, bucket_name, bucket_info, NULL, null_yield, &attrs); + int r = store->getRados()->get_bucket_info(store->svc(), tenant_name, bucket_name, bucket_info, NULL, null_yield, &attrs); if (r < 0) { cerr << "could not get bucket info for bucket=" << bucket_name << ": " << cpp_strerror(-r) << std::endl; return -r; diff --git a/src/rgw/rgw_bucket.cc b/src/rgw/rgw_bucket.cc index a1676fe6a0d..4a777607aeb 100644 --- a/src/rgw/rgw_bucket.cc +++ b/src/rgw/rgw_bucket.cc @@ -149,17 +149,15 @@ void rgw_parse_url_bucket(const string &bucket, const string& auth_tenant, */ int rgw_read_user_buckets(RGWRadosStore * store, const rgw_user& user_id, - RGWUserBuckets& buckets, + rgw::sal::RGWBucketList& buckets, const string& marker, const string& end_marker, uint64_t max, - bool need_stats, - bool *is_truncated, - uint64_t default_amount) + bool need_stats) { - return store->ctl()->user->list_buckets(user_id, marker, end_marker, - max, need_stats, &buckets, - is_truncated, default_amount); + rgw::sal::RGWRadosUser user(store, user_id); + + return user.list_buckets(marker, end_marker, max, need_stats, buckets); } int rgw_bucket_parse_bucket_instance(const string& bucket_instance, string *bucket_name, string *bucket_id, int *shard_id) @@ -255,8 +253,8 @@ static void dump_mulipart_index_results(list<rgw_obj_index_key>& objs_to_unlink, void check_bad_user_bucket_mapping(RGWRadosStore *store, const rgw_user& user_id, bool fix) { - RGWUserBuckets user_buckets; - bool is_truncated = false; + rgw::sal::RGWBucketList user_buckets; + rgw::sal::RGWRadosUser user(store, user_id); string marker; CephContext *cct = store->ctx(); @@ -264,28 +262,24 @@ void check_bad_user_bucket_mapping(RGWRadosStore *store, const rgw_user& user_id size_t max_entries = cct->_conf->rgw_list_buckets_max_chunk; do { - int ret = rgw_read_user_buckets(store, user_id, user_buckets, marker, - string(), max_entries, false, - &is_truncated); + int ret = user.list_buckets(marker, string(), max_entries, false, user_buckets); if (ret < 0) { ldout(store->ctx(), 0) << "failed to read user buckets: " << cpp_strerror(-ret) << dendl; return; } - map<string, RGWBucketEnt>& buckets = user_buckets.get_buckets(); - for (map<string, RGWBucketEnt>::iterator i = buckets.begin(); + map<string, RGWSalBucket*>& buckets = user_buckets.get_buckets(); + for (map<string, RGWSalBucket*>::iterator i = buckets.begin(); i != buckets.end(); ++i) { marker = i->first; - RGWBucketEnt& bucket_ent = i->second; - rgw_bucket& bucket = bucket_ent.bucket; + RGWSalBucket* bucket = i->second; RGWBucketInfo bucket_info; real_time mtime; - RGWSysObjectCtx obj_ctx = store->svc()->sysobj->init_obj_ctx(); - int r = store->getRados()->get_bucket_info(obj_ctx, user_id.tenant, bucket.name, bucket_info, &mtime, null_yield); + int r = store->getRados()->get_bucket_info(store->svc(), user_id.tenant, bucket->get_name(), bucket_info, &mtime, null_yield); if (r < 0) { ldout(store->ctx(), 0) << "could not get bucket info for bucket=" << bucket << dendl; continue; @@ -293,10 +287,10 @@ void check_bad_user_bucket_mapping(RGWRadosStore *store, const rgw_user& user_id rgw_bucket& actual_bucket = bucket_info.bucket; - if (actual_bucket.name.compare(bucket.name) != 0 || - actual_bucket.tenant.compare(bucket.tenant) != 0 || - actual_bucket.marker.compare(bucket.marker) != 0 || - actual_bucket.bucket_id.compare(bucket.bucket_id) != 0) { + if (actual_bucket.name.compare(bucket->get_name()) != 0 || + actual_bucket.tenant.compare(bucket->get_tenant()) != 0 || + actual_bucket.marker.compare(bucket->get_marker()) != 0 || + actual_bucket.bucket_id.compare(bucket->get_bucket_id()) != 0) { cout << "bucket info mismatch: expected " << actual_bucket << " got " << bucket << std::endl; if (fix) { cout << "fixing" << std::endl; @@ -309,7 +303,7 @@ void check_bad_user_bucket_mapping(RGWRadosStore *store, const rgw_user& user_id } } } - } while (is_truncated); + } while (user_buckets.is_truncated()); } static bool bucket_object_check_filter(const string& oid) @@ -332,18 +326,18 @@ int rgw_remove_object(RGWRadosStore *store, const RGWBucketInfo& bucket_info, co return store->getRados()->delete_obj(rctx, bucket_info, obj, bucket_info.versioning_status()); } -int rgw_remove_bucket(RGWRadosStore *store, rgw_bucket& bucket, bool delete_children, optional_yield y) +/* xxx dang */ +static int rgw_remove_bucket(RGWRadosStore *store, rgw_bucket& bucket, bool delete_children, optional_yield y) { int ret; map<RGWObjCategory, RGWStorageStats> stats; std::vector<rgw_bucket_dir_entry> objs; map<string, bool> common_prefixes; RGWBucketInfo info; - RGWSysObjectCtx obj_ctx = store->svc()->sysobj->init_obj_ctx(); string bucket_ver, master_ver; - ret = store->getRados()->get_bucket_info(obj_ctx, bucket.tenant, bucket.name, info, NULL, null_yield); + ret = store->getRados()->get_bucket_info(store->svc(), bucket.tenant, bucket.name, info, NULL, null_yield); if (ret < 0) return ret; @@ -445,12 +439,11 @@ int rgw_remove_bucket_bypass_gc(RGWRadosStore *store, rgw_bucket& bucket, map<string, bool> common_prefixes; RGWBucketInfo info; RGWObjectCtx obj_ctx(store); - RGWSysObjectCtx sysobj_ctx = store->svc()->sysobj->init_obj_ctx(); CephContext *cct = store->ctx(); string bucket_ver, master_ver; - ret = store->getRados()->get_bucket_info(sysobj_ctx, bucket.tenant, bucket.name, info, NULL, null_yield); + ret = store->getRados()->get_bucket_info(store->svc(), bucket.tenant, bucket.name, info, NULL, null_yield); if (ret < 0) return ret; @@ -598,7 +591,6 @@ int RGWBucket::init(RGWRadosStore *storage, RGWBucketAdminOpState& op_state, rgw_user user_id = op_state.get_user_id(); bucket.tenant = user_id.tenant; bucket.name = op_state.get_bucket_name(); - RGWUserBuckets user_buckets; if (bucket.name.empty() && user_id.empty()) return -EINVAL; @@ -800,8 +792,7 @@ int RGWBucket::set_quota(RGWBucketAdminOpState& op_state, std::string *err_msg) rgw_bucket bucket = op_state.get_bucket(); RGWBucketInfo bucket_info; map<string, bufferlist> attrs; - auto obj_ctx = store->svc()->sysobj->init_obj_ctx(); - int r = store->getRados()->get_bucket_info(obj_ctx, bucket.tenant, bucket.name, bucket_info, NULL, null_yield, &attrs); + int r = store->getRados()->get_bucket_info(store->svc(), bucket.tenant, bucket.name, bucket_info, NULL, null_yield, &attrs); if (r < 0) { set_err_msg(err_msg, "could not get bucket info for bucket=" + bucket.name + ": " + cpp_strerror(-r)); return r; @@ -1147,11 +1138,10 @@ int RGWBucket::get_policy(RGWBucketAdminOpState& op_state, RGWAccessControlPolic { std::string object_name = op_state.get_object_name(); rgw_bucket bucket = op_state.get_bucket(); - auto sysobj_ctx = store->svc()->sysobj->init_obj_ctx(); RGWBucketInfo bucket_info; map<string, bufferlist> attrs; - int ret = store->getRados()->get_bucket_info(sysobj_ctx, bucket.tenant, bucket.name, bucket_info, NULL, null_yield, &attrs); + int ret = store->getRados()->get_bucket_info(store->svc(), bucket.tenant, bucket.name, bucket_info, NULL, null_yield, &attrs); if (ret < 0) { return ret; } @@ -1364,8 +1354,7 @@ static int bucket_stats(RGWRadosStore *store, const std::string& tenant_name, st map<string, bufferlist> attrs; real_time mtime; - auto obj_ctx = store->svc()->sysobj->init_obj_ctx(); - int r = store->getRados()->get_bucket_info(obj_ctx, tenant_name, bucket_name, bucket_info, &mtime, null_yield, &attrs); + int r = store->getRados()->get_bucket_info(store->svc(), tenant_name, bucket_name, bucket_info, &mtime, null_yield, &attrs); if (r < 0) return r; @@ -1450,33 +1439,32 @@ int RGWBucketAdminOp::limit_check(RGWRadosStore *store, formatter->open_array_section("buckets"); string marker; - bool is_truncated{false}; + rgw::sal::RGWBucketList buckets; do { - RGWUserBuckets buckets; + rgw::sal::RGWRadosUser user(store, rgw_user(user_id)); + + ret = user.list_buckets(marker, string(), max_entries, false, buckets); - ret = rgw_read_user_buckets(store, rgw_user(user_id), buckets, - marker, string(), max_entries, false, - &is_truncated); if (ret < 0) return ret; - map<string, RGWBucketEnt>& m_buckets = buckets.get_buckets(); + map<string, rgw::sal::RGWSalBucket*>& m_buckets = buckets.get_buckets(); for (const auto& iter : m_buckets) { - auto& bucket = iter.second.bucket; + auto bucket = iter.second; uint32_t num_shards = 1; uint64_t num_objects = 0; /* need info for num_shards */ RGWBucketInfo info; - auto obj_ctx = store->svc()->sysobj->init_obj_ctx(); - marker = bucket.name; /* Casey's location for marker update, - * as we may now not reach the end of - * the loop body */ + marker = bucket->get_name(); /* Casey's location for marker update, + * as we may now not reach the end of + * the loop body */ - ret = store->getRados()->get_bucket_info(obj_ctx, bucket.tenant, bucket.name, - info, nullptr, null_yield); + ret = store->getRados()->get_bucket_info(store->svc(), bucket->get_tenant(), + bucket->get_name(), info, nullptr, + null_yield); if (ret < 0) continue; @@ -1517,8 +1505,8 @@ int RGWBucketAdminOp::limit_check(RGWRadosStore *store, if (warn || (! warnings_only)) { formatter->open_object_section("bucket"); - formatter->dump_string("bucket", bucket.name); - formatter->dump_string("tenant", bucket.tenant); + formatter->dump_string("bucket", bucket->get_name()); + formatter->dump_string("tenant", bucket->get_tenant()); formatter->dump_int("num_objects", num_objects); formatter->dump_int("num_shards", num_shards); formatter->dump_int("objects_per_shard", objs_per_shard); @@ -1528,7 +1516,7 @@ int RGWBucketAdminOp::limit_check(RGWRadosStore *store, } } formatter->flush(cout); - } while (is_truncated); /* foreach: bucket */ + } while (buckets.is_truncated()); /* foreach: bucket */ formatter->close_section(); formatter->close_section(); @@ -1559,19 +1547,18 @@ int RGWBucketAdminOp::info(RGWRadosStore *store, RGWBucketAdminOpState& op_state if (op_state.is_user_op()) { formatter->open_array_section("buckets"); - RGWUserBuckets buckets; + rgw::sal::RGWBucketList buckets; + rgw::sal::RGWRadosUser user(store, op_state.get_user_id()); string marker; bool is_truncated = false; do { - ret = rgw_read_user_buckets(store, op_state.get_user_id(), buckets, - marker, string(), max_entries, false, - &is_truncated); + ret = user.list_buckets(marker, string(), max_entries, false, buckets); if (ret < 0) return ret; - map<string, RGWBucketEnt>& m = buckets.get_buckets(); - map<string, RGWBucketEnt>::iterator iter; + map<string, RGWSalBucket*>& m = buckets.get_buckets(); + map<string, RGWSalBucket*>::iterator iter; for (iter = m.begin(); iter != m.end(); ++iter) { std::string obj_name = iter->first; @@ -1695,7 +1682,7 @@ void get_stale_instances(RGWRadosStore *store, const std::string& bucket_name, // all the instances auto [tenant, bucket] = split_tenant(bucket_name); RGWBucketInfo cur_bucket_info; - int r = store->getRados()->get_bucket_info(obj_ctx, tenant, bucket, cur_bucket_info, nullptr, null_yield); + int r = store->getRados()->get_bucket_info(store->svc(), tenant, bucket, cur_bucket_info, nullptr, null_yield); if (r < 0) { if (r == -ENOENT) { // bucket doesn't exist, everything is stale then @@ -1841,10 +1828,9 @@ static int fix_single_bucket_lc(rgw::sal::RGWRadosStore *store, const std::string& tenant_name, const std::string& bucket_name) { - auto obj_ctx = store->svc()->sysobj->init_obj_ctx(); RGWBucketInfo bucket_info; map <std::string, bufferlist> bucket_attrs; - int ret = store->getRados()->get_bucket_info(obj_ctx, tenant_name, bucket_name, + int ret = store->getRados()->get_bucket_info(store->svc(), tenant_name, bucket_name, bucket_info, nullptr, null_yield, &bucket_attrs); if (ret < 0) { // TODO: Should we handle the case where the bucket could've been removed between diff --git a/src/rgw/rgw_bucket.h b/src/rgw/rgw_bucket.h index cf9ae2038d4..94826b6315c 100644 --- a/src/rgw/rgw_bucket.h +++ b/src/rgw/rgw_bucket.h @@ -26,7 +26,10 @@ class RGWSI_Meta; class RGWBucketMetadataHandler; class RGWBucketInstanceMetadataHandler; class RGWUserCtl; -namespace rgw { namespace sal { class RGWRadosStore; } } +namespace rgw { namespace sal { + class RGWRadosStore; + class RGWBucketList; +} } extern int rgw_bucket_parse_bucket_instance(const string& bucket_instance, string *bucket_name, string *bucket_id, int *shard_id); extern int rgw_bucket_parse_bucket_key(CephContext *cct, const string& key, @@ -211,16 +214,13 @@ public: */ extern int rgw_read_user_buckets(rgw::sal::RGWRadosStore *store, const rgw_user& user_id, - RGWUserBuckets& buckets, + rgw::sal::RGWBucketList& buckets, const string& marker, const string& end_marker, uint64_t max, - bool need_stats, - bool* is_truncated, - uint64_t default_amount = 1000); + bool need_stats); extern int rgw_remove_object(rgw::sal::RGWRadosStore *store, const RGWBucketInfo& bucket_info, const rgw_bucket& bucket, rgw_obj_key& key); -extern int rgw_remove_bucket(rgw::sal::RGWRadosStore *store, rgw_bucket& bucket, bool delete_children, optional_yield y); extern int rgw_remove_bucket_bypass_gc(rgw::sal::RGWRadosStore *store, rgw_bucket& bucket, int concurrent_max, optional_yield y); extern int rgw_object_get_attr(rgw::sal::RGWRadosStore* store, const RGWBucketInfo& bucket_info, diff --git a/src/rgw/rgw_cr_tools.cc b/src/rgw/rgw_cr_tools.cc index 300201ae9b5..b2ced7da5c5 100644 --- a/src/rgw/rgw_cr_tools.cc +++ b/src/rgw/rgw_cr_tools.cc @@ -98,8 +98,7 @@ int RGWGetUserInfoCR::Request::_send_request() template<> int RGWGetBucketInfoCR::Request::_send_request() { - RGWSysObjectCtx obj_ctx(store->svc()->sysobj->init_obj_ctx()); - return store->getRados()->get_bucket_info(obj_ctx, params.tenant, params.bucket_name, + return store->getRados()->get_bucket_info(store->svc(), params.tenant, params.bucket_name, result->bucket_info, &result->mtime, null_yield, &result->attrs); } @@ -108,7 +107,6 @@ int RGWBucketCreateLocalCR::Request::_send_request() { CephContext *cct = store->ctx(); auto& zone_svc = store->svc()->zone; - auto& sysobj_svc = store->svc()->sysobj; const auto& user_info = params.user_info.get(); const auto& user = user_info->user_id; @@ -125,11 +123,10 @@ int RGWBucketCreateLocalCR::Request::_send_request() /* we need to make sure we read bucket info, it's not read before for this * specific request */ - RGWSysObjectCtx sysobj_ctx(sysobj_svc->init_obj_ctx()); RGWBucketInfo bucket_info; map<string, bufferlist> bucket_attrs; - int ret = store->getRados()->get_bucket_info(sysobj_ctx, user.tenant, bucket_name, + int ret = store->getRados()->get_bucket_info(store->svc(), user.tenant, bucket_name, bucket_info, nullptr, null_yield, &bucket_attrs); if (ret < 0 && ret != -ENOENT) return ret; diff --git a/src/rgw/rgw_file.h b/src/rgw/rgw_file.h index 12bca8fab00..92bcda3fcd5 100644 --- a/src/rgw/rgw_file.h +++ b/src/rgw/rgw_file.h @@ -1379,17 +1379,17 @@ public: sent_data = true; } - void send_response_data(RGWUserBuckets& buckets) override { + void send_response_data(rgw::sal::RGWBucketList& buckets) override { if (!sent_data) return; - map<string, RGWBucketEnt>& m = buckets.get_buckets(); + map<string, rgw::sal::RGWSalBucket*>& m = buckets.get_buckets(); for (const auto& iter : m) { boost::string_ref marker{iter.first}; - const RGWBucketEnt& ent = iter.second; - if (! this->operator()(ent.bucket.name, marker)) { + rgw::sal::RGWSalBucket* ent = iter.second; + if (! this->operator()(ent->get_name(), marker)) { /* caller cannot accept more */ lsubdout(cct, rgw, 5) << "ListBuckets rcb failed" - << " dirent=" << ent.bucket.name + << " dirent=" << ent->get_name() << " call count=" << ix << dendl; rcb_eof = true; @@ -2243,7 +2243,7 @@ public: } real_time get_ctime() const { - return bucket.creation_time; + return bucket->get_creation_time(); } bool only_bucket() override { return false; } @@ -2284,16 +2284,16 @@ public: } void send_response() override { - bucket.creation_time = get_state()->bucket_info.creation_time; - bs.size = bucket.size; - bs.size_rounded = bucket.size_rounded; - bs.creation_time = bucket.creation_time; - bs.num_entries = bucket.count; + bucket->get_creation_time() = get_state()->bucket_info.creation_time; + bs.size = bucket->get_size(); + bs.size_rounded = bucket->get_size_rounded(); + bs.creation_time = bucket->get_creation_time(); + bs.num_entries = bucket->get_count(); std::swap(attrs, get_state()->bucket_attrs); } bool matched() { - return (bucket.bucket.name.length() > 0); + return (bucket->get_name().length() > 0); } }; /* RGWStatBucketRequest */ diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc index 18cb5aa84ba..00abcf3020f 100644 --- a/src/rgw/rgw_lc.cc +++ b/src/rgw/rgw_lc.cc @@ -1026,13 +1026,12 @@ int RGWLC::bucket_lc_process(string& shard_id) map<string, bufferlist> bucket_attrs; string no_ns, list_versions; vector<rgw_bucket_dir_entry> objs; - auto obj_ctx = store->svc()->sysobj->init_obj_ctx(); vector<std::string> result; boost::split(result, shard_id, boost::is_any_of(":")); string bucket_tenant = result[0]; string bucket_name = result[1]; string bucket_marker = result[2]; - int ret = store->getRados()->get_bucket_info(obj_ctx, bucket_tenant, bucket_name, bucket_info, NULL, null_yield, &bucket_attrs); + int ret = store->getRados()->get_bucket_info(store->svc(), bucket_tenant, bucket_name, bucket_info, NULL, null_yield, &bucket_attrs); if (ret < 0) { ldpp_dout(this, 0) << "LC:get_bucket_info for " << bucket_name << " failed" << dendl; return ret; diff --git a/src/rgw/rgw_object_expirer_core.cc b/src/rgw/rgw_object_expirer_core.cc index 0e78788649e..739b534c629 100644 --- a/src/rgw/rgw_object_expirer_core.cc +++ b/src/rgw/rgw_object_expirer_core.cc @@ -174,8 +174,6 @@ int RGWObjectExpirer::init_bucket_info(const string& tenant_name, const string& bucket_id, RGWBucketInfo& bucket_info) { - auto obj_ctx = store->svc()->sysobj->init_obj_ctx(); - /* * XXX Here's where it gets tricky. We went to all the trouble of * punching the tenant through the objexp_hint_entry, but now we @@ -186,7 +184,7 @@ int RGWObjectExpirer::init_bucket_info(const string& tenant_name, * are ephemeral, good call encoding tenant info! */ - return store->getRados()->get_bucket_info(obj_ctx, tenant_name, bucket_name, + return store->getRados()->get_bucket_info(store->svc(), tenant_name, bucket_name, bucket_info, nullptr, null_yield, nullptr); } diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index 90279c1314a..e713272ac91 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -588,7 +588,7 @@ int rgw_build_bucket_policies(rgw::sal::RGWRadosStore* store, struct req_state* RGWBucketInfo source_info; if (s->bucket_instance_id.empty()) { - ret = store->getRados()->get_bucket_info(obj_ctx, s->src_tenant_name, s->src_bucket_name, source_info, NULL, s->yield); + ret = store->getRados()->get_bucket_info(store->svc(), s->src_tenant_name, s->src_bucket_name, source_info, NULL, s->yield); } else { ret = store->getRados()->get_bucket_instance_info(obj_ctx, s->bucket_instance_id, source_info, NULL, NULL, s->yield); } @@ -1787,7 +1787,7 @@ int RGWGetObj::handle_user_manifest(const char *prefix) if (bucket_name.compare(s->bucket.name) != 0) { map<string, bufferlist> bucket_attrs; auto obj_ctx = store->svc()->sysobj->init_obj_ctx(); - int r = store->getRados()->get_bucket_info(obj_ctx, s->user->user_id.tenant, + int r = store->getRados()->get_bucket_info(store->svc(), s->user->user_id.tenant, bucket_name, bucket_info, NULL, s->yield, &bucket_attrs); if (r < 0) { @@ -1921,7 +1921,7 @@ int RGWGetObj::handle_slo_manifest(bufferlist& bl) RGWBucketInfo bucket_info; map<string, bufferlist> bucket_attrs; auto obj_ctx = store->svc()->sysobj->init_obj_ctx(); - int r = store->getRados()->get_bucket_info(obj_ctx, s->user->user_id.tenant, + int r = store->getRados()->get_bucket_info(store->svc(), s->user->user_id.tenant, bucket_name, bucket_info, nullptr, s->yield, &bucket_attrs); if (r < 0) { @@ -2318,7 +2318,7 @@ void RGWListBuckets::execute() is_truncated = false; do { - RGWUserBuckets buckets; + rgw::sal::RGWBucketList buckets; uint64_t read_count; if (limit >= 0) { read_count = min(limit - total_count, max_buckets); @@ -2326,10 +2326,10 @@ void RGWListBuckets::execute() read_count = max_buckets; } - op_ret = rgw_read_user_buckets(store, s->user->user_id, buckets, - marker, end_marker, read_count, - should_get_stats(), &is_truncated, - get_default_max()); + rgw::sal::RGWRadosUser user(store, s->user->user_id); + + op_ret = user.list_buckets(marker, end_marker, read_count, should_get_stats(), buckets); + if (op_ret < 0) { /* hmm.. something wrong here.. the user was authenticated, so it should exist */ @@ -2346,21 +2346,21 @@ void RGWListBuckets::execute() decltype(policies_stats)::mapped_type()); } - std::map<std::string, RGWBucketEnt>& m = buckets.get_buckets(); + std::map<std::string, rgw::sal::RGWSalBucket*>& m = buckets.get_buckets(); for (const auto& kv : m) { const auto& bucket = kv.second; - global_stats.bytes_used += bucket.size; - global_stats.bytes_used_rounded += bucket.size_rounded; - global_stats.objects_count += bucket.count; + global_stats.bytes_used += bucket->get_size(); + global_stats.bytes_used_rounded += bucket->get_size_rounded(); + global_stats.objects_count += bucket->get_count(); /* operator[] still can create a new entry for storage policy seen * for first time. */ - auto& policy_stats = policies_stats[bucket.placement_rule.to_str()]; - policy_stats.bytes_used += bucket.size; - policy_stats.bytes_used_rounded += bucket.size_rounded; + auto& policy_stats = policies_stats[bucket->get_placement_rule().to_str()]; + policy_stats.bytes_used += bucket->get_size(); + policy_stats.bytes_used_rounded += bucket->get_size_rounded(); policy_stats.buckets_count++; - policy_stats.objects_count += bucket.count; + policy_stats.objects_count += bucket->get_count(); } global_stats.buckets_count += m.size(); total_count += m.size(); @@ -2373,7 +2373,7 @@ void RGWListBuckets::execute() } if (!m.empty()) { - map<string, RGWBucketEnt>::reverse_iterator riter = m.rbegin(); + map<string, rgw::sal::RGWSalBucket*>::reverse_iterator riter = m.rbegin(); marker = riter->first; handle_listing_chunk(std::move(buckets)); @@ -2464,14 +2464,13 @@ int RGWStatAccount::verify_permission() void RGWStatAccount::execute() { string marker; - bool is_truncated = false; + rgw::sal::RGWBucketList buckets; uint64_t max_buckets = s->cct->_conf->rgw_list_buckets_max_chunk; do { - RGWUserBuckets buckets; op_ret = rgw_read_user_buckets(store, s->user->user_id, buckets, marker, - string(), max_buckets, true, &is_truncated); + string(), max_buckets, true); if (op_ret < 0) { /* hmm.. something wrong here.. the user was authenticated, so it should exist */ @@ -2487,26 +2486,26 @@ void RGWStatAccount::execute() decltype(policies_stats)::mapped_type()); } - std::map<std::string, RGWBucketEnt>& m = buckets.get_buckets(); + std::map<std::string, rgw::sal::RGWSalBucket*>& m = buckets.get_buckets(); for (const auto& kv : m) { const auto& bucket = kv.second; - global_stats.bytes_used += bucket.size; - global_stats.bytes_used_rounded += bucket.size_rounded; - global_stats.objects_count += bucket.count; + global_stats.bytes_used += bucket->get_size(); + global_stats.bytes_used_rounded += bucket->get_size_rounded(); + global_stats.objects_count += bucket->get_count(); /* operator[] still can create a new entry for storage policy seen * for first time. */ - auto& policy_stats = policies_stats[bucket.placement_rule.to_str()]; - policy_stats.bytes_used += bucket.size; - policy_stats.bytes_used_rounded += bucket.size_rounded; + auto& policy_stats = policies_stats[bucket->get_placement_rule().to_str()]; + policy_stats.bytes_used += bucket->get_size(); + policy_stats.bytes_used_rounded += bucket->get_size_rounded(); policy_stats.buckets_count++; - policy_stats.objects_count += bucket.count; + policy_stats.objects_count += bucket->get_count(); } global_stats.buckets_count += m.size(); } - } while (is_truncated); + } while (buckets.is_truncated()); } int RGWGetBucketVersioning::verify_permission() @@ -2716,22 +2715,9 @@ void RGWStatBucket::execute() return; } - RGWUserBuckets buckets; - bucket.bucket = s->bucket; - buckets.add(bucket); - map<string, RGWBucketEnt>& m = buckets.get_buckets(); - op_ret = store->getRados()->update_containers_stats(m); - if (! op_ret) - op_ret = -EEXIST; - if (op_ret > 0) { - op_ret = 0; - map<string, RGWBucketEnt>::iterator iter = m.find(bucket.bucket.name); - if (iter != m.end()) { - bucket = iter->second; - } else { - op_ret = -EINVAL; - } - } + rgw::sal::RGWRadosUser user(store, s->user->user_id); + bucket = new rgw::sal::RGWRadosBucket(store, user, s->bucket); + op_ret = bucket->update_container_stats(); } int RGWListBucket::verify_permission() @@ -2790,13 +2776,7 @@ void RGWListBucket::execute() } if (need_container_stats()) { - map<string, RGWBucketEnt> m; - m[s->bucket.name] = RGWBucketEnt(); - m.begin()->second.bucket = s->bucket; - op_ret = store->getRados()->update_containers_stats(m); - if (op_ret > 0) { - bucket = m.begin()->second; - } + op_ret = bucket->update_container_stats(); } RGWRados::Bucket target(store->getRados(), s->bucket_info); @@ -2857,12 +2837,11 @@ int RGWCreateBucket::verify_permission() } if (s->user->max_buckets) { - RGWUserBuckets buckets; + rgw::sal::RGWBucketList buckets; string marker; - bool is_truncated = false; op_ret = rgw_read_user_buckets(store, s->user->user_id, buckets, marker, string(), s->user->max_buckets, - false, &is_truncated); + false); if (op_ret < 0) { return op_ret; } @@ -3120,7 +3099,7 @@ void RGWCreateBucket::execute() /* we need to make sure we read bucket info, it's not read before for this * specific request */ - op_ret = store->getRados()->get_bucket_info(*s->sysobj_ctx, s->bucket_tenant, s->bucket_name, + op_ret = store->getRados()->get_bucket_info(store->svc(), s->bucket_tenant, s->bucket_name, s->bucket_info, nullptr, s->yield, &s->bucket_attrs); if (op_ret < 0 && op_ret != -ENOENT) return; @@ -3290,7 +3269,7 @@ void RGWCreateBucket::execute() RGWBucketInfo binfo; map<string, bufferlist> battrs; - op_ret = store->getRados()->get_bucket_info(*s->sysobj_ctx, s->bucket_tenant, s->bucket_name, + op_ret = store->getRados()->get_bucket_info(store->svc(), s->bucket_tenant, s->bucket_name, binfo, nullptr, s->yield, &battrs); if (op_ret < 0) { return; @@ -4784,7 +4763,7 @@ void RGWDeleteObj::execute() obj_ctx->set_atomic(obj); bool ver_restored = false; - op_ret = store->getRados()->swift_versioning_restore(*s->sysobj_ctx, *obj_ctx, s->bucket_owner.get_id(), + op_ret = store->getRados()->swift_versioning_restore(*obj_ctx, s->bucket_owner.get_id(), s->bucket_info, obj, ver_restored, this); if (op_ret < 0) { return; @@ -4901,7 +4880,7 @@ int RGWCopyObj::verify_permission() map<string, bufferlist> src_attrs; if (s->bucket_instance_id.empty()) { - op_ret = store->getRados()->get_bucket_info(*s->sysobj_ctx, src_tenant_name, src_bucket_name, src_bucket_info, NULL, s->yield, &src_attrs); + op_ret = store->getRados()->get_bucket_info(store->svc(), src_tenant_name, src_bucket_name, src_bucket_info, NULL, s->yield, &src_attrs); } else { /* will only happen in intra region sync where the source and dest bucket is the same */ rgw_bucket b(rgw_bucket_key(src_tenant_name, src_bucket_name, s->bucket_instance_id)); @@ -4972,7 +4951,7 @@ int RGWCopyObj::verify_permission() dest_bucket_info = src_bucket_info; dest_attrs = src_attrs; } else { - op_ret = store->getRados()->get_bucket_info(*s->sysobj_ctx, dest_tenant_name, dest_bucket_name, + op_ret = store->getRados()->get_bucket_info(store->svc(), dest_tenant_name, dest_bucket_name, dest_bucket_info, nullptr, s->yield, &dest_attrs); if (op_ret < 0) { if (op_ret == -ENOENT) { @@ -6766,12 +6745,11 @@ RGWBulkUploadOp::handle_upload_path(struct req_state *s) int RGWBulkUploadOp::handle_dir_verify_permission() { if (s->user->max_buckets > 0) { - RGWUserBuckets buckets; + rgw::sal::RGWBucketList buckets; std::string marker; - bool is_truncated = false; op_ret = rgw_read_user_buckets(store, s->user->user_id, buckets, marker, std::string(), s->user->max_buckets, - false, &is_truncated); + false); if (op_ret < 0) { return op_ret; } @@ -6826,7 +6804,7 @@ int RGWBulkUploadOp::handle_dir(const boost::string_ref path) * specific request */ RGWBucketInfo binfo; std::map<std::string, ceph::bufferlist> battrs; - op_ret = store->getRados()->get_bucket_info(*dir_ctx, s->bucket_tenant, bucket_name, + op_ret = store->getRados()->get_bucket_info(store->svc(), s->bucket_tenant, bucket_name, binfo, nullptr, s->yield, &battrs); if (op_ret < 0 && op_ret != -ENOENT) { return op_ret; @@ -7020,7 +6998,7 @@ int RGWBulkUploadOp::handle_file(const boost::string_ref path, RGWBucketInfo binfo; std::map<std::string, ceph::bufferlist> battrs; ACLOwner bowner; - op_ret = store->getRados()->get_bucket_info(*s->sysobj_ctx, s->user->user_id.tenant, + op_ret = store->getRados()->get_bucket_info(store->svc(), s->user->user_id.tenant, bucket_name, binfo, nullptr, s->yield, &battrs); if (op_ret == -ENOENT) { ldpp_dout(this, 20) << "non existent directory=" << bucket_name << dendl; diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index 356a79f6c0c..38b2800118c 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -34,6 +34,7 @@ #include "rgw_common.h" #include "rgw_dmclock.h" +#include "rgw_sal.h" #include "rgw_user.h" #include "rgw_bucket.h" #include "rgw_acl.h" @@ -712,15 +713,15 @@ public: void execute() override; virtual int get_params() = 0; - virtual void handle_listing_chunk(RGWUserBuckets&& buckets) { + virtual void handle_listing_chunk(rgw::sal::RGWBucketList&& buckets) { /* The default implementation, used by e.g. S3, just generates a new * part of listing and sends it client immediately. Swift can behave * differently: when the reverse option is requested, all incoming - * instances of RGWUserBuckets are buffered and finally reversed. */ + * instances of RGWBucketList are buffered and finally reversed. */ return send_response_data(buckets); } virtual void send_response_begin(bool has_buckets) = 0; - virtual void send_response_data(RGWUserBuckets& buckets) = 0; + virtual void send_response_data(rgw::sal::RGWBucketList& buckets) = 0; virtual void send_response_end() = 0; void send_response() override {} @@ -780,7 +781,7 @@ public: class RGWListBucket : public RGWOp { protected: - RGWBucketEnt bucket; + rgw::sal::RGWSalBucket* bucket; string prefix; rgw_obj_key marker; rgw_obj_key next_marker; @@ -802,13 +803,19 @@ protected: int parse_max_keys(); public: - RGWListBucket() : list_versions(false), max(0), + RGWListBucket() : bucket(nullptr), list_versions(false), max(0), default_max(0), is_truncated(false), allow_unordered(false), shard_id(-1) {} + ~RGWListBucket() { delete bucket; } int verify_permission() override; void pre_exec() override; void execute() override; + void init(rgw::sal::RGWRadosStore *store, struct req_state *s, RGWHandler *h) override { + RGWOp::init(store, s, h); + rgw::sal::RGWRadosUser user(store, s->user->user_id); + bucket = new rgw::sal::RGWRadosBucket(store, user, s->bucket); + } virtual int get_params() = 0; void send_response() override = 0; const char* name() const override { return "list_bucket"; } @@ -937,11 +944,11 @@ public: class RGWStatBucket : public RGWOp { protected: - RGWBucketEnt bucket; + rgw::sal::RGWSalBucket* bucket; public: - RGWStatBucket() {} - ~RGWStatBucket() override {} + RGWStatBucket() : bucket(nullptr) {} + ~RGWStatBucket() override { delete bucket; } int verify_permission() override; void pre_exec() override; diff --git a/src/rgw/rgw_orphan.cc b/src/rgw/rgw_orphan.cc index 271139aa7c7..9eb21763a3f 100644 --- a/src/rgw/rgw_orphan.cc +++ b/src/rgw/rgw_orphan.cc @@ -500,7 +500,7 @@ int RGWOrphanSearch::build_linked_oids_for_bucket(const string& bucket_instance_ } RGWBucketInfo cur_bucket_info; - ret = store->getRados()->get_bucket_info(sysobj_ctx, orphan_bucket.tenant, + ret = store->getRados()->get_bucket_info(store->svc(), orphan_bucket.tenant, orphan_bucket.name, cur_bucket_info, nullptr, null_yield); if (ret < 0) { if (ret == -ENOENT) { diff --git a/src/rgw/rgw_pubsub.cc b/src/rgw/rgw_pubsub.cc index 5e3d7cc4128..6b439eb6ad1 100644 --- a/src/rgw/rgw_pubsub.cc +++ b/src/rgw/rgw_pubsub.cc @@ -755,8 +755,7 @@ int RGWUserPubSub::SubWithEvents<EventType>::list_events(const string& marker, i RGWBucketInfo bucket_info; string tenant; - RGWSysObjectCtx obj_ctx(store->svc.sysobj->init_obj_ctx()); - ret = store->get_bucket_info(obj_ctx, tenant, sub_conf.dest.bucket_name, bucket_info, nullptr, null_yield, nullptr); + ret = store->get_bucket_info(&store->svc, tenant, sub_conf.dest.bucket_name, bucket_info, nullptr, null_yield, nullptr); if (ret == -ENOENT) { list.is_truncated = false; return 0; @@ -821,8 +820,7 @@ int RGWUserPubSub::SubWithEvents<EventType>::remove_event(const string& event_id RGWBucketInfo bucket_info; string tenant; - RGWSysObjectCtx sysobj_ctx(store->svc()->sysobj->init_obj_ctx()); - ret = store->getRados()->get_bucket_info(sysobj_ctx, tenant, sub_conf.dest.bucket_name, bucket_info, nullptr, null_yield, nullptr); + ret = store->getRados()->get_bucket_info(store->svc(), tenant, sub_conf.dest.bucket_name, bucket_info, nullptr, null_yield, nullptr); if (ret < 0) { ldout(store->ctx(), 1) << "ERROR: failed to read bucket info for events bucket: bucket=" << sub_conf.dest.bucket_name << " ret=" << ret << dendl; return ret; diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index 0399a4c34b9..b1b3ae7a5d7 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -2145,8 +2145,7 @@ int RGWRados::create_bucket(const RGWUserInfo& owner, rgw_bucket& bucket, /* we need to reread the info and return it, caller will have a use for it */ RGWObjVersionTracker instance_ver = info.objv_tracker; info.objv_tracker.clear(); - auto obj_ctx = svc.sysobj->init_obj_ctx(); - r = get_bucket_info(obj_ctx, bucket.tenant, bucket.name, info, NULL, null_yield, NULL); + r = get_bucket_info(&svc, bucket.tenant, bucket.name, info, NULL, null_yield, NULL); if (r < 0) { if (r == -ENOENT) { continue; @@ -2662,9 +2661,7 @@ int RGWRados::swift_versioning_copy(RGWObjectCtx& obj_ctx, RGWBucketInfo dest_bucket_info; - auto sysobj_ctx = svc.sysobj->init_obj_ctx(); - - r = get_bucket_info(sysobj_ctx, bucket_info.bucket.tenant, bucket_info.swift_ver_location, dest_bucket_info, NULL, null_yield, NULL); + r = get_bucket_info(&svc, bucket_info.bucket.tenant, bucket_info.swift_ver_location, dest_bucket_info, NULL, null_yield, NULL); if (r < 0) { ldout(cct, 10) << "failed to read dest bucket info: r=" << r << dendl; if (r == -ENOENT) { @@ -2725,8 +2722,7 @@ int RGWRados::swift_versioning_copy(RGWObjectCtx& obj_ctx, return r; } -int RGWRados::swift_versioning_restore(RGWSysObjectCtx& sysobj_ctx, - RGWObjectCtx& obj_ctx, +int RGWRados::swift_versioning_restore(RGWObjectCtx& obj_ctx, const rgw_user& user, RGWBucketInfo& bucket_info, rgw_obj& obj, @@ -2740,7 +2736,7 @@ int RGWRados::swift_versioning_restore(RGWSysObjectCtx& sysobj_ctx, /* Bucket info of the bucket that stores previous versions of our object. */ RGWBucketInfo archive_binfo; - int ret = get_bucket_info(sysobj_ctx, bucket_info.bucket.tenant, + int ret = get_bucket_info(&svc, bucket_info.bucket.tenant, bucket_info.swift_ver_location, archive_binfo, nullptr, null_yield, nullptr); if (ret < 0) { @@ -4408,10 +4404,11 @@ int RGWRados::set_bucket_owner(rgw_bucket& bucket, ACLOwner& owner) { RGWBucketInfo info; map<string, bufferlist> attrs; - auto obj_ctx = svc.sysobj->init_obj_ctx(); int r; + auto obj_ctx = svc.sysobj->init_obj_ctx(); + if (bucket.bucket_id.empty()) { - r = get_bucket_info(obj_ctx, bucket.tenant, bucket.name, info, NULL, null_yield, &attrs); + r = get_bucket_info(&svc, bucket.tenant, bucket.name, info, NULL, null_yield, &attrs); } else { r = get_bucket_instance_info(obj_ctx, bucket, info, nullptr, &attrs, null_yield); } @@ -4447,8 +4444,7 @@ int RGWRados::set_buckets_enabled(vector<rgw_bucket>& buckets, bool enabled) RGWBucketInfo info; map<string, bufferlist> attrs; - auto obj_ctx = svc.sysobj->init_obj_ctx(); - int r = get_bucket_info(obj_ctx, bucket.tenant, bucket.name, info, NULL, null_yield, &attrs); + int r = get_bucket_info(&svc, bucket.tenant, bucket.name, info, NULL, null_yield, &attrs); if (r < 0) { ldout(cct, 0) << "NOTICE: get_bucket_info on bucket=" << bucket.name << " returned err=" << r << ", skipping bucket" << dendl; ret = r; @@ -4473,8 +4469,7 @@ int RGWRados::set_buckets_enabled(vector<rgw_bucket>& buckets, bool enabled) int RGWRados::bucket_suspended(rgw_bucket& bucket, bool *suspended) { RGWBucketInfo bucket_info; - auto obj_ctx = svc.sysobj->init_obj_ctx(); - int ret = get_bucket_info(obj_ctx, bucket.tenant, bucket.name, bucket_info, NULL, null_yield); + int ret = get_bucket_info(&svc, bucket.tenant, bucket.name, bucket_info, NULL, null_yield); if (ret < 0) { return ret; } @@ -7319,12 +7314,13 @@ int RGWRados::get_bucket_instance_info(RGWSysObjectCtx& obj_ctx, const rgw_bucke .set_bectx_params(bectx_params)); } -int RGWRados::get_bucket_info(RGWSysObjectCtx& obj_ctx, +int RGWRados::get_bucket_info(RGWServices *svc, const string& tenant, const string& bucket_name, RGWBucketInfo& info, real_time *pmtime, optional_yield y, map<string, bufferlist> *pattrs) { + auto obj_ctx = svc->sysobj->init_obj_ctx(); RGWSI_MetaBackend_CtxParams bectx_params = RGWSI_MetaBackend_CtxParams_SObj(&obj_ctx); rgw_bucket bucket; bucket.tenant = tenant; diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index d15caeecf95..e3017e785a1 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -1032,8 +1032,7 @@ public: rgw_obj& obj, /* in */ const DoutPrefixProvider *dpp, /* in/out */ optional_yield y); /* in */ - int swift_versioning_restore(RGWSysObjectCtx& sysobj_ctx, - RGWObjectCtx& obj_ctx, /* in/out */ + int swift_versioning_restore(RGWObjectCtx& obj_ctx, /* in/out */ const rgw_user& user, /* in */ RGWBucketInfo& bucket_info, /* in */ rgw_obj& obj, /* in */ @@ -1310,12 +1309,13 @@ public: int get_bucket_stats_async(RGWBucketInfo& bucket_info, int shard_id, RGWGetBucketStats_CB *cb); int put_bucket_instance_info(RGWBucketInfo& info, bool exclusive, ceph::real_time mtime, map<string, bufferlist> *pattrs); + /* xxx dang obj_ctx -> svc */ int get_bucket_instance_info(RGWSysObjectCtx& obj_ctx, const string& meta_key, RGWBucketInfo& info, ceph::real_time *pmtime, map<string, bufferlist> *pattrs, optional_yield y); int get_bucket_instance_info(RGWSysObjectCtx& obj_ctx, const rgw_bucket& bucket, RGWBucketInfo& info, ceph::real_time *pmtime, map<string, bufferlist> *pattrs, optional_yield y); static void make_bucket_entry_name(const string& tenant_name, const string& bucket_name, string& bucket_entry); - int get_bucket_info(RGWSysObjectCtx& obj_ctx, + int get_bucket_info(RGWServices *svc, const string& tenant_name, const string& bucket_name, RGWBucketInfo& info, ceph::real_time *pmtime, optional_yield y, map<string, bufferlist> *pattrs = NULL); diff --git a/src/rgw/rgw_reshard.cc b/src/rgw/rgw_reshard.cc index 6444aac07ab..dd405a2ef08 100644 --- a/src/rgw/rgw_reshard.cc +++ b/src/rgw/rgw_reshard.cc @@ -979,12 +979,11 @@ int RGWReshard::process_single_logshard(int logshard_num) ldout(store->ctx(), 20) << __func__ << " resharding " << entry.bucket_name << dendl; - auto obj_ctx = store->svc()->sysobj->init_obj_ctx(); rgw_bucket bucket; RGWBucketInfo bucket_info; map<string, bufferlist> attrs; - ret = store->getRados()->get_bucket_info(obj_ctx, entry.tenant, entry.bucket_name, + ret = store->getRados()->get_bucket_info(store->svc(), entry.tenant, entry.bucket_name, bucket_info, nullptr, null_yield, &attrs); if (ret < 0) { ldout(cct, 0) << __func__ << ": Error in get_bucket_info: " << diff --git a/src/rgw/rgw_rest_bucket.cc b/src/rgw/rgw_rest_bucket.cc index 5ee5a8eb6a5..71e5655ff5a 100644 --- a/src/rgw/rgw_rest_bucket.cc +++ b/src/rgw/rgw_rest_bucket.cc @@ -293,8 +293,7 @@ void RGWOp_Set_Bucket_Quota::execute() if (use_http_params) { RGWBucketInfo bucket_info; map<string, bufferlist> attrs; - auto obj_ctx = store->svc()->sysobj->init_obj_ctx(); - http_ret = store->getRados()->get_bucket_info(obj_ctx, uid.tenant, bucket, bucket_info, NULL, s->yield, &attrs); + http_ret = store->getRados()->get_bucket_info(store->svc(), uid.tenant, bucket, bucket_info, NULL, s->yield, &attrs); if (http_ret < 0) { return; } diff --git a/src/rgw/rgw_rest_log.cc b/src/rgw/rgw_rest_log.cc index e4101d6b52e..f8891402a8a 100644 --- a/src/rgw/rgw_rest_log.cc +++ b/src/rgw/rgw_rest_log.cc @@ -404,7 +404,7 @@ void RGWOp_BILog_List::execute() { return; } } else { /* !bucket_name.empty() */ - http_ret = store->getRados()->get_bucket_info(*s->sysobj_ctx, tenant_name, bucket_name, bucket_info, NULL, s->yield, NULL); + http_ret = store->getRados()->get_bucket_info(store->svc(), tenant_name, bucket_name, bucket_info, NULL, s->yield, NULL); if (http_ret < 0) { ldpp_dout(s, 5) << "could not get bucket info for bucket=" << bucket_name << dendl; return; @@ -497,7 +497,7 @@ void RGWOp_BILog_Info::execute() { return; } } else { /* !bucket_name.empty() */ - http_ret = store->getRados()->get_bucket_info(*s->sysobj_ctx, tenant_name, bucket_name, bucket_info, NULL, s->yield, NULL); + http_ret = store->getRados()->get_bucket_info(store->svc(), tenant_name, bucket_name, bucket_info, NULL, s->yield, NULL); if (http_ret < 0) { ldpp_dout(s, 5) << "could not get bucket info for bucket=" << bucket_name << dendl; return; @@ -561,7 +561,7 @@ void RGWOp_BILog_Delete::execute() { return; } } else { /* !bucket_name.empty() */ - http_ret = store->getRados()->get_bucket_info(*s->sysobj_ctx, tenant_name, bucket_name, bucket_info, NULL, s->yield, NULL); + http_ret = store->getRados()->get_bucket_info(store->svc(), tenant_name, bucket_name, bucket_info, NULL, s->yield, NULL); if (http_ret < 0) { ldpp_dout(s, 5) << "could not get bucket info for bucket=" << bucket_name << dendl; return; diff --git a/src/rgw/rgw_rest_pubsub_common.cc b/src/rgw/rgw_rest_pubsub_common.cc index 3b4c9636811..50f567f7fc1 100644 --- a/src/rgw/rgw_rest_pubsub_common.cc +++ b/src/rgw/rgw_rest_pubsub_common.cc @@ -149,7 +149,7 @@ int RGWPSCreateNotifOp::verify_permission() { const auto& id = s->owner.get_id(); - ret = store->getRados()->get_bucket_info(*s->sysobj_ctx, id.tenant, bucket_name, + ret = store->getRados()->get_bucket_info(store->svc(), id.tenant, bucket_name, bucket_info, nullptr, null_yield, nullptr); if (ret < 0) { ldout(s->cct, 1) << "failed to get bucket info, cannot verify ownership" << dendl; @@ -169,7 +169,7 @@ int RGWPSDeleteNotifOp::verify_permission() { return ret; } - ret = store->getRados()->get_bucket_info(*s->sysobj_ctx, s->owner.get_id().tenant, bucket_name, + ret = store->getRados()->get_bucket_info(store->svc(), s->owner.get_id().tenant, bucket_name, bucket_info, nullptr, null_yield, nullptr); if (ret < 0) { return ret; @@ -188,7 +188,7 @@ int RGWPSListNotifsOp::verify_permission() { return ret; } - ret = store->getRados()->get_bucket_info(*s->sysobj_ctx, s->owner.get_id().tenant, bucket_name, + ret = store->getRados()->get_bucket_info(store->svc(), s->owner.get_id().tenant, bucket_name, bucket_info, nullptr, null_yield, nullptr); if (ret < 0) { return ret; diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index 763bc4e8faf..b1c75fe8715 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -75,11 +75,11 @@ void list_all_buckets_end(struct req_state *s) s->formatter->close_section(); } -void dump_bucket(struct req_state *s, RGWBucketEnt& obj) +void dump_bucket(struct req_state *s, rgw::sal::RGWSalBucket& obj) { s->formatter->open_object_section("Bucket"); - s->formatter->dump_string("Name", obj.bucket.name); - dump_time(s, "CreationDate", &obj.creation_time); + s->formatter->dump_string("Name", obj.get_name()); + dump_time(s, "CreationDate", &obj.get_creation_time()); s->formatter->close_section(); } @@ -619,17 +619,17 @@ void RGWListBuckets_ObjStore_S3::send_response_begin(bool has_buckets) } } -void RGWListBuckets_ObjStore_S3::send_response_data(RGWUserBuckets& buckets) +void RGWListBuckets_ObjStore_S3::send_response_data(rgw::sal::RGWBucketList& buckets) { if (!sent_data) return; - map<string, RGWBucketEnt>& m = buckets.get_buckets(); - map<string, RGWBucketEnt>::iterator iter; + map<string, rgw::sal::RGWSalBucket*>& m = buckets.get_buckets(); + map<string, rgw::sal::RGWSalBucket*>::iterator iter; for (iter = m.begin(); iter != m.end(); ++iter) { - RGWBucketEnt obj = iter->second; - dump_bucket(s, obj); + rgw::sal::RGWSalBucket* obj = iter->second; + dump_bucket(s, *obj); } rgw_flush_formatter(s, s->formatter); } @@ -1473,10 +1473,10 @@ void RGWGetBucketWebsite_ObjStore_S3::send_response() rgw_flush_formatter_and_reset(s, s->formatter); } -static void dump_bucket_metadata(struct req_state *s, RGWBucketEnt& bucket) +static void dump_bucket_metadata(struct req_state *s, rgw::sal::RGWSalBucket* bucket) { - dump_header(s, "X-RGW-Object-Count", static_cast<long long>(bucket.count)); - dump_header(s, "X-RGW-Bytes-Used", static_cast<long long>(bucket.size)); + dump_header(s, "X-RGW-Object-Count", static_cast<long long>(bucket->get_count())); + dump_header(s, "X-RGW-Bytes-Used", static_cast<long long>(bucket->get_size())); } void RGWStatBucket_ObjStore_S3::send_response() @@ -1746,7 +1746,7 @@ int RGWPutObj_ObjStore_S3::get_params() return ret; } } - ret = store->getRados()->get_bucket_info(*s->sysobj_ctx, + ret = store->getRados()->get_bucket_info(store->svc(), copy_source_tenant_name, copy_source_bucket_name, copy_source_bucket_info, @@ -4196,7 +4196,7 @@ int RGWHandler_REST_S3Website::retarget(RGWOp* op, RGWOp** new_op) { if (!(s->prot_flags & RGW_REST_WEBSITE)) return 0; - int ret = store->getRados()->get_bucket_info(*s->sysobj_ctx, s->bucket_tenant, + int ret = store->getRados()->get_bucket_info(store->svc(), s->bucket_tenant, s->bucket_name, s->bucket_info, NULL, s->yield, &s->bucket_attrs); if (ret < 0) { diff --git a/src/rgw/rgw_rest_s3.h b/src/rgw/rgw_rest_s3.h index 97bd1db3ff0..24f7143f977 100644 --- a/src/rgw/rgw_rest_s3.h +++ b/src/rgw/rgw_rest_s3.h @@ -28,7 +28,6 @@ #include "rgw_auth.h" #include "rgw_auth_filters.h" #include "rgw_sts.h" -#include "rgw_sal.h" struct rgw_http_error { int http_ret; @@ -113,7 +112,7 @@ public: return 0; } void send_response_begin(bool has_buckets) override; - void send_response_data(RGWUserBuckets& buckets) override; + void send_response_data(rgw::sal::RGWBucketList& buckets) override; void send_response_end() override; }; diff --git a/src/rgw/rgw_rest_swift.cc b/src/rgw/rgw_rest_swift.cc index 4e894680830..41c904fd9cd 100644 --- a/src/rgw/rgw_rest_swift.cc +++ b/src/rgw/rgw_rest_swift.cc @@ -192,7 +192,7 @@ void RGWListBuckets_ObjStore_SWIFT::send_response_begin(bool has_buckets) } } -void RGWListBuckets_ObjStore_SWIFT::handle_listing_chunk(RGWUserBuckets&& buckets) +void RGWListBuckets_ObjStore_SWIFT::handle_listing_chunk(rgw::sal::RGWBucketList&& buckets) { if (wants_reversed) { /* Just store in the reversal buffer. Its content will be handled later, @@ -203,7 +203,7 @@ void RGWListBuckets_ObjStore_SWIFT::handle_listing_chunk(RGWUserBuckets&& bucket } } -void RGWListBuckets_ObjStore_SWIFT::send_response_data(RGWUserBuckets& buckets) +void RGWListBuckets_ObjStore_SWIFT::send_response_data(rgw::sal::RGWBucketList& buckets) { if (! sent_data) { return; @@ -213,22 +213,22 @@ void RGWListBuckets_ObjStore_SWIFT::send_response_data(RGWUserBuckets& buckets) * in applying the filter earlier as we really need to go through all * entries regardless of it (the headers like X-Account-Container-Count * aren't affected by specifying prefix). */ - const std::map<std::string, RGWBucketEnt>& m = buckets.get_buckets(); + const std::map<std::string, rgw::sal::RGWSalBucket*>& m = buckets.get_buckets(); for (auto iter = m.lower_bound(prefix); iter != m.end() && boost::algorithm::starts_with(iter->first, prefix); ++iter) { - dump_bucket_entry(iter->second); + dump_bucket_entry(*iter->second); } } -void RGWListBuckets_ObjStore_SWIFT::dump_bucket_entry(const RGWBucketEnt& obj) +void RGWListBuckets_ObjStore_SWIFT::dump_bucket_entry(const rgw::sal::RGWSalBucket& obj) { s->formatter->open_object_section("container"); - s->formatter->dump_string("name", obj.bucket.name); + s->formatter->dump_string("name", obj.get_name()); if (need_stats) { - s->formatter->dump_int("count", obj.count); - s->formatter->dump_int("bytes", obj.size); + s->formatter->dump_int("count", obj.get_count()); + s->formatter->dump_int("bytes", obj.get_size()); } s->formatter->close_section(); @@ -238,7 +238,7 @@ void RGWListBuckets_ObjStore_SWIFT::dump_bucket_entry(const RGWBucketEnt& obj) } } -void RGWListBuckets_ObjStore_SWIFT::send_response_data_reversed(RGWUserBuckets& buckets) +void RGWListBuckets_ObjStore_SWIFT::send_response_data_reversed(rgw::sal::RGWBucketList& buckets) { if (! sent_data) { return; @@ -248,7 +248,7 @@ void RGWListBuckets_ObjStore_SWIFT::send_response_data_reversed(RGWUserBuckets& * in applying the filter earlier as we really need to go through all * entries regardless of it (the headers like X-Account-Container-Count * aren't affected by specifying prefix). */ - std::map<std::string, RGWBucketEnt>& m = buckets.get_buckets(); + std::map<std::string, rgw::sal::RGWSalBucket*>& m = buckets.get_buckets(); auto iter = m.rbegin(); for (/* initialized above */; @@ -260,7 +260,7 @@ void RGWListBuckets_ObjStore_SWIFT::send_response_data_reversed(RGWUserBuckets& for (/* iter carried */; iter != m.rend() && boost::algorithm::starts_with(iter->first, prefix); ++iter) { - dump_bucket_entry(iter->second); + dump_bucket_entry(*iter->second); } } @@ -340,7 +340,7 @@ int RGWListBucket_ObjStore_SWIFT::get_params() } static void dump_container_metadata(struct req_state *, - const RGWBucketEnt&, + const rgw::sal::RGWSalBucket*, const RGWQuotaInfo&, const RGWBucketWebsiteConf&); @@ -450,16 +450,16 @@ next: } // RGWListBucket_ObjStore_SWIFT::send_response static void dump_container_metadata(struct req_state *s, - const RGWBucketEnt& bucket, + const rgw::sal::RGWSalBucket* bucket, const RGWQuotaInfo& quota, const RGWBucketWebsiteConf& ws_conf) { /* Adding X-Timestamp to keep align with Swift API */ dump_header(s, "X-Timestamp", utime_t(s->bucket_info.creation_time)); - dump_header(s, "X-Container-Object-Count", bucket.count); - dump_header(s, "X-Container-Bytes-Used", bucket.size); - dump_header(s, "X-Container-Bytes-Used-Actual", bucket.size_rounded); + dump_header(s, "X-Container-Object-Count", bucket->get_count()); + dump_header(s, "X-Container-Bytes-Used", bucket->get_size()); + dump_header(s, "X-Container-Bytes-Used-Actual", bucket->get_size_rounded()); if (s->object.empty()) { auto swift_policy = \ @@ -849,8 +849,7 @@ int RGWPutObj_ObjStore_SWIFT::update_slo_segment_size(rgw_slo_entry& entry) { if (bucket_name.compare(s->bucket.name) != 0) { RGWBucketInfo bucket_info; map<string, bufferlist> bucket_attrs; - auto obj_ctx = store->svc()->sysobj->init_obj_ctx(); - r = store->getRados()->get_bucket_info(obj_ctx, s->user->user_id.tenant, + r = store->getRados()->get_bucket_info(store->svc(), s->user->user_id.tenant, bucket_name, bucket_info, nullptr, s->yield, &bucket_attrs); if (r < 0) { @@ -2110,7 +2109,7 @@ void RGWFormPost::get_owner_info(const req_state* const s, /* Need to get user info of bucket owner. */ RGWBucketInfo bucket_info; - int ret = store->getRados()->get_bucket_info(*s->sysobj_ctx, + int ret = store->getRados()->get_bucket_info(store->svc(), bucket_tenant, bucket_name, bucket_info, nullptr, s->yield); if (ret < 0) { diff --git a/src/rgw/rgw_rest_swift.h b/src/rgw/rgw_rest_swift.h index 42878a1ffd1..41bff74e85f 100644 --- a/src/rgw/rgw_rest_swift.h +++ b/src/rgw/rgw_rest_swift.h @@ -39,7 +39,7 @@ class RGWListBuckets_ObjStore_SWIFT : public RGWListBuckets_ObjStore { bool need_stats; bool wants_reversed; std::string prefix; - std::vector<RGWUserBuckets> reverse_buffer; + std::vector<rgw::sal::RGWBucketList> reverse_buffer; uint64_t get_default_max() const override { return 0; @@ -53,11 +53,11 @@ public: ~RGWListBuckets_ObjStore_SWIFT() override {} int get_params() override; - void handle_listing_chunk(RGWUserBuckets&& buckets) override; + void handle_listing_chunk(rgw::sal::RGWBucketList&& buckets) override; void send_response_begin(bool has_buckets) override; - void send_response_data(RGWUserBuckets& buckets) override; - void send_response_data_reversed(RGWUserBuckets& buckets); - void dump_bucket_entry(const RGWBucketEnt& obj); + void send_response_data(rgw::sal::RGWBucketList& buckets) override; + void send_response_data_reversed(rgw::sal::RGWBucketList& buckets); + void dump_bucket_entry(const rgw::sal::RGWSalBucket& obj); void send_response_end() override; bool should_get_stats() override { return need_stats; } diff --git a/src/rgw/rgw_sal.cc b/src/rgw/rgw_sal.cc index 4c2d515837d..df40fdf7254 100644 --- a/src/rgw/rgw_sal.cc +++ b/src/rgw/rgw_sal.cc @@ -22,11 +22,48 @@ #include "common/errno.h" #include "rgw_sal.h" +#include "rgw_bucket.h" +#include "rgw_multi.h" #define dout_subsys ceph_subsys_rgw namespace rgw::sal { +int RGWRadosUser::list_buckets(const string& marker, const string& end_marker, + uint64_t max, bool need_stats, RGWBucketList &buckets) +{ + RGWUserBuckets ulist; + bool is_truncated = false; + int ret; + + ret = store->ctl()->user->list_buckets(user, marker, end_marker, max, need_stats, &ulist, + &is_truncated); + if (ret < 0) + return ret; + + buckets.set_truncated(is_truncated); + for (const auto& ent : ulist.get_buckets()) { + RGWRadosBucket *rb = new RGWRadosBucket(this->store, *this, ent.second); + buckets.add(rb); + } + + return 0; +} + +RGWBucketList::~RGWBucketList() +{ + for (auto itr = buckets.begin(); itr != buckets.end(); itr++) { + delete itr->second; + } + buckets.clear(); +} + +RGWSalBucket* RGWRadosUser::add_bucket(rgw_bucket& bucket, + ceph::real_time creation_time) +{ + return NULL; +} + RGWObject *RGWRadosBucket::create_object(const rgw_obj_key &key) { if (!object) { @@ -36,24 +73,151 @@ RGWObject *RGWRadosBucket::create_object(const rgw_obj_key &key) return object; } -RGWUser *RGWRadosStore::get_user(const rgw_user &u) +int RGWRadosBucket::remove_bucket(bool delete_children, optional_yield y) { - if (!user) { - user = new RGWRadosUser(this, u); + int ret; + map<RGWObjCategory, RGWStorageStats> stats; + std::vector<rgw_bucket_dir_entry> objs; + map<string, bool> common_prefixes; + RGWBucketInfo info; + string bucket_ver, master_ver; + + ret = get_bucket_info(info, y); + if (ret < 0) + return ret; + + ret = get_bucket_stats(info, RGW_NO_SHARD, &bucket_ver, &master_ver, stats); + if (ret < 0) + return ret; + + RGWRados::Bucket target(store->getRados(), info); + RGWRados::Bucket::List list_op(&target); + int max = 1000; + + list_op.params.list_versions = true; + list_op.params.allow_unordered = true; + + bool is_truncated = false; + do { + objs.clear(); + + ret = list_op.list_objects(max, &objs, &common_prefixes, &is_truncated, null_yield); + if (ret < 0) + return ret; + + if (!objs.empty() && !delete_children) { + lderr(store->ctx()) << "ERROR: could not remove non-empty bucket " << ent.bucket.name << dendl; + return -ENOTEMPTY; + } + + for (const auto& obj : objs) { + rgw_obj_key key(obj.key); + /* xxx dang */ + ret = rgw_remove_object(store, info, ent.bucket, key); + if (ret < 0 && ret != -ENOENT) { + return ret; + } + } + } while(is_truncated); + + string prefix, delimiter; + + ret = abort_bucket_multiparts(store, store->ctx(), info, prefix, delimiter); + if (ret < 0) { + return ret; } - return user; + ret = store->ctl()->bucket->sync_user_stats(info.owner, info); + if ( ret < 0) { + ldout(store->ctx(), 1) << "WARNING: failed sync user stats before bucket delete. ret=" << ret << dendl; + } + + RGWObjVersionTracker objv_tracker; + + // if we deleted children above we will force delete, as any that + // remain is detrius from a prior bug + ret = store->getRados()->delete_bucket(info, objv_tracker, null_yield, !delete_children); + if (ret < 0) { + lderr(store->ctx()) << "ERROR: could not remove bucket " << + ent.bucket.name << dendl; + return ret; + } + + ret = store->ctl()->bucket->unlink_bucket(info.owner, ent.bucket, null_yield, false); + if (ret < 0) { + lderr(store->ctx()) << "ERROR: unable to remove user bucket information" << dendl; + } + + return ret; } -RGWSalBucket *RGWRadosStore::create_bucket(RGWUser &u, const cls_user_bucket &b) +int RGWRadosBucket::get_bucket_info(RGWBucketInfo &info, optional_yield y) { - if (!bucket) { - bucket = new RGWRadosBucket(this, u, b); - } + return store->getRados()->get_bucket_info(store->svc(), ent.bucket.tenant, ent.bucket.name, info, + NULL, y); +} + +int RGWRadosBucket::get_bucket_stats(RGWBucketInfo& bucket_info, int shard_id, + std::string *bucket_ver, std::string *master_ver, + std::map<RGWObjCategory, RGWStorageStats>& stats, + std::string *max_marker, bool *syncstopped) +{ + return store->getRados()->get_bucket_stats(bucket_info, shard_id, bucket_ver, master_ver, stats, max_marker, syncstopped); +} + +int RGWRadosBucket::sync_user_stats(RGWBucketInfo& bucket_info) +{ + return store->ctl()->bucket->sync_user_stats(user.user, bucket_info, &ent); +} - return bucket; +int RGWRadosBucket::update_container_stats(void) +{ + int ret; + map<std::string, RGWBucketEnt> m; + + m[ent.bucket.name] = ent; + ret = store->getRados()->update_containers_stats(m); + if (!ret) + return -EEXIST; + if (ret < 0) + return ret; + + map<string, RGWBucketEnt>::iterator iter = m.find(ent.bucket.name); + if (iter == m.end()) + return -EINVAL; + + ent.count = iter->second.count; + ent.size = iter->second.size; + ent.size_rounded = iter->second.size_rounded; + ent.placement_rule = std::move(iter->second.placement_rule); + + return 0; +} + +int RGWRadosBucket::set_acl(RGWAccessControlPolicy &acl, RGWBucketInfo& bucket_info, optional_yield y) +{ + bufferlist aclbl; + + acls = acl; + acl.encode(aclbl); + + return store->ctl()->bucket->set_acl(acl.get_owner(), ent.bucket, bucket_info, aclbl, null_yield); +} + +RGWUser *RGWRadosStore::get_user(const rgw_user &u) +{ + return new RGWRadosUser(this, u); } +//RGWSalBucket *RGWRadosStore::create_bucket(RGWUser &u, const rgw_bucket &b) +//{ + //if (!bucket) { + //bucket = new RGWRadosBucket(this, u, b); + //} +// + //return bucket; +//} +// void RGWRadosStore::finalize(void) { if (rados) rados->finalize(); diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h index c472ca0ed06..bd803b741f2 100644 --- a/src/rgw/rgw_sal.h +++ b/src/rgw/rgw_sal.h @@ -25,8 +25,8 @@ namespace rgw { namespace sal { class RGWUser; class RGWSalBucket; class RGWObject; +class RGWBucketList; -typedef std::vector<RGWSalBucket> RGWBucketList; typedef std::map<string, string> RGWAttrs; class RGWStore { @@ -35,8 +35,8 @@ class RGWStore { virtual ~RGWStore() = default; virtual RGWUser* get_user(const rgw_user &u) = 0; - virtual RGWSalBucket* get_bucket(RGWUser &u, const cls_user_bucket &b) = 0; - virtual RGWSalBucket* create_bucket(RGWUser &u, const cls_user_bucket &b) = 0; + virtual RGWSalBucket* get_bucket(RGWUser &u, const rgw_bucket &b) = 0; + //virtual RGWSalBucket* create_bucket(RGWUser &u, const rgw_bucket &b) = 0; virtual RGWBucketList* list_buckets(void) = 0; virtual void finalize(void)=0; @@ -53,16 +53,24 @@ class RGWUser { RGWUser(const rgw_user &_u) : user(_u) {} virtual ~RGWUser() = default; - virtual RGWBucketList* list_buckets(void) = 0; + virtual int list_buckets(const string& marker, const string& end_marker, + uint64_t max, bool need_stats, RGWBucketList &buckets) = 0; + virtual RGWSalBucket* add_bucket(rgw_bucket& bucket, ceph::real_time creation_time) = 0; + friend class RGWSalBucket; + + std::string& get_tenant() { return user.tenant; } + /* xxx dang temporary; will be removed when User is complete */ + rgw_user& get_user() { return user; } }; class RGWSalBucket { protected: - cls_user_bucket ub; + RGWBucketEnt ent; public: - RGWSalBucket() : ub() {} - RGWSalBucket(const cls_user_bucket &_b) : ub(_b) {} + RGWSalBucket() : ent() {} + RGWSalBucket(const rgw_bucket &_b) { ent.bucket = _b; } + RGWSalBucket(const RGWBucketEnt &_e) : ent(_e) {} virtual ~RGWSalBucket() = default; virtual RGWObject* get_object(const rgw_obj_key &key) = 0; @@ -70,9 +78,60 @@ class RGWSalBucket { virtual RGWObject* create_object(const rgw_obj_key &key /* Attributes */) = 0; virtual RGWAttrs& get_attrs(void) = 0; virtual int set_attrs(RGWAttrs &attrs) = 0; - virtual int delete_bucket(void) = 0; + virtual int remove_bucket(bool delete_children, optional_yield y) = 0; virtual RGWAccessControlPolicy& get_acl(void) = 0; - virtual int set_acl(const RGWAccessControlPolicy &acl) = 0; + virtual int set_acl(RGWAccessControlPolicy &acl, RGWBucketInfo& bucket_info, optional_yield y) = 0; + virtual int get_bucket_info(RGWBucketInfo &info, optional_yield y) = 0; + virtual int get_bucket_stats(RGWBucketInfo& bucket_info, int shard_id, + std::string *bucket_ver, std::string *master_ver, + std::map<RGWObjCategory, RGWStorageStats>& stats, + std::string *max_marker = nullptr, + bool *syncstopped = nullptr) = 0; + virtual int sync_user_stats(RGWBucketInfo &info) = 0; + virtual int update_container_stats(void) = 0; + + std::string get_name() const { return ent.bucket.name; } + std::string get_tenant() const { return ent.bucket.tenant; } + std::string get_marker() const { return ent.bucket.marker; } + std::string get_bucket_id() const { return ent.bucket.bucket_id; } + size_t get_size() const { return ent.size; } + size_t get_size_rounded() const { return ent.size_rounded; } + uint64_t get_count() const { return ent.count; } + rgw_placement_rule get_placement_rule() const { return ent.placement_rule; } + ceph::real_time& get_creation_time() { return ent.creation_time; }; + + /* dang - This is temporary, until the API is completed */ + rgw_bucket& get_bi() { return ent.bucket; } + + friend inline ostream& operator<<(ostream& out, const RGWSalBucket &b) { + out << b.ent.bucket; + return out; + } + + + friend class RGWBucketList; + protected: + virtual void set_ent(RGWBucketEnt &_ent) { ent = _ent; } +}; + +class RGWBucketList { + std::map<std::string, RGWSalBucket*> buckets; + bool truncated; + +public: + RGWBucketList() : buckets(), truncated(false) {} + RGWBucketList(RGWBucketList&&) = default; + RGWBucketList& operator=(const RGWBucketList&) = default; + ~RGWBucketList(); + + map<string, RGWSalBucket*>& get_buckets() { return buckets; } + bool is_truncated(void) { return truncated; } + void set_truncated(bool trunc) { truncated = trunc; } + void add(RGWSalBucket* bucket) { + buckets[bucket->ent.bucket.name] = bucket; + } + size_t count() { return buckets.size(); } + }; class RGWObject { @@ -104,7 +163,11 @@ class RGWRadosUser : public RGWUser { RGWRadosUser(RGWRadosStore *_st, const rgw_user &_u) : RGWUser(_u), store(_st) { } RGWRadosUser() {} - RGWBucketList* list_buckets(void) { return new RGWBucketList(); } + int list_buckets(const string& marker, const string& end_marker, + uint64_t max, bool need_stats, RGWBucketList &buckets); + RGWSalBucket* add_bucket(rgw_bucket& bucket, ceph::real_time creation_time); + + friend class RGWRadosBucket; }; class RGWRadosObject : public RGWObject { @@ -145,13 +208,14 @@ class RGWRadosBucket : public RGWSalBucket { public: RGWRadosBucket() - : object(nullptr), + : store(nullptr), + object(nullptr), attrs(), acls(), user() { } - RGWRadosBucket(RGWRadosStore *_st, RGWUser &_u, const cls_user_bucket &_b) + RGWRadosBucket(RGWRadosStore *_st, RGWUser &_u, const rgw_bucket &_b) : RGWSalBucket(_b), store(_st), object(nullptr), @@ -160,40 +224,54 @@ class RGWRadosBucket : public RGWSalBucket { user(dynamic_cast<RGWRadosUser&>(_u)) { } + RGWRadosBucket(RGWRadosStore *_st, RGWUser &_u, const RGWBucketEnt &_e) + : RGWSalBucket(_e), + store(_st), + object(nullptr), + attrs(), + acls(), + user(dynamic_cast<RGWRadosUser&>(_u)) { + } + + ~RGWRadosBucket() { } + RGWObject* get_object(const rgw_obj_key &key) { return object; } RGWBucketList* list(void) { return new RGWBucketList(); } RGWObject* create_object(const rgw_obj_key &key /* Attributes */) override; RGWAttrs& get_attrs(void) { return attrs; } int set_attrs(RGWAttrs &a) { attrs = a; return 0; } - int delete_bucket(void) { return 0; } + virtual int remove_bucket(bool delete_children, optional_yield y) override; RGWAccessControlPolicy& get_acl(void) { return acls; } - int set_acl(const RGWAccessControlPolicy &acl) { acls = acl; return 0; } + virtual int set_acl(RGWAccessControlPolicy &acl, RGWBucketInfo& bucket_info, optional_yield y) override; + virtual int get_bucket_info(RGWBucketInfo &info, optional_yield y) override; + virtual int get_bucket_stats(RGWBucketInfo& bucket_info, int shard_id, + std::string *bucket_ver, std::string *master_ver, + std::map<RGWObjCategory, RGWStorageStats>& stats, + std::string *max_marker = nullptr, + bool *syncstopped = nullptr) override; + virtual int sync_user_stats(RGWBucketInfo &info) override; + virtual int update_container_stats(void) override; }; class RGWRadosStore : public RGWStore { private: RGWRados *rados; - RGWRadosUser *user; RGWRadosBucket *bucket; + RGWUserCtl *user_ctl; public: RGWRadosStore() : rados(nullptr), - user(nullptr), bucket(nullptr) { } ~RGWRadosStore() { - if (bucket) - delete bucket; - if (user) - delete user; - if (rados) - delete rados; + delete bucket; + delete rados; } virtual RGWUser* get_user(const rgw_user &u); - virtual RGWSalBucket* get_bucket(RGWUser &u, const cls_user_bucket &b) { return bucket; } - virtual RGWSalBucket* create_bucket(RGWUser &u, const cls_user_bucket &b); + virtual RGWSalBucket* get_bucket(RGWUser &u, const rgw_bucket &b) { return bucket; } + //virtual RGWSalBucket* create_bucket(RGWUser &u, const rgw_bucket &b); virtual RGWBucketList* list_buckets(void) { return new RGWBucketList(); } void setRados(RGWRados * st) { rados = st; } @@ -202,6 +280,8 @@ class RGWRadosStore : public RGWStore { RGWServices *svc() { return &rados->svc; } RGWCtl *ctl() { return &rados->ctl; } + void setUserCtl(RGWUserCtl *_ctl) { user_ctl = _ctl; } + void finalize(void) override; virtual CephContext *ctx(void) { return rados->ctx(); } diff --git a/src/rgw/rgw_tools.cc b/src/rgw/rgw_tools.cc index 7b25c982ee4..948862a591f 100644 --- a/src/rgw/rgw_tools.cc +++ b/src/rgw/rgw_tools.cc @@ -424,7 +424,7 @@ int RGWDataAccess::Bucket::finish_init() int RGWDataAccess::Bucket::init() { - int ret = sd->store->getRados()->get_bucket_info(*sd->sysobj_ctx, + int ret = sd->store->getRados()->get_bucket_info(sd->store->svc(), tenant, name, bucket_info, &mtime, diff --git a/src/rgw/rgw_user.cc b/src/rgw/rgw_user.cc index 359237c6eca..6876d2320c5 100644 --- a/src/rgw/rgw_user.cc +++ b/src/rgw/rgw_user.cc @@ -49,58 +49,48 @@ void rgw_get_anon_user(RGWUserInfo& info) int rgw_user_sync_all_stats(rgw::sal::RGWRadosStore *store, const rgw_user& user_id) { + rgw::sal::RGWBucketList user_buckets; + rgw::sal::RGWRadosUser user(store, user_id); + CephContext *cct = store->ctx(); size_t max_entries = cct->_conf->rgw_list_buckets_max_chunk; bool is_truncated = false; string marker; int ret; - RGWSysObjectCtx obj_ctx = store->svc()->sysobj->init_obj_ctx(); do { - RGWUserBuckets user_buckets; - ret = store->ctl()->user->list_buckets(user_id, - marker, string(), - max_entries, - false, - &user_buckets, - &is_truncated); + ret = user.list_buckets(marker, string(), max_entries, false, user_buckets); if (ret < 0) { ldout(cct, 0) << "failed to read user buckets: ret=" << ret << dendl; return ret; } - map<string, RGWBucketEnt>& buckets = user_buckets.get_buckets(); - for (map<string, RGWBucketEnt>::iterator i = buckets.begin(); + map<string, rgw::sal::RGWSalBucket*>& buckets = user_buckets.get_buckets(); + for (map<string, rgw::sal::RGWSalBucket*>::iterator i = buckets.begin(); i != buckets.end(); ++i) { marker = i->first; - RGWBucketEnt& bucket_ent = i->second; + rgw::sal::RGWSalBucket* bucket = i->second; RGWBucketInfo bucket_info; - rgw_bucket bucket; - bucket.tenant = user_id.tenant; - bucket.name = bucket_ent.bucket.name; - - ret = store->getRados()->get_bucket_info(obj_ctx, user_id.tenant, bucket_ent.bucket.name, - bucket_info, nullptr, null_yield, nullptr); + ret = bucket->get_bucket_info(bucket_info, null_yield); if (ret < 0) { - ldout(cct, 0) << "ERROR: could not read bucket info: bucket=" << bucket_ent.bucket << " ret=" << ret << dendl; + ldout(cct, 0) << "ERROR: could not read bucket info: bucket=" << bucket << " ret=" << ret << dendl; continue; } - RGWBucketEnt ent; - ret = store->ctl()->bucket->sync_user_stats(user_id, bucket_info, &ent); + ret = bucket->sync_user_stats(bucket_info); if (ret < 0) { ldout(cct, 0) << "ERROR: could not sync bucket stats: ret=" << ret << dendl; return ret; } - ret = store->getRados()->check_bucket_shards(bucket_info, bucket_info.bucket, ent.count); + ret = store->getRados()->check_bucket_shards(bucket_info, bucket_info.bucket, bucket->get_count()); if (ret < 0) { ldout(cct, 0) << "ERROR in check_bucket_shards: " << cpp_strerror(-ret)<< dendl; } } } while (is_truncated); - ret = store->ctl()->user->complete_flush_stats(user_id); + ret = store->ctl()->user->complete_flush_stats(user.get_user()); if (ret < 0) { cerr << "ERROR: failed to complete syncing user stats: ret=" << ret << std::endl; return ret; @@ -114,34 +104,33 @@ int rgw_user_get_all_buckets_stats(rgw::sal::RGWRadosStore *store, const rgw_use CephContext *cct = store->ctx(); size_t max_entries = cct->_conf->rgw_list_buckets_max_chunk; bool done; - bool is_truncated; string marker; int ret; do { - RGWUserBuckets user_buckets; - ret = rgw_read_user_buckets(store, user_id, user_buckets, marker, - string(), max_entries, false, &is_truncated); + rgw::sal::RGWBucketList buckets; + ret = rgw_read_user_buckets(store, user_id, buckets, marker, + string(), max_entries, false); if (ret < 0) { ldout(cct, 0) << "failed to read user buckets: ret=" << ret << dendl; return ret; } - map<string, RGWBucketEnt>& buckets = user_buckets.get_buckets(); - for (const auto& i : buckets) { + std::map<std::string, rgw::sal::RGWSalBucket*>& m = buckets.get_buckets(); + for (const auto& i : m) { marker = i.first; - const RGWBucketEnt& bucket_ent = i.second; + rgw::sal::RGWSalBucket* bucket_ent = i.second; RGWBucketEnt stats; - ret = store->ctl()->bucket->read_bucket_stats(bucket_ent.bucket, &stats, null_yield); + ret = store->ctl()->bucket->read_bucket_stats(bucket_ent->get_bi(), &stats, null_yield); if (ret < 0) { ldout(cct, 0) << "ERROR: could not get bucket stats: ret=" << ret << dendl; return ret; } cls_user_bucket_entry entry; stats.convert(&entry); - buckets_usage_map.emplace(bucket_ent.bucket.name, entry); + buckets_usage_map.emplace(bucket_ent->get_name(), entry); } - done = (buckets.size() < max_entries); + done = (buckets.count() < max_entries); } while (!done); return 0; @@ -1613,19 +1602,18 @@ int RGWUser::execute_rename(RGWUserAdminOpState& op_state, std::string *err_msg) } } - rgw_user& old_uid = op_state.get_user_id(); + rgw::sal::RGWUser* old_user = new rgw::sal::RGWRadosUser(store, op_state.get_user_id()); RGWUserInfo old_user_info = op_state.get_user_info(); - - rgw_user& uid = op_state.get_new_uid(); - if (old_uid.tenant != uid.tenant) { + rgw::sal::RGWUser* new_user = new rgw::sal::RGWRadosUser(store, op_state.get_new_uid()); + if (old_user->get_tenant() != new_user->get_tenant()) { set_err_msg(err_msg, "users have to be under the same tenant namespace " - + old_uid.tenant + " != " + uid.tenant); + + old_user->get_tenant() + " != " + new_user->get_tenant()); return -EINVAL; } // create a stub user and write only the uid index and buckets object RGWUserInfo stub_user_info; - stub_user_info.user_id = uid; + stub_user_info.user_id = new_user->get_user(); RGWObjVersionTracker objv; const bool exclusive = !op_state.get_overwrite_new_user(); // overwrite if requested @@ -1643,85 +1631,76 @@ int RGWUser::execute_rename(RGWUserAdminOpState& op_state, std::string *err_msg) return ret; } - ACLOwner owner; RGWAccessControlPolicy policy_instance; - policy_instance.create_default(uid, old_user_info.display_name); - owner = policy_instance.get_owner(); - bufferlist aclbl; - policy_instance.encode(aclbl); + policy_instance.create_default(new_user->get_user(), old_user_info.display_name); //unlink and link buckets to new user - bool is_truncated = false; string marker; string obj_marker; CephContext *cct = store->ctx(); size_t max_buckets = cct->_conf->rgw_list_buckets_max_chunk; RGWBucketCtl* bucket_ctl = store->ctl()->bucket; + rgw::sal::RGWBucketList buckets; do { - RGWUserBuckets buckets; - ret = user_ctl->list_buckets(old_uid, marker, "", max_buckets, - false, &buckets, &is_truncated); + ret = old_user->list_buckets(marker, "", max_buckets, false, buckets); if (ret < 0) { set_err_msg(err_msg, "unable to list user buckets"); return ret; } - map<string, bufferlist> attrs; - map<std::string, RGWBucketEnt>& m = buckets.get_buckets(); - std::map<std::string, RGWBucketEnt>::iterator it; + map<std::string, rgw::sal::RGWSalBucket*>& m = buckets.get_buckets(); + std::map<std::string, rgw::sal::RGWSalBucket*>::iterator it; for (it = m.begin(); it != m.end(); ++it) { - RGWBucketEnt obj = it->second; + rgw::sal::RGWSalBucket* bucket = it->second; marker = it->first; RGWBucketInfo bucket_info; - ret = bucket_ctl->read_bucket_info(obj.bucket, &bucket_info, null_yield, - RGWBucketCtl::BucketInstance::GetParams() - .set_attrs(&attrs)); + ret = bucket->get_bucket_info(bucket_info, null_yield); if (ret < 0) { - set_err_msg(err_msg, "failed to fetch bucket info for bucket=" + obj.bucket.name); + set_err_msg(err_msg, "failed to fetch bucket info for bucket=" + bucket->get_name()); return ret; } - ret = bucket_ctl->set_acl(owner, obj.bucket, bucket_info, aclbl, null_yield); + ret = bucket->set_acl(policy_instance, bucket_info, null_yield); if (ret < 0) { - set_err_msg(err_msg, "failed to set acl on bucket " + obj.bucket.name); + set_err_msg(err_msg, "failed to set acl on bucket " + bucket->get_name()); return ret; } RGWBucketEntryPoint ep; ep.bucket = bucket_info.bucket; - ep.owner = uid; + ep.owner = new_user->get_user(); ep.creation_time = bucket_info.creation_time; ep.linked = true; map<string, bufferlist> ep_attrs; rgw_ep_info ep_data{ep, ep_attrs}; - ret = bucket_ctl->link_bucket(uid, bucket_info.bucket, ceph::real_time(), - null_yield, true, &ep_data); + ret = bucket_ctl->link_bucket(new_user->get_user(), bucket_info.bucket, + ceph::real_time(), null_yield, true, &ep_data); if (ret < 0) { - set_err_msg(err_msg, "failed to link bucket " + obj.bucket.name + " to new user"); + set_err_msg(err_msg, "failed to link bucket " + bucket->get_name()); return ret; } - ret = bucket_ctl->chown(store, bucket_info, uid, old_user_info.display_name, - obj_marker, null_yield); + ret = bucket_ctl->chown(store, bucket_info, new_user->get_user(), + old_user_info.display_name, obj_marker, null_yield); if (ret < 0) { set_err_msg(err_msg, "failed to run bucket chown" + cpp_strerror(-ret)); return ret; } } - } while (is_truncated); + } while (buckets.is_truncated()); // update the 'stub user' with all of the other fields and rewrite all of the // associated index objects RGWUserInfo& user_info = op_state.get_user_info(); - user_info.user_id = uid; + user_info.user_id = new_user->get_user(); op_state.objv = objv; - rename_swift_keys(uid, user_info.swift_keys); + rename_swift_keys(new_user->get_user(), user_info.swift_keys); return update(op_state, err_msg); } @@ -1907,28 +1886,27 @@ int RGWUser::execute_remove(RGWUserAdminOpState& op_state, std::string *err_msg, return -ENOENT; } - bool is_truncated = false; + rgw::sal::RGWBucketList buckets; string marker; CephContext *cct = store->ctx(); size_t max_buckets = cct->_conf->rgw_list_buckets_max_chunk; do { - RGWUserBuckets buckets; ret = rgw_read_user_buckets(store, uid, buckets, marker, string(), - max_buckets, false, &is_truncated); + max_buckets, false); if (ret < 0) { set_err_msg(err_msg, "unable to read user bucket info"); return ret; } - map<std::string, RGWBucketEnt>& m = buckets.get_buckets(); + std::map<std::string, rgw::sal::RGWSalBucket*>& m = buckets.get_buckets(); if (!m.empty() && !purge_data) { set_err_msg(err_msg, "must specify purge data to remove user with buckets"); return -EEXIST; // change to code that maps to 409: conflict } - std::map<std::string, RGWBucketEnt>::iterator it; + std::map<std::string, rgw::sal::RGWSalBucket*>::iterator it; for (it = m.begin(); it != m.end(); ++it) { - ret = rgw_remove_bucket(store, ((*it).second).bucket, true, y); + ret = it->second->remove_bucket(true, y); if (ret < 0) { set_err_msg(err_msg, "unable to delete user data"); return ret; @@ -1937,7 +1915,7 @@ int RGWUser::execute_remove(RGWUserAdminOpState& op_state, std::string *err_msg, marker = it->first; } - } while (is_truncated); + } while (buckets.is_truncated()); ret = user_ctl->remove_info(user_info, y, RGWUserCtl::RemoveParams() .set_objv_tracker(&op_state.objv)); @@ -2057,32 +2035,31 @@ int RGWUser::execute_modify(RGWUserAdminOpState& op_state, std::string *err_msg) __u8 suspended = op_state.get_suspension_status(); user_info.suspended = suspended; - RGWUserBuckets buckets; + rgw::sal::RGWBucketList buckets; if (user_id.empty()) { set_err_msg(err_msg, "empty user id passed...aborting"); return -EINVAL; } - bool is_truncated = false; string marker; CephContext *cct = store->ctx(); size_t max_buckets = cct->_conf->rgw_list_buckets_max_chunk; do { ret = rgw_read_user_buckets(store, user_id, buckets, marker, string(), - max_buckets, false, &is_truncated); + max_buckets, false); if (ret < 0) { set_err_msg(err_msg, "could not get buckets for uid: " + user_id.to_str()); return ret; } - map<string, RGWBucketEnt>& m = buckets.get_buckets(); - map<string, RGWBucketEnt>::iterator iter; + std::map<std::string, rgw::sal::RGWSalBucket*>& m = buckets.get_buckets(); + std::map<std::string, rgw::sal::RGWSalBucket*>::iterator iter; vector<rgw_bucket> bucket_names; for (iter = m.begin(); iter != m.end(); ++iter) { - RGWBucketEnt obj = iter->second; - bucket_names.push_back(obj.bucket); + rgw::sal::RGWSalBucket* obj = iter->second; + bucket_names.push_back(obj->get_bi()); marker = iter->first; } @@ -2093,7 +2070,7 @@ int RGWUser::execute_modify(RGWUserAdminOpState& op_state, std::string *err_msg) return ret; } - } while (is_truncated); + } while (buckets.is_truncated()); } if (op_state.mfa_ids_specified) { |