diff options
Diffstat (limited to 'src')
-rw-r--r-- | src/rgw/rgw_admin.cc | 5 | ||||
-rw-r--r-- | src/rgw/rgw_bucket.cc | 106 | ||||
-rw-r--r-- | src/rgw/rgw_bucket.h | 12 | ||||
-rw-r--r-- | src/rgw/rgw_cr_tools.cc | 7 | ||||
-rw-r--r-- | src/rgw/rgw_file.h | 24 | ||||
-rw-r--r-- | src/rgw/rgw_lc.cc | 3 | ||||
-rw-r--r-- | src/rgw/rgw_object_expirer_core.cc | 4 | ||||
-rw-r--r-- | src/rgw/rgw_op.cc | 108 | ||||
-rw-r--r-- | src/rgw/rgw_op.h | 23 | ||||
-rw-r--r-- | src/rgw/rgw_orphan.cc | 2 | ||||
-rw-r--r-- | src/rgw/rgw_pubsub.cc | 6 | ||||
-rw-r--r-- | src/rgw/rgw_rados.cc | 26 | ||||
-rw-r--r-- | src/rgw/rgw_rados.h | 6 | ||||
-rw-r--r-- | src/rgw/rgw_reshard.cc | 3 | ||||
-rw-r--r-- | src/rgw/rgw_rest_bucket.cc | 3 | ||||
-rw-r--r-- | src/rgw/rgw_rest_log.cc | 6 | ||||
-rw-r--r-- | src/rgw/rgw_rest_pubsub_common.cc | 6 | ||||
-rw-r--r-- | src/rgw/rgw_rest_s3.cc | 26 | ||||
-rw-r--r-- | src/rgw/rgw_rest_s3.h | 3 | ||||
-rw-r--r-- | src/rgw/rgw_rest_swift.cc | 37 | ||||
-rw-r--r-- | src/rgw/rgw_rest_swift.h | 10 | ||||
-rw-r--r-- | src/rgw/rgw_sal.cc | 182 | ||||
-rw-r--r-- | src/rgw/rgw_sal.h | 128 | ||||
-rw-r--r-- | src/rgw/rgw_tools.cc | 2 | ||||
-rw-r--r-- | src/rgw/rgw_user.cc | 139 |
25 files changed, 526 insertions, 351 deletions
diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc index 6e7fa46e5a1..bf47739161f 100644 --- a/src/rgw/rgw_admin.cc +++ b/src/rgw/rgw_admin.cc @@ -1207,7 +1207,7 @@ static int init_bucket(const string& tenant_name, const string& bucket_name, con auto obj_ctx = store->svc()->sysobj->init_obj_ctx(); int r; if (bucket_id.empty()) { - r = store->getRados()->get_bucket_info(obj_ctx, tenant_name, bucket_name, bucket_info, nullptr, null_yield, pattrs); + r = store->getRados()->get_bucket_info(store->svc(), tenant_name, bucket_name, bucket_info, nullptr, null_yield, pattrs); } else { string bucket_instance_id = bucket_name + ":" + bucket_id; r = store->getRados()->get_bucket_instance_info(obj_ctx, bucket_instance_id, bucket_info, NULL, pattrs, null_yield); @@ -1390,8 +1390,7 @@ int set_bucket_quota(rgw::sal::RGWRadosStore *store, int opt_cmd, { RGWBucketInfo bucket_info; map<string, bufferlist> attrs; - auto obj_ctx = store->svc()->sysobj->init_obj_ctx(); - int r = store->getRados()->get_bucket_info(obj_ctx, tenant_name, bucket_name, bucket_info, NULL, null_yield, &attrs); + int r = store->getRados()->get_bucket_info(store->svc(), tenant_name, bucket_name, bucket_info, NULL, null_yield, &attrs); if (r < 0) { cerr << "could not get bucket info for bucket=" << bucket_name << ": " << cpp_strerror(-r) << std::endl; return -r; diff --git a/src/rgw/rgw_bucket.cc b/src/rgw/rgw_bucket.cc index a1676fe6a0d..4a777607aeb 100644 --- a/src/rgw/rgw_bucket.cc +++ b/src/rgw/rgw_bucket.cc @@ -149,17 +149,15 @@ void rgw_parse_url_bucket(const string &bucket, const string& auth_tenant, */ int rgw_read_user_buckets(RGWRadosStore * store, const rgw_user& user_id, - RGWUserBuckets& buckets, + rgw::sal::RGWBucketList& buckets, const string& marker, const string& end_marker, uint64_t max, - bool need_stats, - bool *is_truncated, - uint64_t default_amount) + bool need_stats) { - return store->ctl()->user->list_buckets(user_id, marker, end_marker, - max, need_stats, &buckets, - is_truncated, default_amount); + rgw::sal::RGWRadosUser user(store, user_id); + + return user.list_buckets(marker, end_marker, max, need_stats, buckets); } int rgw_bucket_parse_bucket_instance(const string& bucket_instance, string *bucket_name, string *bucket_id, int *shard_id) @@ -255,8 +253,8 @@ static void dump_mulipart_index_results(list<rgw_obj_index_key>& objs_to_unlink, void check_bad_user_bucket_mapping(RGWRadosStore *store, const rgw_user& user_id, bool fix) { - RGWUserBuckets user_buckets; - bool is_truncated = false; + rgw::sal::RGWBucketList user_buckets; + rgw::sal::RGWRadosUser user(store, user_id); string marker; CephContext *cct = store->ctx(); @@ -264,28 +262,24 @@ void check_bad_user_bucket_mapping(RGWRadosStore *store, const rgw_user& user_id size_t max_entries = cct->_conf->rgw_list_buckets_max_chunk; do { - int ret = rgw_read_user_buckets(store, user_id, user_buckets, marker, - string(), max_entries, false, - &is_truncated); + int ret = user.list_buckets(marker, string(), max_entries, false, user_buckets); if (ret < 0) { ldout(store->ctx(), 0) << "failed to read user buckets: " << cpp_strerror(-ret) << dendl; return; } - map<string, RGWBucketEnt>& buckets = user_buckets.get_buckets(); - for (map<string, RGWBucketEnt>::iterator i = buckets.begin(); + map<string, RGWSalBucket*>& buckets = user_buckets.get_buckets(); + for (map<string, RGWSalBucket*>::iterator i = buckets.begin(); i != buckets.end(); ++i) { marker = i->first; - RGWBucketEnt& bucket_ent = i->second; - rgw_bucket& bucket = bucket_ent.bucket; + RGWSalBucket* bucket = i->second; RGWBucketInfo bucket_info; real_time mtime; - RGWSysObjectCtx obj_ctx = store->svc()->sysobj->init_obj_ctx(); - int r = store->getRados()->get_bucket_info(obj_ctx, user_id.tenant, bucket.name, bucket_info, &mtime, null_yield); + int r = store->getRados()->get_bucket_info(store->svc(), user_id.tenant, bucket->get_name(), bucket_info, &mtime, null_yield); if (r < 0) { ldout(store->ctx(), 0) << "could not get bucket info for bucket=" << bucket << dendl; continue; @@ -293,10 +287,10 @@ void check_bad_user_bucket_mapping(RGWRadosStore *store, const rgw_user& user_id rgw_bucket& actual_bucket = bucket_info.bucket; - if (actual_bucket.name.compare(bucket.name) != 0 || - actual_bucket.tenant.compare(bucket.tenant) != 0 || - actual_bucket.marker.compare(bucket.marker) != 0 || - actual_bucket.bucket_id.compare(bucket.bucket_id) != 0) { + if (actual_bucket.name.compare(bucket->get_name()) != 0 || + actual_bucket.tenant.compare(bucket->get_tenant()) != 0 || + actual_bucket.marker.compare(bucket->get_marker()) != 0 || + actual_bucket.bucket_id.compare(bucket->get_bucket_id()) != 0) { cout << "bucket info mismatch: expected " << actual_bucket << " got " << bucket << std::endl; if (fix) { cout << "fixing" << std::endl; @@ -309,7 +303,7 @@ void check_bad_user_bucket_mapping(RGWRadosStore *store, const rgw_user& user_id } } } - } while (is_truncated); + } while (user_buckets.is_truncated()); } static bool bucket_object_check_filter(const string& oid) @@ -332,18 +326,18 @@ int rgw_remove_object(RGWRadosStore *store, const RGWBucketInfo& bucket_info, co return store->getRados()->delete_obj(rctx, bucket_info, obj, bucket_info.versioning_status()); } -int rgw_remove_bucket(RGWRadosStore *store, rgw_bucket& bucket, bool delete_children, optional_yield y) +/* xxx dang */ +static int rgw_remove_bucket(RGWRadosStore *store, rgw_bucket& bucket, bool delete_children, optional_yield y) { int ret; map<RGWObjCategory, RGWStorageStats> stats; std::vector<rgw_bucket_dir_entry> objs; map<string, bool> common_prefixes; RGWBucketInfo info; - RGWSysObjectCtx obj_ctx = store->svc()->sysobj->init_obj_ctx(); string bucket_ver, master_ver; - ret = store->getRados()->get_bucket_info(obj_ctx, bucket.tenant, bucket.name, info, NULL, null_yield); + ret = store->getRados()->get_bucket_info(store->svc(), bucket.tenant, bucket.name, info, NULL, null_yield); if (ret < 0) return ret; @@ -445,12 +439,11 @@ int rgw_remove_bucket_bypass_gc(RGWRadosStore *store, rgw_bucket& bucket, map<string, bool> common_prefixes; RGWBucketInfo info; RGWObjectCtx obj_ctx(store); - RGWSysObjectCtx sysobj_ctx = store->svc()->sysobj->init_obj_ctx(); CephContext *cct = store->ctx(); string bucket_ver, master_ver; - ret = store->getRados()->get_bucket_info(sysobj_ctx, bucket.tenant, bucket.name, info, NULL, null_yield); + ret = store->getRados()->get_bucket_info(store->svc(), bucket.tenant, bucket.name, info, NULL, null_yield); if (ret < 0) return ret; @@ -598,7 +591,6 @@ int RGWBucket::init(RGWRadosStore *storage, RGWBucketAdminOpState& op_state, rgw_user user_id = op_state.get_user_id(); bucket.tenant = user_id.tenant; bucket.name = op_state.get_bucket_name(); - RGWUserBuckets user_buckets; if (bucket.name.empty() && user_id.empty()) return -EINVAL; @@ -800,8 +792,7 @@ int RGWBucket::set_quota(RGWBucketAdminOpState& op_state, std::string *err_msg) rgw_bucket bucket = op_state.get_bucket(); RGWBucketInfo bucket_info; map<string, bufferlist> attrs; - auto obj_ctx = store->svc()->sysobj->init_obj_ctx(); - int r = store->getRados()->get_bucket_info(obj_ctx, bucket.tenant, bucket.name, bucket_info, NULL, null_yield, &attrs); + int r = store->getRados()->get_bucket_info(store->svc(), bucket.tenant, bucket.name, bucket_info, NULL, null_yield, &attrs); if (r < 0) { set_err_msg(err_msg, "could not get bucket info for bucket=" + bucket.name + ": " + cpp_strerror(-r)); return r; @@ -1147,11 +1138,10 @@ int RGWBucket::get_policy(RGWBucketAdminOpState& op_state, RGWAccessControlPolic { std::string object_name = op_state.get_object_name(); rgw_bucket bucket = op_state.get_bucket(); - auto sysobj_ctx = store->svc()->sysobj->init_obj_ctx(); RGWBucketInfo bucket_info; map<string, bufferlist> attrs; - int ret = store->getRados()->get_bucket_info(sysobj_ctx, bucket.tenant, bucket.name, bucket_info, NULL, null_yield, &attrs); + int ret = store->getRados()->get_bucket_info(store->svc(), bucket.tenant, bucket.name, bucket_info, NULL, null_yield, &attrs); if (ret < 0) { return ret; } @@ -1364,8 +1354,7 @@ static int bucket_stats(RGWRadosStore *store, const std::string& tenant_name, st map<string, bufferlist> attrs; real_time mtime; - auto obj_ctx = store->svc()->sysobj->init_obj_ctx(); - int r = store->getRados()->get_bucket_info(obj_ctx, tenant_name, bucket_name, bucket_info, &mtime, null_yield, &attrs); + int r = store->getRados()->get_bucket_info(store->svc(), tenant_name, bucket_name, bucket_info, &mtime, null_yield, &attrs); if (r < 0) return r; @@ -1450,33 +1439,32 @@ int RGWBucketAdminOp::limit_check(RGWRadosStore *store, formatter->open_array_section("buckets"); string marker; - bool is_truncated{false}; + rgw::sal::RGWBucketList buckets; do { - RGWUserBuckets buckets; + rgw::sal::RGWRadosUser user(store, rgw_user(user_id)); + + ret = user.list_buckets(marker, string(), max_entries, false, buckets); - ret = rgw_read_user_buckets(store, rgw_user(user_id), buckets, - marker, string(), max_entries, false, - &is_truncated); if (ret < 0) return ret; - map<string, RGWBucketEnt>& m_buckets = buckets.get_buckets(); + map<string, rgw::sal::RGWSalBucket*>& m_buckets = buckets.get_buckets(); for (const auto& iter : m_buckets) { - auto& bucket = iter.second.bucket; + auto bucket = iter.second; uint32_t num_shards = 1; uint64_t num_objects = 0; /* need info for num_shards */ RGWBucketInfo info; - auto obj_ctx = store->svc()->sysobj->init_obj_ctx(); - marker = bucket.name; /* Casey's location for marker update, - * as we may now not reach the end of - * the loop body */ + marker = bucket->get_name(); /* Casey's location for marker update, + * as we may now not reach the end of + * the loop body */ - ret = store->getRados()->get_bucket_info(obj_ctx, bucket.tenant, bucket.name, - info, nullptr, null_yield); + ret = store->getRados()->get_bucket_info(store->svc(), bucket->get_tenant(), + bucket->get_name(), info, nullptr, + null_yield); if (ret < 0) continue; @@ -1517,8 +1505,8 @@ int RGWBucketAdminOp::limit_check(RGWRadosStore *store, if (warn || (! warnings_only)) { formatter->open_object_section("bucket"); - formatter->dump_string("bucket", bucket.name); - formatter->dump_string("tenant", bucket.tenant); + formatter->dump_string("bucket", bucket->get_name()); + formatter->dump_string("tenant", bucket->get_tenant()); formatter->dump_int("num_objects", num_objects); formatter->dump_int("num_shards", num_shards); formatter->dump_int("objects_per_shard", objs_per_shard); @@ -1528,7 +1516,7 @@ int RGWBucketAdminOp::limit_check(RGWRadosStore *store, } } formatter->flush(cout); - } while (is_truncated); /* foreach: bucket */ + } while (buckets.is_truncated()); /* foreach: bucket */ formatter->close_section(); formatter->close_section(); @@ -1559,19 +1547,18 @@ int RGWBucketAdminOp::info(RGWRadosStore *store, RGWBucketAdminOpState& op_state if (op_state.is_user_op()) { formatter->open_array_section("buckets"); - RGWUserBuckets buckets; + rgw::sal::RGWBucketList buckets; + rgw::sal::RGWRadosUser user(store, op_state.get_user_id()); string marker; bool is_truncated = false; do { - ret = rgw_read_user_buckets(store, op_state.get_user_id(), buckets, - marker, string(), max_entries, false, - &is_truncated); + ret = user.list_buckets(marker, string(), max_entries, false, buckets); if (ret < 0) return ret; - map<string, RGWBucketEnt>& m = buckets.get_buckets(); - map<string, RGWBucketEnt>::iterator iter; + map<string, RGWSalBucket*>& m = buckets.get_buckets(); + map<string, RGWSalBucket*>::iterator iter; for (iter = m.begin(); iter != m.end(); ++iter) { std::string obj_name = iter->first; @@ -1695,7 +1682,7 @@ void get_stale_instances(RGWRadosStore *store, const std::string& bucket_name, // all the instances auto [tenant, bucket] = split_tenant(bucket_name); RGWBucketInfo cur_bucket_info; - int r = store->getRados()->get_bucket_info(obj_ctx, tenant, bucket, cur_bucket_info, nullptr, null_yield); + int r = store->getRados()->get_bucket_info(store->svc(), tenant, bucket, cur_bucket_info, nullptr, null_yield); if (r < 0) { if (r == -ENOENT) { // bucket doesn't exist, everything is stale then @@ -1841,10 +1828,9 @@ static int fix_single_bucket_lc(rgw::sal::RGWRadosStore *store, const std::string& tenant_name, const std::string& bucket_name) { - auto obj_ctx = store->svc()->sysobj->init_obj_ctx(); RGWBucketInfo bucket_info; map <std::string, bufferlist> bucket_attrs; - int ret = store->getRados()->get_bucket_info(obj_ctx, tenant_name, bucket_name, + int ret = store->getRados()->get_bucket_info(store->svc(), tenant_name, bucket_name, bucket_info, nullptr, null_yield, &bucket_attrs); if (ret < 0) { // TODO: Should we handle the case where the bucket could've been removed between diff --git a/src/rgw/rgw_bucket.h b/src/rgw/rgw_bucket.h index cf9ae2038d4..94826b6315c 100644 --- a/src/rgw/rgw_bucket.h +++ b/src/rgw/rgw_bucket.h @@ -26,7 +26,10 @@ class RGWSI_Meta; class RGWBucketMetadataHandler; class RGWBucketInstanceMetadataHandler; class RGWUserCtl; -namespace rgw { namespace sal { class RGWRadosStore; } } +namespace rgw { namespace sal { + class RGWRadosStore; + class RGWBucketList; +} } extern int rgw_bucket_parse_bucket_instance(const string& bucket_instance, string *bucket_name, string *bucket_id, int *shard_id); extern int rgw_bucket_parse_bucket_key(CephContext *cct, const string& key, @@ -211,16 +214,13 @@ public: */ extern int rgw_read_user_buckets(rgw::sal::RGWRadosStore *store, const rgw_user& user_id, - RGWUserBuckets& buckets, + rgw::sal::RGWBucketList& buckets, const string& marker, const string& end_marker, uint64_t max, - bool need_stats, - bool* is_truncated, - uint64_t default_amount = 1000); + bool need_stats); extern int rgw_remove_object(rgw::sal::RGWRadosStore *store, const RGWBucketInfo& bucket_info, const rgw_bucket& bucket, rgw_obj_key& key); -extern int rgw_remove_bucket(rgw::sal::RGWRadosStore *store, rgw_bucket& bucket, bool delete_children, optional_yield y); extern int rgw_remove_bucket_bypass_gc(rgw::sal::RGWRadosStore *store, rgw_bucket& bucket, int concurrent_max, optional_yield y); extern int rgw_object_get_attr(rgw::sal::RGWRadosStore* store, const RGWBucketInfo& bucket_info, diff --git a/src/rgw/rgw_cr_tools.cc b/src/rgw/rgw_cr_tools.cc index 300201ae9b5..b2ced7da5c5 100644 --- a/src/rgw/rgw_cr_tools.cc +++ b/src/rgw/rgw_cr_tools.cc @@ -98,8 +98,7 @@ int RGWGetUserInfoCR::Request::_send_request() template<> int RGWGetBucketInfoCR::Request::_send_request() { - RGWSysObjectCtx obj_ctx(store->svc()->sysobj->init_obj_ctx()); - return store->getRados()->get_bucket_info(obj_ctx, params.tenant, params.bucket_name, + return store->getRados()->get_bucket_info(store->svc(), params.tenant, params.bucket_name, result->bucket_info, &result->mtime, null_yield, &result->attrs); } @@ -108,7 +107,6 @@ int RGWBucketCreateLocalCR::Request::_send_request() { CephContext *cct = store->ctx(); auto& zone_svc = store->svc()->zone; - auto& sysobj_svc = store->svc()->sysobj; const auto& user_info = params.user_info.get(); const auto& user = user_info->user_id; @@ -125,11 +123,10 @@ int RGWBucketCreateLocalCR::Request::_send_request() /* we need to make sure we read bucket info, it's not read before for this * specific request */ - RGWSysObjectCtx sysobj_ctx(sysobj_svc->init_obj_ctx()); RGWBucketInfo bucket_info; map<string, bufferlist> bucket_attrs; - int ret = store->getRados()->get_bucket_info(sysobj_ctx, user.tenant, bucket_name, + int ret = store->getRados()->get_bucket_info(store->svc(), user.tenant, bucket_name, bucket_info, nullptr, null_yield, &bucket_attrs); if (ret < 0 && ret != -ENOENT) return ret; diff --git a/src/rgw/rgw_file.h b/src/rgw/rgw_file.h index 12bca8fab00..92bcda3fcd5 100644 --- a/src/rgw/rgw_file.h +++ b/src/rgw/rgw_file.h @@ -1379,17 +1379,17 @@ public: sent_data = true; } - void send_response_data(RGWUserBuckets& buckets) override { + void send_response_data(rgw::sal::RGWBucketList& buckets) override { if (!sent_data) return; - map<string, RGWBucketEnt>& m = buckets.get_buckets(); + map<string, rgw::sal::RGWSalBucket*>& m = buckets.get_buckets(); for (const auto& iter : m) { boost::string_ref marker{iter.first}; - const RGWBucketEnt& ent = iter.second; - if (! this->operator()(ent.bucket.name, marker)) { + rgw::sal::RGWSalBucket* ent = iter.second; + if (! this->operator()(ent->get_name(), marker)) { /* caller cannot accept more */ lsubdout(cct, rgw, 5) << "ListBuckets rcb failed" - << " dirent=" << ent.bucket.name + << " dirent=" << ent->get_name() << " call count=" << ix << dendl; rcb_eof = true; @@ -2243,7 +2243,7 @@ public: } real_time get_ctime() const { - return bucket.creation_time; + return bucket->get_creation_time(); } bool only_bucket() override { return false; } @@ -2284,16 +2284,16 @@ public: } void send_response() override { - bucket.creation_time = get_state()->bucket_info.creation_time; - bs.size = bucket.size; - bs.size_rounded = bucket.size_rounded; - bs.creation_time = bucket.creation_time; - bs.num_entries = bucket.count; + bucket->get_creation_time() = get_state()->bucket_info.creation_time; + bs.size = bucket->get_size(); + bs.size_rounded = bucket->get_size_rounded(); + bs.creation_time = bucket->get_creation_time(); + bs.num_entries = bucket->get_count(); std::swap(attrs, get_state()->bucket_attrs); } bool matched() { - return (bucket.bucket.name.length() > 0); + return (bucket->get_name().length() > 0); } }; /* RGWStatBucketRequest */ diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc index 18cb5aa84ba..00abcf3020f 100644 --- a/src/rgw/rgw_lc.cc +++ b/src/rgw/rgw_lc.cc @@ -1026,13 +1026,12 @@ int RGWLC::bucket_lc_process(string& shard_id) map<string, bufferlist> bucket_attrs; string no_ns, list_versions; vector<rgw_bucket_dir_entry> objs; - auto obj_ctx = store->svc()->sysobj->init_obj_ctx(); vector<std::string> result; boost::split(result, shard_id, boost::is_any_of(":")); string bucket_tenant = result[0]; string bucket_name = result[1]; string bucket_marker = result[2]; - int ret = store->getRados()->get_bucket_info(obj_ctx, bucket_tenant, bucket_name, bucket_info, NULL, null_yield, &bucket_attrs); + int ret = store->getRados()->get_bucket_info(store->svc(), bucket_tenant, bucket_name, bucket_info, NULL, null_yield, &bucket_attrs); if (ret < 0) { ldpp_dout(this, 0) << "LC:get_bucket_info for " << bucket_name << " failed" << dendl; return ret; diff --git a/src/rgw/rgw_object_expirer_core.cc b/src/rgw/rgw_object_expirer_core.cc index 0e78788649e..739b534c629 100644 --- a/src/rgw/rgw_object_expirer_core.cc +++ b/src/rgw/rgw_object_expirer_core.cc @@ -174,8 +174,6 @@ int RGWObjectExpirer::init_bucket_info(const string& tenant_name, const string& bucket_id, RGWBucketInfo& bucket_info) { - auto obj_ctx = store->svc()->sysobj->init_obj_ctx(); - /* * XXX Here's where it gets tricky. We went to all the trouble of * punching the tenant through the objexp_hint_entry, but now we @@ -186,7 +184,7 @@ int RGWObjectExpirer::init_bucket_info(const string& tenant_name, * are ephemeral, good call encoding tenant info! */ - return store->getRados()->get_bucket_info(obj_ctx, tenant_name, bucket_name, + return store->getRados()->get_bucket_info(store->svc(), tenant_name, bucket_name, bucket_info, nullptr, null_yield, nullptr); } diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index 90279c1314a..e713272ac91 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -588,7 +588,7 @@ int rgw_build_bucket_policies(rgw::sal::RGWRadosStore* store, struct req_state* RGWBucketInfo source_info; if (s->bucket_instance_id.empty()) { - ret = store->getRados()->get_bucket_info(obj_ctx, s->src_tenant_name, s->src_bucket_name, source_info, NULL, s->yield); + ret = store->getRados()->get_bucket_info(store->svc(), s->src_tenant_name, s->src_bucket_name, source_info, NULL, s->yield); } else { ret = store->getRados()->get_bucket_instance_info(obj_ctx, s->bucket_instance_id, source_info, NULL, NULL, s->yield); } @@ -1787,7 +1787,7 @@ int RGWGetObj::handle_user_manifest(const char *prefix) if (bucket_name.compare(s->bucket.name) != 0) { map<string, bufferlist> bucket_attrs; auto obj_ctx = store->svc()->sysobj->init_obj_ctx(); - int r = store->getRados()->get_bucket_info(obj_ctx, s->user->user_id.tenant, + int r = store->getRados()->get_bucket_info(store->svc(), s->user->user_id.tenant, bucket_name, bucket_info, NULL, s->yield, &bucket_attrs); if (r < 0) { @@ -1921,7 +1921,7 @@ int RGWGetObj::handle_slo_manifest(bufferlist& bl) RGWBucketInfo bucket_info; map<string, bufferlist> bucket_attrs; auto obj_ctx = store->svc()->sysobj->init_obj_ctx(); - int r = store->getRados()->get_bucket_info(obj_ctx, s->user->user_id.tenant, + int r = store->getRados()->get_bucket_info(store->svc(), s->user->user_id.tenant, bucket_name, bucket_info, nullptr, s->yield, &bucket_attrs); if (r < 0) { @@ -2318,7 +2318,7 @@ void RGWListBuckets::execute() is_truncated = false; do { - RGWUserBuckets buckets; + rgw::sal::RGWBucketList buckets; uint64_t read_count; if (limit >= 0) { read_count = min(limit - total_count, max_buckets); @@ -2326,10 +2326,10 @@ void RGWListBuckets::execute() read_count = max_buckets; } - op_ret = rgw_read_user_buckets(store, s->user->user_id, buckets, - marker, end_marker, read_count, - should_get_stats(), &is_truncated, - get_default_max()); + rgw::sal::RGWRadosUser user(store, s->user->user_id); + + op_ret = user.list_buckets(marker, end_marker, read_count, should_get_stats(), buckets); + if (op_ret < 0) { /* hmm.. something wrong here.. the user was authenticated, so it should exist */ @@ -2346,21 +2346,21 @@ void RGWListBuckets::execute() decltype(policies_stats)::mapped_type()); } - std::map<std::string, RGWBucketEnt>& m = buckets.get_buckets(); + std::map<std::string, rgw::sal::RGWSalBucket*>& m = buckets.get_buckets(); for (const auto& kv : m) { const auto& bucket = kv.second; - global_stats.bytes_used += bucket.size; - global_stats.bytes_used_rounded += bucket.size_rounded; - global_stats.objects_count += bucket.count; + global_stats.bytes_used += bucket->get_size(); + global_stats.bytes_used_rounded += bucket->get_size_rounded(); + global_stats.objects_count += bucket->get_count(); /* operator[] still can create a new entry for storage policy seen * for first time. */ - auto& policy_stats = policies_stats[bucket.placement_rule.to_str()]; - policy_stats.bytes_used += bucket.size; - policy_stats.bytes_used_rounded += bucket.size_rounded; + auto& policy_stats = policies_stats[bucket->get_placement_rule().to_str()]; + policy_stats.bytes_used += bucket->get_size(); + policy_stats.bytes_used_rounded += bucket->get_size_rounded(); policy_stats.buckets_count++; - policy_stats.objects_count += bucket.count; + policy_stats.objects_count += bucket->get_count(); } global_stats.buckets_count += m.size(); total_count += m.size(); @@ -2373,7 +2373,7 @@ void RGWListBuckets::execute() } if (!m.empty()) { - map<string, RGWBucketEnt>::reverse_iterator riter = m.rbegin(); + map<string, rgw::sal::RGWSalBucket*>::reverse_iterator riter = m.rbegin(); marker = riter->first; handle_listing_chunk(std::move(buckets)); @@ -2464,14 +2464,13 @@ int RGWStatAccount::verify_permission() void RGWStatAccount::execute() { string marker; - bool is_truncated = false; + rgw::sal::RGWBucketList buckets; uint64_t max_buckets = s->cct->_conf->rgw_list_buckets_max_chunk; do { - RGWUserBuckets buckets; op_ret = rgw_read_user_buckets(store, s->user->user_id, buckets, marker, - string(), max_buckets, true, &is_truncated); + string(), max_buckets, true); if (op_ret < 0) { /* hmm.. something wrong here.. the user was authenticated, so it should exist */ @@ -2487,26 +2486,26 @@ void RGWStatAccount::execute() decltype(policies_stats)::mapped_type()); } - std::map<std::string, RGWBucketEnt>& m = buckets.get_buckets(); + std::map<std::string, rgw::sal::RGWSalBucket*>& m = buckets.get_buckets(); for (const auto& kv : m) { const auto& bucket = kv.second; - global_stats.bytes_used += bucket.size; - global_stats.bytes_used_rounded += bucket.size_rounded; - global_stats.objects_count += bucket.count; + global_stats.bytes_used += bucket->get_size(); + global_stats.bytes_used_rounded += bucket->get_size_rounded(); + global_stats.objects_count += bucket->get_count(); /* operator[] still can create a new entry for storage policy seen * for first time. */ - auto& policy_stats = policies_stats[bucket.placement_rule.to_str()]; - policy_stats.bytes_used += bucket.size; - policy_stats.bytes_used_rounded += bucket.size_rounded; + auto& policy_stats = policies_stats[bucket->get_placement_rule().to_str()]; + policy_stats.bytes_used += bucket->get_size(); + policy_stats.bytes_used_rounded += bucket->get_size_rounded(); policy_stats.buckets_count++; - policy_stats.objects_count += bucket.count; + policy_stats.objects_count += bucket->get_count(); } global_stats.buckets_count += m.size(); } - } while (is_truncated); + } while (buckets.is_truncated()); } int RGWGetBucketVersioning::verify_permission() @@ -2716,22 +2715,9 @@ void RGWStatBucket::execute() return; } - RGWUserBuckets buckets; - bucket.bucket = s->bucket; - buckets.add(bucket); - map<string, RGWBucketEnt>& m = buckets.get_buckets(); - op_ret = store->getRados()->update_containers_stats(m); - if (! op_ret) - op_ret = -EEXIST; - if (op_ret > 0) { - op_ret = 0; - map<string, RGWBucketEnt>::iterator iter = m.find(bucket.bucket.name); - if (iter != m.end()) { - bucket = iter->second; - } else { - op_ret = -EINVAL; - } - } + rgw::sal::RGWRadosUser user(store, s->user->user_id); + bucket = new rgw::sal::RGWRadosBucket(store, user, s->bucket); + op_ret = bucket->update_container_stats(); } int RGWListBucket::verify_permission() @@ -2790,13 +2776,7 @@ void RGWListBucket::execute() } if (need_container_stats()) { - map<string, RGWBucketEnt> m; - m[s->bucket.name] = RGWBucketEnt(); - m.begin()->second.bucket = s->bucket; - op_ret = store->getRados()->update_containers_stats(m); - if (op_ret > 0) { - bucket = m.begin()->second; - } + op_ret = bucket->update_container_stats(); } RGWRados::Bucket target(store->getRados(), s->bucket_info); @@ -2857,12 +2837,11 @@ int RGWCreateBucket::verify_permission() } if (s->user->max_buckets) { - RGWUserBuckets buckets; + rgw::sal::RGWBucketList buckets; string marker; - bool is_truncated = false; op_ret = rgw_read_user_buckets(store, s->user->user_id, buckets, marker, string(), s->user->max_buckets, - false, &is_truncated); + false); if (op_ret < 0) { return op_ret; } @@ -3120,7 +3099,7 @@ void RGWCreateBucket::execute() /* we need to make sure we read bucket info, it's not read before for this * specific request */ - op_ret = store->getRados()->get_bucket_info(*s->sysobj_ctx, s->bucket_tenant, s->bucket_name, + op_ret = store->getRados()->get_bucket_info(store->svc(), s->bucket_tenant, s->bucket_name, s->bucket_info, nullptr, s->yield, &s->bucket_attrs); if (op_ret < 0 && op_ret != -ENOENT) return; @@ -3290,7 +3269,7 @@ void RGWCreateBucket::execute() RGWBucketInfo binfo; map<string, bufferlist> battrs; - op_ret = store->getRados()->get_bucket_info(*s->sysobj_ctx, s->bucket_tenant, s->bucket_name, + op_ret = store->getRados()->get_bucket_info(store->svc(), s->bucket_tenant, s->bucket_name, binfo, nullptr, s->yield, &battrs); if (op_ret < 0) { return; @@ -4784,7 +4763,7 @@ void RGWDeleteObj::execute() obj_ctx->set_atomic(obj); bool ver_restored = false; - op_ret = store->getRados()->swift_versioning_restore(*s->sysobj_ctx, *obj_ctx, s->bucket_owner.get_id(), + op_ret = store->getRados()->swift_versioning_restore(*obj_ctx, s->bucket_owner.get_id(), s->bucket_info, obj, ver_restored, this); if (op_ret < 0) { return; @@ -4901,7 +4880,7 @@ int RGWCopyObj::verify_permission() map<string, bufferlist> src_attrs; if (s->bucket_instance_id.empty()) { - op_ret = store->getRados()->get_bucket_info(*s->sysobj_ctx, src_tenant_name, src_bucket_name, src_bucket_info, NULL, s->yield, &src_attrs); + op_ret = store->getRados()->get_bucket_info(store->svc(), src_tenant_name, src_bucket_name, src_bucket_info, NULL, s->yield, &src_attrs); } else { /* will only happen in intra region sync where the source and dest bucket is the same */ rgw_bucket b(rgw_bucket_key(src_tenant_name, src_bucket_name, s->bucket_instance_id)); @@ -4972,7 +4951,7 @@ int RGWCopyObj::verify_permission() dest_bucket_info = src_bucket_info; dest_attrs = src_attrs; } else { - op_ret = store->getRados()->get_bucket_info(*s->sysobj_ctx, dest_tenant_name, dest_bucket_name, + op_ret = store->getRados()->get_bucket_info(store->svc(), dest_tenant_name, dest_bucket_name, dest_bucket_info, nullptr, s->yield, &dest_attrs); if (op_ret < 0) { if (op_ret == -ENOENT) { @@ -6766,12 +6745,11 @@ RGWBulkUploadOp::handle_upload_path(struct req_state *s) int RGWBulkUploadOp::handle_dir_verify_permission() { if (s->user->max_buckets > 0) { - RGWUserBuckets buckets; + rgw::sal::RGWBucketList buckets; std::string marker; - bool is_truncated = false; op_ret = rgw_read_user_buckets(store, s->user->user_id, buckets, marker, std::string(), s->user->max_buckets, - false, &is_truncated); + false); if (op_ret < 0) { return op_ret; } @@ -6826,7 +6804,7 @@ int RGWBulkUploadOp::handle_dir(const boost::string_ref path) * specific request */ RGWBucketInfo binfo; std::map<std::string, ceph::bufferlist> battrs; - op_ret = store->getRados()->get_bucket_info(*dir_ctx, s->bucket_tenant, bucket_name, + op_ret = store->getRados()->get_bucket_info(store->svc(), s->bucket_tenant, bucket_name, binfo, nullptr, s->yield, &battrs); if (op_ret < 0 && op_ret != -ENOENT) { return op_ret; @@ -7020,7 +6998,7 @@ int RGWBulkUploadOp::handle_file(const boost::string_ref path, RGWBucketInfo binfo; std::map<std::string, ceph::bufferlist> battrs; ACLOwner bowner; - op_ret = store->getRados()->get_bucket_info(*s->sysobj_ctx, s->user->user_id.tenant, + op_ret = store->getRados()->get_bucket_info(store->svc(), s->user->user_id.tenant, bucket_name, binfo, nullptr, s->yield, &battrs); if (op_ret == -ENOENT) { ldpp_dout(this, 20) << "non existent directory=" << bucket_name << dendl; diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index 356a79f6c0c..38b2800118c 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -34,6 +34,7 @@ #include "rgw_common.h" #include "rgw_dmclock.h" +#include "rgw_sal.h" #include "rgw_user.h" #include "rgw_bucket.h" #include "rgw_acl.h" @@ -712,15 +713,15 @@ public: void execute() override; virtual int get_params() = 0; - virtual void handle_listing_chunk(RGWUserBuckets&& buckets) { + virtual void handle_listing_chunk(rgw::sal::RGWBucketList&& buckets) { /* The default implementation, used by e.g. S3, just generates a new * part of listing and sends it client immediately. Swift can behave * differently: when the reverse option is requested, all incoming - * instances of RGWUserBuckets are buffered and finally reversed. */ + * instances of RGWBucketList are buffered and finally reversed. */ return send_response_data(buckets); } virtual void send_response_begin(bool has_buckets) = 0; - virtual void send_response_data(RGWUserBuckets& buckets) = 0; + virtual void send_response_data(rgw::sal::RGWBucketList& buckets) = 0; virtual void send_response_end() = 0; void send_response() override {} @@ -780,7 +781,7 @@ public: class RGWListBucket : public RGWOp { protected: - RGWBucketEnt bucket; + rgw::sal::RGWSalBucket* bucket; string prefix; rgw_obj_key marker; rgw_obj_key next_marker; @@ -802,13 +803,19 @@ protected: int parse_max_keys(); public: - RGWListBucket() : list_versions(false), max(0), + RGWListBucket() : bucket(nullptr), list_versions(false), max(0), default_max(0), is_truncated(false), allow_unordered(false), shard_id(-1) {} + ~RGWListBucket() { delete bucket; } int verify_permission() override; void pre_exec() override; void execute() override; + void init(rgw::sal::RGWRadosStore *store, struct req_state *s, RGWHandler *h) override { + RGWOp::init(store, s, h); + rgw::sal::RGWRadosUser user(store, s->user->user_id); + bucket = new rgw::sal::RGWRadosBucket(store, user, s->bucket); + } virtual int get_params() = 0; void send_response() override = 0; const char* name() const override { return "list_bucket"; } @@ -937,11 +944,11 @@ public: class RGWStatBucket : public RGWOp { protected: - RGWBucketEnt bucket; + rgw::sal::RGWSalBucket* bucket; public: - RGWStatBucket() {} - ~RGWStatBucket() override {} + RGWStatBucket() : bucket(nullptr) {} + ~RGWStatBucket() override { delete bucket; } int verify_permission() override; void pre_exec() override; diff --git a/src/rgw/rgw_orphan.cc b/src/rgw/rgw_orphan.cc index 271139aa7c7..9eb21763a3f 100644 --- a/src/rgw/rgw_orphan.cc +++ b/src/rgw/rgw_orphan.cc @@ -500,7 +500,7 @@ int RGWOrphanSearch::build_linked_oids_for_bucket(const string& bucket_instance_ } RGWBucketInfo cur_bucket_info; - ret = store->getRados()->get_bucket_info(sysobj_ctx, orphan_bucket.tenant, + ret = store->getRados()->get_bucket_info(store->svc(), orphan_bucket.tenant, orphan_bucket.name, cur_bucket_info, nullptr, null_yield); if (ret < 0) { if (ret == -ENOENT) { diff --git a/src/rgw/rgw_pubsub.cc b/src/rgw/rgw_pubsub.cc index 5e3d7cc4128..6b439eb6ad1 100644 --- a/src/rgw/rgw_pubsub.cc +++ b/src/rgw/rgw_pubsub.cc @@ -755,8 +755,7 @@ int RGWUserPubSub::SubWithEvents<EventType>::list_events(const string& marker, i RGWBucketInfo bucket_info; string tenant; - RGWSysObjectCtx obj_ctx(store->svc.sysobj->init_obj_ctx()); - ret = store->get_bucket_info(obj_ctx, tenant, sub_conf.dest.bucket_name, bucket_info, nullptr, null_yield, nullptr); + ret = store->get_bucket_info(&store->svc, tenant, sub_conf.dest.bucket_name, bucket_info, nullptr, null_yield, nullptr); if (ret == -ENOENT) { list.is_truncated = false; return 0; @@ -821,8 +820,7 @@ int RGWUserPubSub::SubWithEvents<EventType>::remove_event(const string& event_id RGWBucketInfo bucket_info; string tenant; - RGWSysObjectCtx sysobj_ctx(store->svc()->sysobj->init_obj_ctx()); - ret = store->getRados()->get_bucket_info(sysobj_ctx, tenant, sub_conf.dest.bucket_name, bucket_info, nullptr, null_yield, nullptr); + ret = store->getRados()->get_bucket_info(store->svc(), tenant, sub_conf.dest.bucket_name, bucket_info, nullptr, null_yield, nullptr); if (ret < 0) { ldout(store->ctx(), 1) << "ERROR: failed to read bucket info for events bucket: bucket=" << sub_conf.dest.bucket_name << " ret=" << ret << dendl; return ret; diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index 0399a4c34b9..b1b3ae7a5d7 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -2145,8 +2145,7 @@ int RGWRados::create_bucket(const RGWUserInfo& owner, rgw_bucket& bucket, /* we need to reread the info and return it, caller will have a use for it */ RGWObjVersionTracker instance_ver = info.objv_tracker; info.objv_tracker.clear(); - auto obj_ctx = svc.sysobj->init_obj_ctx(); - r = get_bucket_info(obj_ctx, bucket.tenant, bucket.name, info, NULL, null_yield, NULL); + r = get_bucket_info(&svc, bucket.tenant, bucket.name, info, NULL, null_yield, NULL); if (r < 0) { if (r == -ENOENT) { continue; @@ -2662,9 +2661,7 @@ int RGWRados::swift_versioning_copy(RGWObjectCtx& obj_ctx, RGWBucketInfo dest_bucket_info; - auto sysobj_ctx = svc.sysobj->init_obj_ctx(); - - r = get_bucket_info(sysobj_ctx, bucket_info.bucket.tenant, bucket_info.swift_ver_location, dest_bucket_info, NULL, null_yield, NULL); + r = get_bucket_info(&svc, bucket_info.bucket.tenant, bucket_info.swift_ver_location, dest_bucket_info, NULL, null_yield, NULL); if (r < 0) { ldout(cct, 10) << "failed to read dest bucket info: r=" << r << dendl; if (r == -ENOENT) { @@ -2725,8 +2722,7 @@ int RGWRados::swift_versioning_copy(RGWObjectCtx& obj_ctx, return r; } -int RGWRados::swift_versioning_restore(RGWSysObjectCtx& sysobj_ctx, - RGWObjectCtx& obj_ctx, +int RGWRados::swift_versioning_restore(RGWObjectCtx& obj_ctx, const rgw_user& user, RGWBucketInfo& bucket_info, rgw_obj& obj, @@ -2740,7 +2736,7 @@ int RGWRados::swift_versioning_restore(RGWSysObjectCtx& sysobj_ctx, /* Bucket info of the bucket that stores previous versions of our object. */ RGWBucketInfo archive_binfo; - int ret = get_bucket_info(sysobj_ctx, bucket_info.bucket.tenant, + int ret = get_bucket_info(&svc, bucket_info.bucket.tenant, bucket_info.swift_ver_location, archive_binfo, nullptr, null_yield, nullptr); if (ret < 0) { @@ -4408,10 +4404,11 @@ int RGWRados::set_bucket_owner(rgw_bucket& bucket, ACLOwner& owner) { RGWBucketInfo info; map<string, bufferlist> attrs; - auto obj_ctx = svc.sysobj->init_obj_ctx(); int r; + auto obj_ctx = svc.sysobj->init_obj_ctx(); + if (bucket.bucket_id.empty()) { - r = get_bucket_info(obj_ctx, bucket.tenant, bucket.name, info, NULL, null_yield, &attrs); + r = get_bucket_info(&svc, bucket.tenant, bucket.name, info, NULL, null_yield, &attrs); } else { r = get_bucket_instance_info(obj_ctx, bucket, info, nullptr, &attrs, null_yield); } @@ -4447,8 +4444,7 @@ int RGWRados::set_buckets_enabled(vector<rgw_bucket>& buckets, bool enabled) RGWBucketInfo info; map<string, bufferlist> attrs; - auto obj_ctx = svc.sysobj->init_obj_ctx(); - int r = get_bucket_info(obj_ctx, bucket.tenant, bucket.name, info, NULL, null_yield, &attrs); + int r = get_bucket_info(&svc, bucket.tenant, bucket.name, info, NULL, null_yield, &attrs); if (r < 0) { ldout(cct, 0) << "NOTICE: get_bucket_info on bucket=" << bucket.name << " returned err=" << r << ", skipping bucket" << dendl; ret = r; @@ -4473,8 +4469,7 @@ int RGWRados::set_buckets_enabled(vector<rgw_bucket>& buckets, bool enabled) int RGWRados::bucket_suspended(rgw_bucket& bucket, bool *suspended) { RGWBucketInfo bucket_info; - auto obj_ctx = svc.sysobj->init_obj_ctx(); - int ret = get_bucket_info(obj_ctx, bucket.tenant, bucket.name, bucket_info, NULL, null_yield); + int ret = get_bucket_info(&svc, bucket.tenant, bucket.name, bucket_info, NULL, null_yield); if (ret < 0) { return ret; } @@ -7319,12 +7314,13 @@ int RGWRados::get_bucket_instance_info(RGWSysObjectCtx& obj_ctx, const rgw_bucke .set_bectx_params(bectx_params)); } -int RGWRados::get_bucket_info(RGWSysObjectCtx& obj_ctx, +int RGWRados::get_bucket_info(RGWServices *svc, const string& tenant, const string& bucket_name, RGWBucketInfo& info, real_time *pmtime, optional_yield y, map<string, bufferlist> *pattrs) { + auto obj_ctx = svc->sysobj->init_obj_ctx(); RGWSI_MetaBackend_CtxParams bectx_params = RGWSI_MetaBackend_CtxParams_SObj(&obj_ctx); rgw_bucket bucket; bucket.tenant = tenant; diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index d15caeecf95..e3017e785a1 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -1032,8 +1032,7 @@ public: rgw_obj& obj, /* in */ const DoutPrefixProvider *dpp, /* in/out */ optional_yield y); /* in */ - int swift_versioning_restore(RGWSysObjectCtx& sysobj_ctx, - RGWObjectCtx& obj_ctx, /* in/out */ + int swift_versioning_restore(RGWObjectCtx& obj_ctx, /* in/out */ const rgw_user& user, /* in */ RGWBucketInfo& bucket_info, /* in */ rgw_obj& obj, /* in */ @@ -1310,12 +1309,13 @@ public: int get_bucket_stats_async(RGWBucketInfo& bucket_info, int shard_id, RGWGetBucketStats_CB *cb); int put_bucket_instance_info(RGWBucketInfo& info, bool exclusive, ceph::real_time mtime, map<string, bufferlist> *pattrs); + /* xxx dang obj_ctx -> svc */ int get_bucket_instance_info(RGWSysObjectCtx& obj_ctx, const string& meta_key, RGWBucketInfo& info, ceph::real_time *pmtime, map<string, bufferlist> *pattrs, optional_yield y); int get_bucket_instance_info(RGWSysObjectCtx& obj_ctx, const rgw_bucket& bucket, RGWBucketInfo& info, ceph::real_time *pmtime, map<string, bufferlist> *pattrs, optional_yield y); static void make_bucket_entry_name(const string& tenant_name, const string& bucket_name, string& bucket_entry); - int get_bucket_info(RGWSysObjectCtx& obj_ctx, + int get_bucket_info(RGWServices *svc, const string& tenant_name, const string& bucket_name, RGWBucketInfo& info, ceph::real_time *pmtime, optional_yield y, map<string, bufferlist> *pattrs = NULL); diff --git a/src/rgw/rgw_reshard.cc b/src/rgw/rgw_reshard.cc index 6444aac07ab..dd405a2ef08 100644 --- a/src/rgw/rgw_reshard.cc +++ b/src/rgw/rgw_reshard.cc @@ -979,12 +979,11 @@ int RGWReshard::process_single_logshard(int logshard_num) ldout(store->ctx(), 20) << __func__ << " resharding " << entry.bucket_name << dendl; - auto obj_ctx = store->svc()->sysobj->init_obj_ctx(); rgw_bucket bucket; RGWBucketInfo bucket_info; map<string, bufferlist> attrs; - ret = store->getRados()->get_bucket_info(obj_ctx, entry.tenant, entry.bucket_name, + ret = store->getRados()->get_bucket_info(store->svc(), entry.tenant, entry.bucket_name, bucket_info, nullptr, null_yield, &attrs); if (ret < 0) { ldout(cct, 0) << __func__ << ": Error in get_bucket_info: " << diff --git a/src/rgw/rgw_rest_bucket.cc b/src/rgw/rgw_rest_bucket.cc index 5ee5a8eb6a5..71e5655ff5a 100644 --- a/src/rgw/rgw_rest_bucket.cc +++ b/src/rgw/rgw_rest_bucket.cc @@ -293,8 +293,7 @@ void RGWOp_Set_Bucket_Quota::execute() if (use_http_params) { RGWBucketInfo bucket_info; map<string, bufferlist> attrs; - auto obj_ctx = store->svc()->sysobj->init_obj_ctx(); - http_ret = store->getRados()->get_bucket_info(obj_ctx, uid.tenant, bucket, bucket_info, NULL, s->yield, &attrs); + http_ret = store->getRados()->get_bucket_info(store->svc(), uid.tenant, bucket, bucket_info, NULL, s->yield, &attrs); if (http_ret < 0) { return; } diff --git a/src/rgw/rgw_rest_log.cc b/src/rgw/rgw_rest_log.cc index e4101d6b52e..f8891402a8a 100644 --- a/src/rgw/rgw_rest_log.cc +++ b/src/rgw/rgw_rest_log.cc @@ -404,7 +404,7 @@ void RGWOp_BILog_List::execute() { return; } } else { /* !bucket_name.empty() */ - http_ret = store->getRados()->get_bucket_info(*s->sysobj_ctx, tenant_name, bucket_name, bucket_info, NULL, s->yield, NULL); + http_ret = store->getRados()->get_bucket_info(store->svc(), tenant_name, bucket_name, bucket_info, NULL, s->yield, NULL); if (http_ret < 0) { ldpp_dout(s, 5) << "could not get bucket info for bucket=" << bucket_name << dendl; return; @@ -497,7 +497,7 @@ void RGWOp_BILog_Info::execute() { return; } } else { /* !bucket_name.empty() */ - http_ret = store->getRados()->get_bucket_info(*s->sysobj_ctx, tenant_name, bucket_name, bucket_info, NULL, s->yield, NULL); + http_ret = store->getRados()->get_bucket_info(store->svc(), tenant_name, bucket_name, bucket_info, NULL, s->yield, NULL); if (http_ret < 0) { ldpp_dout(s, 5) << "could not get bucket info for bucket=" << bucket_name << dendl; return; @@ -561,7 +561,7 @@ void RGWOp_BILog_Delete::execute() { return; } } else { /* !bucket_name.empty() */ - http_ret = store->getRados()->get_bucket_info(*s->sysobj_ctx, tenant_name, bucket_name, bucket_info, NULL, s->yield, NULL); + http_ret = store->getRados()->get_bucket_info(store->svc(), tenant_name, bucket_name, bucket_info, NULL, s->yield, NULL); if (http_ret < 0) { ldpp_dout(s, 5) << "could not get bucket info for bucket=" << bucket_name << dendl; return; diff --git a/src/rgw/rgw_rest_pubsub_common.cc b/src/rgw/rgw_rest_pubsub_common.cc index 3b4c9636811..50f567f7fc1 100644 --- a/src/rgw/rgw_rest_pubsub_common.cc +++ b/src/rgw/rgw_rest_pubsub_common.cc @@ -149,7 +149,7 @@ int RGWPSCreateNotifOp::verify_permission() { const auto& id = s->owner.get_id(); - ret = store->getRados()->get_bucket_info(*s->sysobj_ctx, id.tenant, bucket_name, + ret = store->getRados()->get_bucket_info(store->svc(), id.tenant, bucket_name, bucket_info, nullptr, null_yield, nullptr); if (ret < 0) { ldout(s->cct, 1) << "failed to get bucket info, cannot verify ownership" << dendl; @@ -169,7 +169,7 @@ int RGWPSDeleteNotifOp::verify_permission() { return ret; } - ret = store->getRados()->get_bucket_info(*s->sysobj_ctx, s->owner.get_id().tenant, bucket_name, + ret = store->getRados()->get_bucket_info(store->svc(), s->owner.get_id().tenant, bucket_name, bucket_info, nullptr, null_yield, nullptr); if (ret < 0) { return ret; @@ -188,7 +188,7 @@ int RGWPSListNotifsOp::verify_permission() { return ret; } - ret = store->getRados()->get_bucket_info(*s->sysobj_ctx, s->owner.get_id().tenant, bucket_name, + ret = store->getRados()->get_bucket_info(store->svc(), s->owner.get_id().tenant, bucket_name, bucket_info, nullptr, null_yield, nullptr); if (ret < 0) { return ret; diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index 763bc4e8faf..b1c75fe8715 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -75,11 +75,11 @@ void list_all_buckets_end(struct req_state *s) s->formatter->close_section(); } -void dump_bucket(struct req_state *s, RGWBucketEnt& obj) +void dump_bucket(struct req_state *s, rgw::sal::RGWSalBucket& obj) { s->formatter->open_object_section("Bucket"); - s->formatter->dump_string("Name", obj.bucket.name); - dump_time(s, "CreationDate", &obj.creation_time); + s->formatter->dump_string("Name", obj.get_name()); + dump_time(s, "CreationDate", &obj.get_creation_time()); s->formatter->close_section(); } @@ -619,17 +619,17 @@ void RGWListBuckets_ObjStore_S3::send_response_begin(bool has_buckets) } } -void RGWListBuckets_ObjStore_S3::send_response_data(RGWUserBuckets& buckets) +void RGWListBuckets_ObjStore_S3::send_response_data(rgw::sal::RGWBucketList& buckets) { if (!sent_data) return; - map<string, RGWBucketEnt>& m = buckets.get_buckets(); - map<string, RGWBucketEnt>::iterator iter; + map<string, rgw::sal::RGWSalBucket*>& m = buckets.get_buckets(); + map<string, rgw::sal::RGWSalBucket*>::iterator iter; for (iter = m.begin(); iter != m.end(); ++iter) { - RGWBucketEnt obj = iter->second; - dump_bucket(s, obj); + rgw::sal::RGWSalBucket* obj = iter->second; + dump_bucket(s, *obj); } rgw_flush_formatter(s, s->formatter); } @@ -1473,10 +1473,10 @@ void RGWGetBucketWebsite_ObjStore_S3::send_response() rgw_flush_formatter_and_reset(s, s->formatter); } -static void dump_bucket_metadata(struct req_state *s, RGWBucketEnt& bucket) +static void dump_bucket_metadata(struct req_state *s, rgw::sal::RGWSalBucket* bucket) { - dump_header(s, "X-RGW-Object-Count", static_cast<long long>(bucket.count)); - dump_header(s, "X-RGW-Bytes-Used", static_cast<long long>(bucket.size)); + dump_header(s, "X-RGW-Object-Count", static_cast<long long>(bucket->get_count())); + dump_header(s, "X-RGW-Bytes-Used", static_cast<long long>(bucket->get_size())); } void RGWStatBucket_ObjStore_S3::send_response() @@ -1746,7 +1746,7 @@ int RGWPutObj_ObjStore_S3::get_params() return ret; } } - ret = store->getRados()->get_bucket_info(*s->sysobj_ctx, + ret = store->getRados()->get_bucket_info(store->svc(), copy_source_tenant_name, copy_source_bucket_name, copy_source_bucket_info, @@ -4196,7 +4196,7 @@ int RGWHandler_REST_S3Website::retarget(RGWOp* op, RGWOp** new_op) { if (!(s->prot_flags & RGW_REST_WEBSITE)) return 0; - int ret = store->getRados()->get_bucket_info(*s->sysobj_ctx, s->bucket_tenant, + int ret = store->getRados()->get_bucket_info(store->svc(), s->bucket_tenant, s->bucket_name, s->bucket_info, NULL, s->yield, &s->bucket_attrs); if (ret < 0) { diff --git a/src/rgw/rgw_rest_s3.h b/src/rgw/rgw_rest_s3.h index 97bd1db3ff0..24f7143f977 100644 --- a/src/rgw/rgw_rest_s3.h +++ b/src/rgw/rgw_rest_s3.h @@ -28,7 +28,6 @@ #include "rgw_auth.h" #include "rgw_auth_filters.h" #include "rgw_sts.h" -#include "rgw_sal.h" struct rgw_http_error { int http_ret; @@ -113,7 +112,7 @@ public: return 0; } void send_response_begin(bool has_buckets) override; - void send_response_data(RGWUserBuckets& buckets) override; + void send_response_data(rgw::sal::RGWBucketList& buckets) override; void send_response_end() override; }; diff --git a/src/rgw/rgw_rest_swift.cc b/src/rgw/rgw_rest_swift.cc index 4e894680830..41c904fd9cd 100644 --- a/src/rgw/rgw_rest_swift.cc +++ b/src/rgw/rgw_rest_swift.cc @@ -192,7 +192,7 @@ void RGWListBuckets_ObjStore_SWIFT::send_response_begin(bool has_buckets) } } -void RGWListBuckets_ObjStore_SWIFT::handle_listing_chunk(RGWUserBuckets&& buckets) +void RGWListBuckets_ObjStore_SWIFT::handle_listing_chunk(rgw::sal::RGWBucketList&& buckets) { if (wants_reversed) { /* Just store in the reversal buffer. Its content will be handled later, @@ -203,7 +203,7 @@ void RGWListBuckets_ObjStore_SWIFT::handle_listing_chunk(RGWUserBuckets&& bucket } } -void RGWListBuckets_ObjStore_SWIFT::send_response_data(RGWUserBuckets& buckets) +void RGWListBuckets_ObjStore_SWIFT::send_response_data(rgw::sal::RGWBucketList& buckets) { if (! sent_data) { return; @@ -213,22 +213,22 @@ void RGWListBuckets_ObjStore_SWIFT::send_response_data(RGWUserBuckets& buckets) * in applying the filter earlier as we really need to go through all * entries regardless of it (the headers like X-Account-Container-Count * aren't affected by specifying prefix). */ - const std::map<std::string, RGWBucketEnt>& m = buckets.get_buckets(); + const std::map<std::string, rgw::sal::RGWSalBucket*>& m = buckets.get_buckets(); for (auto iter = m.lower_bound(prefix); iter != m.end() && boost::algorithm::starts_with(iter->first, prefix); ++iter) { - dump_bucket_entry(iter->second); + dump_bucket_entry(*iter->second); } } -void RGWListBuckets_ObjStore_SWIFT::dump_bucket_entry(const RGWBucketEnt& obj) +void RGWListBuckets_ObjStore_SWIFT::dump_bucket_entry(const rgw::sal::RGWSalBucket& obj) { s->formatter->open_object_section("container"); - s->formatter->dump_string("name", obj.bucket.name); + s->formatter->dump_string("name", obj.get_name()); if (need_stats) { - s->formatter->dump_int("count", obj.count); - s->formatter->dump_int("bytes", obj.size); + s->formatter->dump_int("count", obj.get_count()); + s->formatter->dump_int("bytes", obj.get_size()); } s->formatter->close_section(); @@ -238,7 +238,7 @@ void RGWListBuckets_ObjStore_SWIFT::dump_bucket_entry(const RGWBucketEnt& obj) } } -void RGWListBuckets_ObjStore_SWIFT::send_response_data_reversed(RGWUserBuckets& buckets) +void RGWListBuckets_ObjStore_SWIFT::send_response_data_reversed(rgw::sal::RGWBucketList& buckets) { if (! sent_data) { return; @@ -248,7 +248,7 @@ void RGWListBuckets_ObjStore_SWIFT::send_response_data_reversed(RGWUserBuckets& * in applying the filter earlier as we really need to go through all * entries regardless of it (the headers like X-Account-Container-Count * aren't affected by specifying prefix). */ - std::map<std::string, RGWBucketEnt>& m = buckets.get_buckets(); + std::map<std::string, rgw::sal::RGWSalBucket*>& m = buckets.get_buckets(); auto iter = m.rbegin(); for (/* initialized above */; @@ -260,7 +260,7 @@ void RGWListBuckets_ObjStore_SWIFT::send_response_data_reversed(RGWUserBuckets& for (/* iter carried */; iter != m.rend() && boost::algorithm::starts_with(iter->first, prefix); ++iter) { - dump_bucket_entry(iter->second); + dump_bucket_entry(*iter->second); } } @@ -340,7 +340,7 @@ int RGWListBucket_ObjStore_SWIFT::get_params() } static void dump_container_metadata(struct req_state *, - const RGWBucketEnt&, + const rgw::sal::RGWSalBucket*, const RGWQuotaInfo&, const RGWBucketWebsiteConf&); @@ -450,16 +450,16 @@ next: } // RGWListBucket_ObjStore_SWIFT::send_response static void dump_container_metadata(struct req_state *s, - const RGWBucketEnt& bucket, + const rgw::sal::RGWSalBucket* bucket, const RGWQuotaInfo& quota, const RGWBucketWebsiteConf& ws_conf) { /* Adding X-Timestamp to keep align with Swift API */ dump_header(s, "X-Timestamp", utime_t(s->bucket_info.creation_time)); - dump_header(s, "X-Container-Object-Count", bucket.count); - dump_header(s, "X-Container-Bytes-Used", bucket.size); - dump_header(s, "X-Container-Bytes-Used-Actual", bucket.size_rounded); + dump_header(s, "X-Container-Object-Count", bucket->get_count()); + dump_header(s, "X-Container-Bytes-Used", bucket->get_size()); + dump_header(s, "X-Container-Bytes-Used-Actual", bucket->get_size_rounded()); if (s->object.empty()) { auto swift_policy = \ @@ -849,8 +849,7 @@ int RGWPutObj_ObjStore_SWIFT::update_slo_segment_size(rgw_slo_entry& entry) { if (bucket_name.compare(s->bucket.name) != 0) { RGWBucketInfo bucket_info; map<string, bufferlist> bucket_attrs; - auto obj_ctx = store->svc()->sysobj->init_obj_ctx(); - r = store->getRados()->get_bucket_info(obj_ctx, s->user->user_id.tenant, + r = store->getRados()->get_bucket_info(store->svc(), s->user->user_id.tenant, bucket_name, bucket_info, nullptr, s->yield, &bucket_attrs); if (r < 0) { @@ -2110,7 +2109,7 @@ void RGWFormPost::get_owner_info(const req_state* const s, /* Need to get user info of bucket owner. */ RGWBucketInfo bucket_info; - int ret = store->getRados()->get_bucket_info(*s->sysobj_ctx, + int ret = store->getRados()->get_bucket_info(store->svc(), bucket_tenant, bucket_name, bucket_info, nullptr, s->yield); if (ret < 0) { diff --git a/src/rgw/rgw_rest_swift.h b/src/rgw/rgw_rest_swift.h index 42878a1ffd1..41bff74e85f 100644 --- a/src/rgw/rgw_rest_swift.h +++ b/src/rgw/rgw_rest_swift.h @@ -39,7 +39,7 @@ class RGWListBuckets_ObjStore_SWIFT : public RGWListBuckets_ObjStore { bool need_stats; bool wants_reversed; std::string prefix; - std::vector<RGWUserBuckets> reverse_buffer; + std::vector<rgw::sal::RGWBucketList> reverse_buffer; uint64_t get_default_max() const override { return 0; @@ -53,11 +53,11 @@ public: ~RGWListBuckets_ObjStore_SWIFT() override {} int get_params() override; - void handle_listing_chunk(RGWUserBuckets&& buckets) override; + void handle_listing_chunk(rgw::sal::RGWBucketList&& buckets) override; void send_response_begin(bool has_buckets) override; - void send_response_data(RGWUserBuckets& buckets) override; - void send_response_data_reversed(RGWUserBuckets& buckets); - void dump_bucket_entry(const RGWBucketEnt& obj); + void send_response_data(rgw::sal::RGWBucketList& buckets) override; + void send_response_data_reversed(rgw::sal::RGWBucketList& buckets); + void dump_bucket_entry(const rgw::sal::RGWSalBucket& obj); void send_response_end() override; bool should_get_stats() override { return need_stats; } diff --git a/src/rgw/rgw_sal.cc b/src/rgw/rgw_sal.cc index 4c2d515837d..df40fdf7254 100644 --- a/src/rgw/rgw_sal.cc +++ b/src/rgw/rgw_sal.cc @@ -22,11 +22,48 @@ #include "common/errno.h" #include "rgw_sal.h" +#include "rgw_bucket.h" +#include "rgw_multi.h" #define dout_subsys ceph_subsys_rgw namespace rgw::sal { +int RGWRadosUser::list_buckets(const string& marker, const string& end_marker, + uint64_t max, bool need_stats, RGWBucketList &buckets) +{ + RGWUserBuckets ulist; + bool is_truncated = false; + int ret; + + ret = store->ctl()->user->list_buckets(user, marker, end_marker, max, need_stats, &ulist, + &is_truncated); + if (ret < 0) + return ret; + + buckets.set_truncated(is_truncated); + for (const auto& ent : ulist.get_buckets()) { + RGWRadosBucket *rb = new RGWRadosBucket(this->store, *this, ent.second); + buckets.add(rb); + } + + return 0; +} + +RGWBucketList::~RGWBucketList() +{ + for (auto itr = buckets.begin(); itr != buckets.end(); itr++) { + delete itr->second; + } + buckets.clear(); +} + +RGWSalBucket* RGWRadosUser::add_bucket(rgw_bucket& bucket, + ceph::real_time creation_time) +{ + return NULL; +} + RGWObject *RGWRadosBucket::create_object(const rgw_obj_key &key) { if (!object) { @@ -36,24 +73,151 @@ RGWObject *RGWRadosBucket::create_object(const rgw_obj_key &key) return object; } -RGWUser *RGWRadosStore::get_user(const rgw_user &u) +int RGWRadosBucket::remove_bucket(bool delete_children, optional_yield y) { - if (!user) { - user = new RGWRadosUser(this, u); + int ret; + map<RGWObjCategory, RGWStorageStats> stats; + std::vector<rgw_bucket_dir_entry> objs; + map<string, bool> common_prefixes; + RGWBucketInfo info; + string bucket_ver, master_ver; + + ret = get_bucket_info(info, y); + if (ret < 0) + return ret; + + ret = get_bucket_stats(info, RGW_NO_SHARD, &bucket_ver, &master_ver, stats); + if (ret < 0) + return ret; + + RGWRados::Bucket target(store->getRados(), info); + RGWRados::Bucket::List list_op(&target); + int max = 1000; + + list_op.params.list_versions = true; + list_op.params.allow_unordered = true; + + bool is_truncated = false; + do { + objs.clear(); + + ret = list_op.list_objects(max, &objs, &common_prefixes, &is_truncated, null_yield); + if (ret < 0) + return ret; + + if (!objs.empty() && !delete_children) { + lderr(store->ctx()) << "ERROR: could not remove non-empty bucket " << ent.bucket.name << dendl; + return -ENOTEMPTY; + } + + for (const auto& obj : objs) { + rgw_obj_key key(obj.key); + /* xxx dang */ + ret = rgw_remove_object(store, info, ent.bucket, key); + if (ret < 0 && ret != -ENOENT) { + return ret; + } + } + } while(is_truncated); + + string prefix, delimiter; + + ret = abort_bucket_multiparts(store, store->ctx(), info, prefix, delimiter); + if (ret < 0) { + return ret; } - return user; + ret = store->ctl()->bucket->sync_user_stats(info.owner, info); + if ( ret < 0) { + ldout(store->ctx(), 1) << "WARNING: failed sync user stats before bucket delete. ret=" << ret << dendl; + } + + RGWObjVersionTracker objv_tracker; + + // if we deleted children above we will force delete, as any that + // remain is detrius from a prior bug + ret = store->getRados()->delete_bucket(info, objv_tracker, null_yield, !delete_children); + if (ret < 0) { + lderr(store->ctx()) << "ERROR: could not remove bucket " << + ent.bucket.name << dendl; + return ret; + } + + ret = store->ctl()->bucket->unlink_bucket(info.owner, ent.bucket, null_yield, false); + if (ret < 0) { + lderr(store->ctx()) << "ERROR: unable to remove user bucket information" << dendl; + } + + return ret; } -RGWSalBucket *RGWRadosStore::create_bucket(RGWUser &u, const cls_user_bucket &b) +int RGWRadosBucket::get_bucket_info(RGWBucketInfo &info, optional_yield y) { - if (!bucket) { - bucket = new RGWRadosBucket(this, u, b); - } + return store->getRados()->get_bucket_info(store->svc(), ent.bucket.tenant, ent.bucket.name, info, + NULL, y); +} + +int RGWRadosBucket::get_bucket_stats(RGWBucketInfo& bucket_info, int shard_id, + std::string *bucket_ver, std::string *master_ver, + std::map<RGWObjCategory, RGWStorageStats>& stats, + std::string *max_marker, bool *syncstopped) +{ + return store->getRados()->get_bucket_stats(bucket_info, shard_id, bucket_ver, master_ver, stats, max_marker, syncstopped); +} + +int RGWRadosBucket::sync_user_stats(RGWBucketInfo& bucket_info) +{ + return store->ctl()->bucket->sync_user_stats(user.user, bucket_info, &ent); +} - return bucket; +int RGWRadosBucket::update_container_stats(void) +{ + int ret; + map<std::string, RGWBucketEnt> m; + + m[ent.bucket.name] = ent; + ret = store->getRados()->update_containers_stats(m); + if (!ret) + return -EEXIST; + if (ret < 0) + return ret; + + map<string, RGWBucketEnt>::iterator iter = m.find(ent.bucket.name); + if (iter == m.end()) + return -EINVAL; + + ent.count = iter->second.count; + ent.size = iter->second.size; + ent.size_rounded = iter->second.size_rounded; + ent.placement_rule = std::move(iter->second.placement_rule); + + return 0; +} + +int RGWRadosBucket::set_acl(RGWAccessControlPolicy &acl, RGWBucketInfo& bucket_info, optional_yield y) +{ + bufferlist aclbl; + + acls = acl; + acl.encode(aclbl); + + return store->ctl()->bucket->set_acl(acl.get_owner(), ent.bucket, bucket_info, aclbl, null_yield); +} + +RGWUser *RGWRadosStore::get_user(const rgw_user &u) +{ + return new RGWRadosUser(this, u); } +//RGWSalBucket *RGWRadosStore::create_bucket(RGWUser &u, const rgw_bucket &b) +//{ + //if (!bucket) { + //bucket = new RGWRadosBucket(this, u, b); + //} +// + //return bucket; +//} +// void RGWRadosStore::finalize(void) { if (rados) rados->finalize(); diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h index c472ca0ed06..bd803b741f2 100644 --- a/src/rgw/rgw_sal.h +++ b/src/rgw/rgw_sal.h @@ -25,8 +25,8 @@ namespace rgw { namespace sal { class RGWUser; class RGWSalBucket; class RGWObject; +class RGWBucketList; -typedef std::vector<RGWSalBucket> RGWBucketList; typedef std::map<string, string> RGWAttrs; class RGWStore { @@ -35,8 +35,8 @@ class RGWStore { virtual ~RGWStore() = default; virtual RGWUser* get_user(const rgw_user &u) = 0; - virtual RGWSalBucket* get_bucket(RGWUser &u, const cls_user_bucket &b) = 0; - virtual RGWSalBucket* create_bucket(RGWUser &u, const cls_user_bucket &b) = 0; + virtual RGWSalBucket* get_bucket(RGWUser &u, const rgw_bucket &b) = 0; + //virtual RGWSalBucket* create_bucket(RGWUser &u, const rgw_bucket &b) = 0; virtual RGWBucketList* list_buckets(void) = 0; virtual void finalize(void)=0; @@ -53,16 +53,24 @@ class RGWUser { RGWUser(const rgw_user &_u) : user(_u) {} virtual ~RGWUser() = default; - virtual RGWBucketList* list_buckets(void) = 0; + virtual int list_buckets(const string& marker, const string& end_marker, + uint64_t max, bool need_stats, RGWBucketList &buckets) = 0; + virtual RGWSalBucket* add_bucket(rgw_bucket& bucket, ceph::real_time creation_time) = 0; + friend class RGWSalBucket; + + std::string& get_tenant() { return user.tenant; } + /* xxx dang temporary; will be removed when User is complete */ + rgw_user& get_user() { return user; } }; class RGWSalBucket { protected: - cls_user_bucket ub; + RGWBucketEnt ent; public: - RGWSalBucket() : ub() {} - RGWSalBucket(const cls_user_bucket &_b) : ub(_b) {} + RGWSalBucket() : ent() {} + RGWSalBucket(const rgw_bucket &_b) { ent.bucket = _b; } + RGWSalBucket(const RGWBucketEnt &_e) : ent(_e) {} virtual ~RGWSalBucket() = default; virtual RGWObject* get_object(const rgw_obj_key &key) = 0; @@ -70,9 +78,60 @@ class RGWSalBucket { virtual RGWObject* create_object(const rgw_obj_key &key /* Attributes */) = 0; virtual RGWAttrs& get_attrs(void) = 0; virtual int set_attrs(RGWAttrs &attrs) = 0; - virtual int delete_bucket(void) = 0; + virtual int remove_bucket(bool delete_children, optional_yield y) = 0; virtual RGWAccessControlPolicy& get_acl(void) = 0; - virtual int set_acl(const RGWAccessControlPolicy &acl) = 0; + virtual int set_acl(RGWAccessControlPolicy &acl, RGWBucketInfo& bucket_info, optional_yield y) = 0; + virtual int get_bucket_info(RGWBucketInfo &info, optional_yield y) = 0; + virtual int get_bucket_stats(RGWBucketInfo& bucket_info, int shard_id, + std::string *bucket_ver, std::string *master_ver, + std::map<RGWObjCategory, RGWStorageStats>& stats, + std::string *max_marker = nullptr, + bool *syncstopped = nullptr) = 0; + virtual int sync_user_stats(RGWBucketInfo &info) = 0; + virtual int update_container_stats(void) = 0; + + std::string get_name() const { return ent.bucket.name; } + std::string get_tenant() const { return ent.bucket.tenant; } + std::string get_marker() const { return ent.bucket.marker; } + std::string get_bucket_id() const { return ent.bucket.bucket_id; } + size_t get_size() const { return ent.size; } + size_t get_size_rounded() const { return ent.size_rounded; } + uint64_t get_count() const { return ent.count; } + rgw_placement_rule get_placement_rule() const { return ent.placement_rule; } + ceph::real_time& get_creation_time() { return ent.creation_time; }; + + /* dang - This is temporary, until the API is completed */ + rgw_bucket& get_bi() { return ent.bucket; } + + friend inline ostream& operator<<(ostream& out, const RGWSalBucket &b) { + out << b.ent.bucket; + return out; + } + + + friend class RGWBucketList; + protected: + virtual void set_ent(RGWBucketEnt &_ent) { ent = _ent; } +}; + +class RGWBucketList { + std::map<std::string, RGWSalBucket*> buckets; + bool truncated; + +public: + RGWBucketList() : buckets(), truncated(false) {} + RGWBucketList(RGWBucketList&&) = default; + RGWBucketList& operator=(const RGWBucketList&) = default; + ~RGWBucketList(); + + map<string, RGWSalBucket*>& get_buckets() { return buckets; } + bool is_truncated(void) { return truncated; } + void set_truncated(bool trunc) { truncated = trunc; } + void add(RGWSalBucket* bucket) { + buckets[bucket->ent.bucket.name] = bucket; + } + size_t count() { return buckets.size(); } + }; class RGWObject { @@ -104,7 +163,11 @@ class RGWRadosUser : public RGWUser { RGWRadosUser(RGWRadosStore *_st, const rgw_user &_u) : RGWUser(_u), store(_st) { } RGWRadosUser() {} - RGWBucketList* list_buckets(void) { return new RGWBucketList(); } + int list_buckets(const string& marker, const string& end_marker, + uint64_t max, bool need_stats, RGWBucketList &buckets); + RGWSalBucket* add_bucket(rgw_bucket& bucket, ceph::real_time creation_time); + + friend class RGWRadosBucket; }; class RGWRadosObject : public RGWObject { @@ -145,13 +208,14 @@ class RGWRadosBucket : public RGWSalBucket { public: RGWRadosBucket() - : object(nullptr), + : store(nullptr), + object(nullptr), attrs(), acls(), user() { } - RGWRadosBucket(RGWRadosStore *_st, RGWUser &_u, const cls_user_bucket &_b) + RGWRadosBucket(RGWRadosStore *_st, RGWUser &_u, const rgw_bucket &_b) : RGWSalBucket(_b), store(_st), object(nullptr), @@ -160,40 +224,54 @@ class RGWRadosBucket : public RGWSalBucket { user(dynamic_cast<RGWRadosUser&>(_u)) { } + RGWRadosBucket(RGWRadosStore *_st, RGWUser &_u, const RGWBucketEnt &_e) + : RGWSalBucket(_e), + store(_st), + object(nullptr), + attrs(), + acls(), + user(dynamic_cast<RGWRadosUser&>(_u)) { + } + + ~RGWRadosBucket() { } + RGWObject* get_object(const rgw_obj_key &key) { return object; } RGWBucketList* list(void) { return new RGWBucketList(); } RGWObject* create_object(const rgw_obj_key &key /* Attributes */) override; RGWAttrs& get_attrs(void) { return attrs; } int set_attrs(RGWAttrs &a) { attrs = a; return 0; } - int delete_bucket(void) { return 0; } + virtual int remove_bucket(bool delete_children, optional_yield y) override; RGWAccessControlPolicy& get_acl(void) { return acls; } - int set_acl(const RGWAccessControlPolicy &acl) { acls = acl; return 0; } + virtual int set_acl(RGWAccessControlPolicy &acl, RGWBucketInfo& bucket_info, optional_yield y) override; + virtual int get_bucket_info(RGWBucketInfo &info, optional_yield y) override; + virtual int get_bucket_stats(RGWBucketInfo& bucket_info, int shard_id, + std::string *bucket_ver, std::string *master_ver, + std::map<RGWObjCategory, RGWStorageStats>& stats, + std::string *max_marker = nullptr, + bool *syncstopped = nullptr) override; + virtual int sync_user_stats(RGWBucketInfo &info) override; + virtual int update_container_stats(void) override; }; class RGWRadosStore : public RGWStore { private: RGWRados *rados; - RGWRadosUser *user; RGWRadosBucket *bucket; + RGWUserCtl *user_ctl; public: RGWRadosStore() : rados(nullptr), - user(nullptr), bucket(nullptr) { } ~RGWRadosStore() { - if (bucket) - delete bucket; - if (user) - delete user; - if (rados) - delete rados; + delete bucket; + delete rados; } virtual RGWUser* get_user(const rgw_user &u); - virtual RGWSalBucket* get_bucket(RGWUser &u, const cls_user_bucket &b) { return bucket; } - virtual RGWSalBucket* create_bucket(RGWUser &u, const cls_user_bucket &b); + virtual RGWSalBucket* get_bucket(RGWUser &u, const rgw_bucket &b) { return bucket; } + //virtual RGWSalBucket* create_bucket(RGWUser &u, const rgw_bucket &b); virtual RGWBucketList* list_buckets(void) { return new RGWBucketList(); } void setRados(RGWRados * st) { rados = st; } @@ -202,6 +280,8 @@ class RGWRadosStore : public RGWStore { RGWServices *svc() { return &rados->svc; } RGWCtl *ctl() { return &rados->ctl; } + void setUserCtl(RGWUserCtl *_ctl) { user_ctl = _ctl; } + void finalize(void) override; virtual CephContext *ctx(void) { return rados->ctx(); } diff --git a/src/rgw/rgw_tools.cc b/src/rgw/rgw_tools.cc index 7b25c982ee4..948862a591f 100644 --- a/src/rgw/rgw_tools.cc +++ b/src/rgw/rgw_tools.cc @@ -424,7 +424,7 @@ int RGWDataAccess::Bucket::finish_init() int RGWDataAccess::Bucket::init() { - int ret = sd->store->getRados()->get_bucket_info(*sd->sysobj_ctx, + int ret = sd->store->getRados()->get_bucket_info(sd->store->svc(), tenant, name, bucket_info, &mtime, diff --git a/src/rgw/rgw_user.cc b/src/rgw/rgw_user.cc index 359237c6eca..6876d2320c5 100644 --- a/src/rgw/rgw_user.cc +++ b/src/rgw/rgw_user.cc @@ -49,58 +49,48 @@ void rgw_get_anon_user(RGWUserInfo& info) int rgw_user_sync_all_stats(rgw::sal::RGWRadosStore *store, const rgw_user& user_id) { + rgw::sal::RGWBucketList user_buckets; + rgw::sal::RGWRadosUser user(store, user_id); + CephContext *cct = store->ctx(); size_t max_entries = cct->_conf->rgw_list_buckets_max_chunk; bool is_truncated = false; string marker; int ret; - RGWSysObjectCtx obj_ctx = store->svc()->sysobj->init_obj_ctx(); do { - RGWUserBuckets user_buckets; - ret = store->ctl()->user->list_buckets(user_id, - marker, string(), - max_entries, - false, - &user_buckets, - &is_truncated); + ret = user.list_buckets(marker, string(), max_entries, false, user_buckets); if (ret < 0) { ldout(cct, 0) << "failed to read user buckets: ret=" << ret << dendl; return ret; } - map<string, RGWBucketEnt>& buckets = user_buckets.get_buckets(); - for (map<string, RGWBucketEnt>::iterator i = buckets.begin(); + map<string, rgw::sal::RGWSalBucket*>& buckets = user_buckets.get_buckets(); + for (map<string, rgw::sal::RGWSalBucket*>::iterator i = buckets.begin(); i != buckets.end(); ++i) { marker = i->first; - RGWBucketEnt& bucket_ent = i->second; + rgw::sal::RGWSalBucket* bucket = i->second; RGWBucketInfo bucket_info; - rgw_bucket bucket; - bucket.tenant = user_id.tenant; - bucket.name = bucket_ent.bucket.name; - - ret = store->getRados()->get_bucket_info(obj_ctx, user_id.tenant, bucket_ent.bucket.name, - bucket_info, nullptr, null_yield, nullptr); + ret = bucket->get_bucket_info(bucket_info, null_yield); if (ret < 0) { - ldout(cct, 0) << "ERROR: could not read bucket info: bucket=" << bucket_ent.bucket << " ret=" << ret << dendl; + ldout(cct, 0) << "ERROR: could not read bucket info: bucket=" << bucket << " ret=" << ret << dendl; continue; } - RGWBucketEnt ent; - ret = store->ctl()->bucket->sync_user_stats(user_id, bucket_info, &ent); + ret = bucket->sync_user_stats(bucket_info); if (ret < 0) { ldout(cct, 0) << "ERROR: could not sync bucket stats: ret=" << ret << dendl; return ret; } - ret = store->getRados()->check_bucket_shards(bucket_info, bucket_info.bucket, ent.count); + ret = store->getRados()->check_bucket_shards(bucket_info, bucket_info.bucket, bucket->get_count()); if (ret < 0) { ldout(cct, 0) << "ERROR in check_bucket_shards: " << cpp_strerror(-ret)<< dendl; } } } while (is_truncated); - ret = store->ctl()->user->complete_flush_stats(user_id); + ret = store->ctl()->user->complete_flush_stats(user.get_user()); if (ret < 0) { cerr << "ERROR: failed to complete syncing user stats: ret=" << ret << std::endl; return ret; @@ -114,34 +104,33 @@ int rgw_user_get_all_buckets_stats(rgw::sal::RGWRadosStore *store, const rgw_use CephContext *cct = store->ctx(); size_t max_entries = cct->_conf->rgw_list_buckets_max_chunk; bool done; - bool is_truncated; string marker; int ret; do { - RGWUserBuckets user_buckets; - ret = rgw_read_user_buckets(store, user_id, user_buckets, marker, - string(), max_entries, false, &is_truncated); + rgw::sal::RGWBucketList buckets; + ret = rgw_read_user_buckets(store, user_id, buckets, marker, + string(), max_entries, false); if (ret < 0) { ldout(cct, 0) << "failed to read user buckets: ret=" << ret << dendl; return ret; } - map<string, RGWBucketEnt>& buckets = user_buckets.get_buckets(); - for (const auto& i : buckets) { + std::map<std::string, rgw::sal::RGWSalBucket*>& m = buckets.get_buckets(); + for (const auto& i : m) { marker = i.first; - const RGWBucketEnt& bucket_ent = i.second; + rgw::sal::RGWSalBucket* bucket_ent = i.second; RGWBucketEnt stats; - ret = store->ctl()->bucket->read_bucket_stats(bucket_ent.bucket, &stats, null_yield); + ret = store->ctl()->bucket->read_bucket_stats(bucket_ent->get_bi(), &stats, null_yield); if (ret < 0) { ldout(cct, 0) << "ERROR: could not get bucket stats: ret=" << ret << dendl; return ret; } cls_user_bucket_entry entry; stats.convert(&entry); - buckets_usage_map.emplace(bucket_ent.bucket.name, entry); + buckets_usage_map.emplace(bucket_ent->get_name(), entry); } - done = (buckets.size() < max_entries); + done = (buckets.count() < max_entries); } while (!done); return 0; @@ -1613,19 +1602,18 @@ int RGWUser::execute_rename(RGWUserAdminOpState& op_state, std::string *err_msg) } } - rgw_user& old_uid = op_state.get_user_id(); + rgw::sal::RGWUser* old_user = new rgw::sal::RGWRadosUser(store, op_state.get_user_id()); RGWUserInfo old_user_info = op_state.get_user_info(); - - rgw_user& uid = op_state.get_new_uid(); - if (old_uid.tenant != uid.tenant) { + rgw::sal::RGWUser* new_user = new rgw::sal::RGWRadosUser(store, op_state.get_new_uid()); + if (old_user->get_tenant() != new_user->get_tenant()) { set_err_msg(err_msg, "users have to be under the same tenant namespace " - + old_uid.tenant + " != " + uid.tenant); + + old_user->get_tenant() + " != " + new_user->get_tenant()); return -EINVAL; } // create a stub user and write only the uid index and buckets object RGWUserInfo stub_user_info; - stub_user_info.user_id = uid; + stub_user_info.user_id = new_user->get_user(); RGWObjVersionTracker objv; const bool exclusive = !op_state.get_overwrite_new_user(); // overwrite if requested @@ -1643,85 +1631,76 @@ int RGWUser::execute_rename(RGWUserAdminOpState& op_state, std::string *err_msg) return ret; } - ACLOwner owner; RGWAccessControlPolicy policy_instance; - policy_instance.create_default(uid, old_user_info.display_name); - owner = policy_instance.get_owner(); - bufferlist aclbl; - policy_instance.encode(aclbl); + policy_instance.create_default(new_user->get_user(), old_user_info.display_name); //unlink and link buckets to new user - bool is_truncated = false; string marker; string obj_marker; CephContext *cct = store->ctx(); size_t max_buckets = cct->_conf->rgw_list_buckets_max_chunk; RGWBucketCtl* bucket_ctl = store->ctl()->bucket; + rgw::sal::RGWBucketList buckets; do { - RGWUserBuckets buckets; - ret = user_ctl->list_buckets(old_uid, marker, "", max_buckets, - false, &buckets, &is_truncated); + ret = old_user->list_buckets(marker, "", max_buckets, false, buckets); if (ret < 0) { set_err_msg(err_msg, "unable to list user buckets"); return ret; } - map<string, bufferlist> attrs; - map<std::string, RGWBucketEnt>& m = buckets.get_buckets(); - std::map<std::string, RGWBucketEnt>::iterator it; + map<std::string, rgw::sal::RGWSalBucket*>& m = buckets.get_buckets(); + std::map<std::string, rgw::sal::RGWSalBucket*>::iterator it; for (it = m.begin(); it != m.end(); ++it) { - RGWBucketEnt obj = it->second; + rgw::sal::RGWSalBucket* bucket = it->second; marker = it->first; RGWBucketInfo bucket_info; - ret = bucket_ctl->read_bucket_info(obj.bucket, &bucket_info, null_yield, - RGWBucketCtl::BucketInstance::GetParams() - .set_attrs(&attrs)); + ret = bucket->get_bucket_info(bucket_info, null_yield); if (ret < 0) { - set_err_msg(err_msg, "failed to fetch bucket info for bucket=" + obj.bucket.name); + set_err_msg(err_msg, "failed to fetch bucket info for bucket=" + bucket->get_name()); return ret; } - ret = bucket_ctl->set_acl(owner, obj.bucket, bucket_info, aclbl, null_yield); + ret = bucket->set_acl(policy_instance, bucket_info, null_yield); if (ret < 0) { - set_err_msg(err_msg, "failed to set acl on bucket " + obj.bucket.name); + set_err_msg(err_msg, "failed to set acl on bucket " + bucket->get_name()); return ret; } RGWBucketEntryPoint ep; ep.bucket = bucket_info.bucket; - ep.owner = uid; + ep.owner = new_user->get_user(); ep.creation_time = bucket_info.creation_time; ep.linked = true; map<string, bufferlist> ep_attrs; rgw_ep_info ep_data{ep, ep_attrs}; - ret = bucket_ctl->link_bucket(uid, bucket_info.bucket, ceph::real_time(), - null_yield, true, &ep_data); + ret = bucket_ctl->link_bucket(new_user->get_user(), bucket_info.bucket, + ceph::real_time(), null_yield, true, &ep_data); if (ret < 0) { - set_err_msg(err_msg, "failed to link bucket " + obj.bucket.name + " to new user"); + set_err_msg(err_msg, "failed to link bucket " + bucket->get_name()); return ret; } - ret = bucket_ctl->chown(store, bucket_info, uid, old_user_info.display_name, - obj_marker, null_yield); + ret = bucket_ctl->chown(store, bucket_info, new_user->get_user(), + old_user_info.display_name, obj_marker, null_yield); if (ret < 0) { set_err_msg(err_msg, "failed to run bucket chown" + cpp_strerror(-ret)); return ret; } } - } while (is_truncated); + } while (buckets.is_truncated()); // update the 'stub user' with all of the other fields and rewrite all of the // associated index objects RGWUserInfo& user_info = op_state.get_user_info(); - user_info.user_id = uid; + user_info.user_id = new_user->get_user(); op_state.objv = objv; - rename_swift_keys(uid, user_info.swift_keys); + rename_swift_keys(new_user->get_user(), user_info.swift_keys); return update(op_state, err_msg); } @@ -1907,28 +1886,27 @@ int RGWUser::execute_remove(RGWUserAdminOpState& op_state, std::string *err_msg, return -ENOENT; } - bool is_truncated = false; + rgw::sal::RGWBucketList buckets; string marker; CephContext *cct = store->ctx(); size_t max_buckets = cct->_conf->rgw_list_buckets_max_chunk; do { - RGWUserBuckets buckets; ret = rgw_read_user_buckets(store, uid, buckets, marker, string(), - max_buckets, false, &is_truncated); + max_buckets, false); if (ret < 0) { set_err_msg(err_msg, "unable to read user bucket info"); return ret; } - map<std::string, RGWBucketEnt>& m = buckets.get_buckets(); + std::map<std::string, rgw::sal::RGWSalBucket*>& m = buckets.get_buckets(); if (!m.empty() && !purge_data) { set_err_msg(err_msg, "must specify purge data to remove user with buckets"); return -EEXIST; // change to code that maps to 409: conflict } - std::map<std::string, RGWBucketEnt>::iterator it; + std::map<std::string, rgw::sal::RGWSalBucket*>::iterator it; for (it = m.begin(); it != m.end(); ++it) { - ret = rgw_remove_bucket(store, ((*it).second).bucket, true, y); + ret = it->second->remove_bucket(true, y); if (ret < 0) { set_err_msg(err_msg, "unable to delete user data"); return ret; @@ -1937,7 +1915,7 @@ int RGWUser::execute_remove(RGWUserAdminOpState& op_state, std::string *err_msg, marker = it->first; } - } while (is_truncated); + } while (buckets.is_truncated()); ret = user_ctl->remove_info(user_info, y, RGWUserCtl::RemoveParams() .set_objv_tracker(&op_state.objv)); @@ -2057,32 +2035,31 @@ int RGWUser::execute_modify(RGWUserAdminOpState& op_state, std::string *err_msg) __u8 suspended = op_state.get_suspension_status(); user_info.suspended = suspended; - RGWUserBuckets buckets; + rgw::sal::RGWBucketList buckets; if (user_id.empty()) { set_err_msg(err_msg, "empty user id passed...aborting"); return -EINVAL; } - bool is_truncated = false; string marker; CephContext *cct = store->ctx(); size_t max_buckets = cct->_conf->rgw_list_buckets_max_chunk; do { ret = rgw_read_user_buckets(store, user_id, buckets, marker, string(), - max_buckets, false, &is_truncated); + max_buckets, false); if (ret < 0) { set_err_msg(err_msg, "could not get buckets for uid: " + user_id.to_str()); return ret; } - map<string, RGWBucketEnt>& m = buckets.get_buckets(); - map<string, RGWBucketEnt>::iterator iter; + std::map<std::string, rgw::sal::RGWSalBucket*>& m = buckets.get_buckets(); + std::map<std::string, rgw::sal::RGWSalBucket*>::iterator iter; vector<rgw_bucket> bucket_names; for (iter = m.begin(); iter != m.end(); ++iter) { - RGWBucketEnt obj = iter->second; - bucket_names.push_back(obj.bucket); + rgw::sal::RGWSalBucket* obj = iter->second; + bucket_names.push_back(obj->get_bi()); marker = iter->first; } @@ -2093,7 +2070,7 @@ int RGWUser::execute_modify(RGWUserAdminOpState& op_state, std::string *err_msg) return ret; } - } while (is_truncated); + } while (buckets.is_truncated()); } if (op_state.mfa_ids_specified) { |