From 4ffc765c4c5debc665ade7769c4647c3a7278fd2 Mon Sep 17 00:00:00 2001 From: albIN7 Date: Wed, 15 May 2019 15:19:11 +0530 Subject: rgw:listobjectsv2 ListObjectsV2 is the revised List Objects API. Returns some or all (up to 1000) of the objects in a bucket. You can use the request parameters as selection criteria to return a subset of the objects in a bucket. Signed-off-by: Albin Antony --- src/rgw/rgw_op.h | 1 + src/rgw/rgw_rest_s3.cc | 420 +++++++++++++++++++++++++++++++++++++------------ src/rgw/rgw_rest_s3.h | 19 ++- 3 files changed, 339 insertions(+), 101 deletions(-) (limited to 'src') diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index 49ed254a4ff..e6a4cf63d02 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -780,6 +780,7 @@ protected: rgw_obj_key next_marker; rgw_obj_key end_marker; string max_keys; + string startAfter; string delimiter; string encoding_type; bool list_versions; diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index ae1065160ad..39d8b5a9fc7 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -753,52 +753,78 @@ void RGWGetUsage_ObjStore_S3::send_response() rgw_flush_formatter_and_reset(s, s->formatter); } -int RGWListBucket_ObjStore_S3::get_params() +int RGWListBucket_ObjStore_S3::get_common_params() + { - list_versions = s->info.args.exists("versions"); - prefix = s->info.args.get("prefix"); - if (!list_versions) { - marker = s->info.args.get("marker"); - } else { - marker.name = s->info.args.get("key-marker"); - marker.instance = s->info.args.get("version-id-marker"); - } +list_versions = s->info.args.exists("versions"); +prefix = s->info.args.get("prefix"); - // non-standard - s->info.args.get_bool("allow-unordered", &allow_unordered, false); +// non-standard +s->info.args.get_bool("allow-unordered", &allow_unordered, false); +delimiter = s->info.args.get("delimiter"); +max_keys = s->info.args.get("max-keys"); +op_ret = parse_max_keys(); +if (op_ret < 0) { + return op_ret; +} - delimiter = s->info.args.get("delimiter"); +encoding_type = s->info.args.get("encoding-type"); +if (s->system_request) { + s->info.args.get_bool("objs-container", &objs_container, false); + const char *shard_id_str = s->info.env->get("HTTP_RGWX_SHARD_ID"); + if (shard_id_str) { + string err; + shard_id = strict_strtol(shard_id_str, 10, &err); + if (!err.empty()) { + ldout(s->cct, 5) << "bad shard id specified: " << shard_id_str << dendl; + return -EINVAL; +} +} + else { + shard_id = s->bucket_instance_shard_id; +} +} +return 0; +} - max_keys = s->info.args.get("max-keys"); - op_ret = parse_max_keys(); - if (op_ret < 0) { - return op_ret; - } +int RGWListBucket_ObjStore_S3::get_params() +{ +int ret = get_common_params(); +if (ret < 0) { + return ret; +} +if (!list_versions) { + marker = s->info.args.get("marker"); +} +else { + marker.name = s->info.args.get("key-marker"); + marker.instance = s->info.args.get("version-id-marker"); +} +return 0; +} - encoding_type = s->info.args.get("encoding-type"); - if (s->system_request) { - s->info.args.get_bool("objs-container", &objs_container, false); - const char *shard_id_str = s->info.env->get("HTTP_RGWX_SHARD_ID"); - if (shard_id_str) { - string err; - shard_id = strict_strtol(shard_id_str, 10, &err); - if (!err.empty()) { - ldpp_dout(this, 5) << "bad shard id specified: " << shard_id_str << dendl; - return -EINVAL; - } - } else { - shard_id = s->bucket_instance_shard_id; - } - } - return 0; +int RGWListBucket_ObjStore_S3v2::get_params() +{ +int ret = get_common_params(); +if (ret < 0) { + return ret; +} +s->info.args.get_bool("fetch-owner", &fetchOwner, false); +startAfter = s->info.args.get("start-after"); +marker = s->info.args.get("ContinuationToken"); +if(marker.empty()) { + marker = startAfter; +} +return 0; } -void RGWListBucket_ObjStore_S3::send_versioned_response() +void RGWListBucket_ObjStore_S3::send_common_versioned_response() { - s->formatter->open_object_section_in_ns("ListVersionsResult", XMLNS_AWS_S3); - if (!s->bucket_tenant.empty()) + + if (!s->bucket_tenant.empty()) { s->formatter->dump_string("Tenant", s->bucket_tenant); + } s->formatter->dump_string("Name", s->bucket_name); s->formatter->dump_string("Prefix", prefix); s->formatter->dump_string("KeyMarker", marker.name); @@ -811,11 +837,40 @@ void RGWListBucket_ObjStore_S3::send_versioned_response() s->formatter->dump_string("NextVersionIdMarker", next_marker.instance); } s->formatter->dump_int("MaxKeys", max); - if (!delimiter.empty()) + if (!delimiter.empty()) { s->formatter->dump_string("Delimiter", delimiter); - + } s->formatter->dump_string("IsTruncated", (max && is_truncated ? "true" - : "false")); + : "false")); + + if (!common_prefixes.empty()) { + map::iterator pref_iter; + for (pref_iter = common_prefixes.begin(); + pref_iter != common_prefixes.end(); ++pref_iter) { + s->formatter->open_array_section("CommonPrefixes"); + s->formatter->dump_string("Prefix", pref_iter->first); + s->formatter->close_section(); + } + } + } + + + +void RGWListBucket_ObjStore_S3::send_versioned_response() +{ + s->formatter->open_object_section_in_ns("ListVersionsResult", XMLNS_AWS_S3); + RGWListBucket_ObjStore_S3::send_common_versioned_response(); + s->formatter->dump_string("KeyMarker", marker.name); + s->formatter->dump_string("VersionIdMarker", marker.instance); + if (is_truncated && !next_marker.empty()) { + s->formatter->dump_string("NextKeyMarker", next_marker.name); + if (next_marker.instance.empty()) { + s->formatter->dump_string("NextVersionIdMarker", "null"); + } + else { + s->formatter->dump_string("NextVersionIdMarker", next_marker.instance); + } + } bool encode_key = false; if (strcasecmp(encoding_type.c_str(), "url") == 0) { @@ -827,26 +882,27 @@ void RGWListBucket_ObjStore_S3::send_versioned_response() if (objs_container) { s->formatter->open_array_section("Entries"); } - + vector::iterator iter; for (iter = objs.begin(); iter != objs.end(); ++iter) { const char *section_name = (iter->is_delete_marker() ? "DeleteMarker" - : "Version"); + : "Version"); s->formatter->open_object_section(section_name); if (objs_container) { s->formatter->dump_bool("IsDeleteMarker", iter->is_delete_marker()); } rgw_obj_key key(iter->key); if (encode_key) { - string key_name; - url_encode(key.name, key_name); - s->formatter->dump_string("Key", key_name); - } else { - s->formatter->dump_string("Key", key.name); + string key_name; + url_encode(key.name, key_name); + s->formatter->dump_string("Key", key_name); + } + else { + s->formatter->dump_string("Key", key.name); } string version_id = key.instance; if (version_id.empty()) { - version_id = "null"; + version_id = "null"; } if (s->system_request) { if (iter->versioned_epoch > 0) { @@ -860,10 +916,10 @@ void RGWListBucket_ObjStore_S3::send_versioned_response() s->formatter->dump_bool("IsLatest", iter->is_current()); dump_time(s, "LastModified", &iter->meta.mtime); if (!iter->is_delete_marker()) { - s->formatter->dump_format("ETag", "\"%s\"", iter->meta.etag.c_str()); - s->formatter->dump_int("Size", iter->meta.accounted_size); - auto& storage_class = rgw_placement_rule::get_canonical_storage_class(iter->meta.storage_class); - s->formatter->dump_string("StorageClass", storage_class.c_str()); + s->formatter->dump_format("ETag", "\"%s\"", iter->meta.etag.c_str()); + s->formatter->dump_int("Size", iter->meta.accounted_size); + auto& storage_class = rgw_placement_rule::get_canonical_storage_class(iter->meta.storage_class); + s->formatter->dump_string("StorageClass", storage_class.c_str()); } dump_owner(s, iter->meta.owner, iter->meta.owner_display_name); if (iter->meta.appendable) { @@ -873,56 +929,114 @@ void RGWListBucket_ObjStore_S3::send_versioned_response() } s->formatter->close_section(); } - if (objs_container) { - s->formatter->close_section(); - } + + s->formatter->close_section(); + rgw_flush_formatter_and_reset(s, s->formatter); +} +} + + +void RGWListBucket_ObjStore_S3::send_common_response() +{ + if (!s->bucket_tenant.empty()) { + s->formatter->dump_string("Tenant", s->bucket_tenant); + } + s->formatter->dump_string("Name", s->bucket_name); + s->formatter->dump_string("Prefix", prefix); + s->formatter->dump_int("MaxKeys", max); + if (!delimiter.empty()) { + s->formatter->dump_string("Delimiter", delimiter); + } + s->formatter->dump_string("IsTruncated", (max && is_truncated ? "true" + : "false")); if (!common_prefixes.empty()) { map::iterator pref_iter; for (pref_iter = common_prefixes.begin(); - pref_iter != common_prefixes.end(); ++pref_iter) { - s->formatter->open_array_section("CommonPrefixes"); - s->formatter->dump_string("Prefix", pref_iter->first); - s->formatter->close_section(); + pref_iter != common_prefixes.end(); ++pref_iter) { + s->formatter->open_array_section("CommonPrefixes"); + s->formatter->dump_string("Prefix", pref_iter->first); + s->formatter->close_section(); } } } - s->formatter->close_section(); - rgw_flush_formatter_and_reset(s, s->formatter); -} + + void RGWListBucket_ObjStore_S3::send_response() { - if (op_ret < 0) + +if(op_ret < 0) { set_req_state_err(s, op_ret); + } dump_errno(s); // Explicitly use chunked transfer encoding so that we can stream the result // to the user without having to wait for the full length of it. end_header(s, this, "application/xml", CHUNKED_TRANSFER_ENCODING); dump_start(s); - if (op_ret < 0) + if (op_ret < 0) { return; - + } if (list_versions) { send_versioned_response(); return; } s->formatter->open_object_section_in_ns("ListBucketResult", XMLNS_AWS_S3); - if (!s->bucket_tenant.empty()) - s->formatter->dump_string("Tenant", s->bucket_tenant); - s->formatter->dump_string("Name", s->bucket_name); - s->formatter->dump_string("Prefix", prefix); - s->formatter->dump_string("Marker", marker.name); - if (is_truncated && !next_marker.empty()) - s->formatter->dump_string("NextMarker", next_marker.name); - s->formatter->dump_int("MaxKeys", max); - if (!delimiter.empty()) - s->formatter->dump_string("Delimiter", delimiter); - - s->formatter->dump_string("IsTruncated", (max && is_truncated ? "true" - : "false")); + RGWListBucket_ObjStore_S3::send_common_response(); + bool encode_key = false; + if (strcasecmp(encoding_type.c_str(), "url") == 0) { + s->formatter->dump_string("EncodingType", "url"); + encode_key = true; + } + if (op_ret >= 0) { + vector::iterator iter; + for (iter = objs.begin(); iter != objs.end(); ++iter) { + rgw_obj_key key(iter->key); + s->formatter->open_array_section("Contents"); + if (encode_key) { + string key_name; + url_encode(key.name, key_name); + s->formatter->dump_string("Key", key_name); + } else { + s->formatter->dump_string("Key", key.name); + } + dump_time(s, "LastModified", &iter->meta.mtime); + s->formatter->dump_format("ETag", "\"%s\"", iter->meta.etag.c_str()); + s->formatter->dump_int("Size", iter->meta.accounted_size); + auto& storage_class = rgw_placement_rule::get_canonical_storage_class(iter->meta.storage_class); + s->formatter->dump_string("StorageClass", storage_class.c_str()); + dump_owner(s, iter->meta.owner, iter->meta.owner_display_name); + if (s->system_request) { + s->formatter->dump_string("RgwxTag", iter->tag); + } + if (iter->meta.appendable) { + s->formatter->dump_string("Type", "Appendable"); + } else { + s->formatter->dump_string("Type", "Normal"); + } + s->formatter->close_section(); + } + } + s->formatter->dump_string("Marker", marker.name); + if (is_truncated && !next_marker.empty()) { + s->formatter->dump_string("NextMarker", next_marker.name); + } + s->formatter->close_section(); + rgw_flush_formatter_and_reset(s, s->formatter); +} + +void RGWListBucket_ObjStore_S3v2::send_versioned_response() +{ + s->formatter->open_object_section_in_ns("ListVersionsResult", XMLNS_AWS_S3); + RGWListBucket_ObjStore_S3v2::send_common_versioned_response(); + s->formatter->dump_string("KeyContinuationToken", marker.name); + s->formatter->dump_string("VersionIdContinuationToken", marker.instance); + if (is_truncated && !next_marker.empty()) { + s->formatter->dump_string("NextKeyContinuationToken", next_marker.name); + s->formatter->dump_string("NextVersionIdContinuationToken", next_marker.instance); + } bool encode_key = false; if (strcasecmp(encoding_type.c_str(), "url") == 0) { @@ -931,23 +1045,125 @@ void RGWListBucket_ObjStore_S3::send_response() } if (op_ret >= 0) { + if (objs_container) { + s->formatter->open_array_section("Entries"); + } + vector::iterator iter; for (iter = objs.begin(); iter != objs.end(); ++iter) { + const char *section_name = (iter->is_delete_marker() ? "DeleteContinuationToken" + : "Version"); + s->formatter->open_object_section(section_name); + if (objs_container) { + s->formatter->dump_bool("IsDeleteContinuationToken", iter->is_delete_marker()); + } rgw_obj_key key(iter->key); - s->formatter->open_array_section("Contents"); if (encode_key) { - string key_name; - url_encode(key.name, key_name); - s->formatter->dump_string("Key", key_name); - } else { - s->formatter->dump_string("Key", key.name); + string key_name; + url_encode(key.name, key_name); + s->formatter->dump_string("Key", key_name); + } + else { + s->formatter->dump_string("Key", key.name); + } + string version_id = key.instance; + if (version_id.empty()) { + version_id = "null"; + } + if (s->system_request) { + if (iter->versioned_epoch > 0) { + s->formatter->dump_int("VersionedEpoch", iter->versioned_epoch); + } + s->formatter->dump_string("RgwxTag", iter->tag); + utime_t ut(iter->meta.mtime); + ut.gmtime_nsec(s->formatter->dump_stream("RgwxMtime")); + } + s->formatter->dump_string("VersionId", version_id); + s->formatter->dump_bool("IsLatest", iter->is_current()); + dump_time(s, "LastModified", &iter->meta.mtime); + if (!iter->is_delete_marker()) { + s->formatter->dump_format("ETag", "\"%s\"", iter->meta.etag.c_str()); + s->formatter->dump_int("Size", iter->meta.accounted_size); + auto& storage_class = rgw_placement_rule::get_canonical_storage_class(iter->meta.storage_class); + s->formatter->dump_string("StorageClass", storage_class.c_str()); + } + if(fetchOwner == true) { + dump_owner(s, s->user->user_id, s->user->display_name); + } + s->formatter->close_section(); + } + + + if (objs_container) { + s->formatter->close_section(); + } + + if (!common_prefixes.empty()) { + map::iterator pref_iter; + for (pref_iter = common_prefixes.begin(); + pref_iter != common_prefixes.end(); ++pref_iter) { + s->formatter->open_array_section("CommonPrefixes"); + s->formatter->dump_string("Prefix", pref_iter->first); + s->formatter->dump_int("KeyCount",objs.size()); + s->formatter->dump_string("StartAfter", startAfter); + s->formatter->close_section(); + } + } + + s->formatter->close_section(); + rgw_flush_formatter_and_reset(s, s->formatter); +} +} + +void RGWListBucket_ObjStore_S3v2::send_response() +{ + + if(op_ret < 0) { + set_req_state_err(s, op_ret); + } + dump_errno(s); + + // Explicitly use chunked transfer encoding so that we can stream the result + // to the user without having to wait for the full length of it. + end_header(s, this, "application/xml", CHUNKED_TRANSFER_ENCODING); + dump_start(s); + if (op_ret < 0) { + return; + } + if (list_versions) { + send_versioned_response(); + return; + } + + s->formatter->open_object_section_in_ns("ListBucketResult", XMLNS_AWS_S3); + + RGWListBucket_ObjStore_S3::send_common_response(); + bool encode_key = false; + if (strcasecmp(encoding_type.c_str(), "url") == 0) { + s->formatter->dump_string("EncodingType", "url"); + encode_key = true; + } + if (op_ret >= 0) { + vector::iterator iter; + for (iter = objs.begin(); iter != objs.end(); ++iter) { + rgw_obj_key key(iter->key); + s->formatter->open_array_section("Contents"); + if (encode_key) { + string key_name; + url_encode(key.name, key_name); + s->formatter->dump_string("Key", key_name); + } + else { + s->formatter->dump_string("Key", key.name); } dump_time(s, "LastModified", &iter->meta.mtime); s->formatter->dump_format("ETag", "\"%s\"", iter->meta.etag.c_str()); s->formatter->dump_int("Size", iter->meta.accounted_size); auto& storage_class = rgw_placement_rule::get_canonical_storage_class(iter->meta.storage_class); s->formatter->dump_string("StorageClass", storage_class.c_str()); - dump_owner(s, iter->meta.owner, iter->meta.owner_display_name); + if(fetchOwner == true) { + dump_owner(s, s->user->user_id, s->user->display_name); + } if (s->system_request) { s->formatter->dump_string("RgwxTag", iter->tag); } @@ -958,20 +1174,20 @@ void RGWListBucket_ObjStore_S3::send_response() } s->formatter->close_section(); } - if (!common_prefixes.empty()) { - map::iterator pref_iter; - for (pref_iter = common_prefixes.begin(); - pref_iter != common_prefixes.end(); ++pref_iter) { - s->formatter->open_array_section("CommonPrefixes"); - s->formatter->dump_string("Prefix", pref_iter->first); - s->formatter->close_section(); - } - } } + s->formatter->dump_string("ContinuationToken", marker.name); + ldpp_dout(this, 0) << "ContinuationToken: " <formatter->dump_string("NextContinuationToken", next_marker.name); + } + s->formatter->dump_int("KeyCount",objs.size()); + s->formatter->dump_string("StartAfter", startAfter); s->formatter->close_section(); rgw_flush_formatter_and_reset(s, s->formatter); } + + void RGWGetBucketLogging_ObjStore_S3::send_response() { dump_errno(s); @@ -3204,12 +3420,18 @@ RGWOp *RGWHandler_REST_Service_S3::op_post() RGWOp *RGWHandler_REST_Bucket_S3::get_obj_op(bool get_data) { // Non-website mode - if (get_data) { - return new RGWListBucket_ObjStore_S3; - } else { - return new RGWStatBucket_ObjStore_S3; - } -} + int list_type = 1; + s->info.args.get_int("list-type", &list_type, 1); + + // Non-website mode // Non-website mode + if (get_data) { + if (list_type == 1) { + return new RGWListBucket_ObjStore_S3; + } else if(list_type == 2) { + return new RGWListBucket_ObjStore_S3v2; + } } else { + return new RGWStatBucket_ObjStore_S3; + } } RGWOp *RGWHandler_REST_Bucket_S3::op_get() { diff --git a/src/rgw/rgw_rest_s3.h b/src/rgw/rgw_rest_s3.h index 023a9f0d68d..a61d5492b58 100644 --- a/src/rgw/rgw_rest_s3.h +++ b/src/rgw/rgw_rest_s3.h @@ -126,8 +126,11 @@ public: }; class RGWListBucket_ObjStore_S3 : public RGWListBucket_ObjStore { - bool objs_container; -public: + protected: bool objs_container; + int get_common_params(); + void send_common_response(); + void send_common_versioned_response(); + public: RGWListBucket_ObjStore_S3() : objs_container(false) { default_max = 1000; } @@ -138,6 +141,18 @@ public: void send_versioned_response(); }; +class RGWListBucket_ObjStore_S3v2 : public RGWListBucket_ObjStore_S3 { + bool fetchOwner; +public: + RGWListBucket_ObjStore_S3v2() : fetchOwner(false) { + } + ~RGWListBucket_ObjStore_S3v2() override {} + + int get_params() override; + void send_response() override; + void send_versioned_response(); +}; + class RGWGetBucketLogging_ObjStore_S3 : public RGWGetBucketLogging { public: RGWGetBucketLogging_ObjStore_S3() {} -- cgit v1.2.3