Merge pull request #51715 from awojno-bloomberg/replication-race-condition-fix

RGW: Pass and check destination zone trace when fetching object

Reviewed-by: Casey Bodley <cbodley@redhat.com>
Reviewed-by: Mark Kogan <mkogan@redhat.com>

7 files changed, 34 insertions, 5 deletions

diff --git a/src/cls/rgw/cls_rgw_types.h b/src/cls/rgw/cls_rgw_types.h
index 37a06defc31..f94bf114fa1 100644
--- a/src/cls/rgw/cls_rgw_types.h
+++ b/src/cls/rgw/cls_rgw_types.h
@@ -38,6 +38,10 @@ struct rgw_zone_set_entry {
     return (location_key < e.location_key);
   }
 
+  bool operator==(const rgw_zone_set_entry& e) const {
+    return zone == e.zone && location_key == e.location_key;
+  }
+
   rgw_zone_set_entry() {}
   rgw_zone_set_entry(const std::string& _zone,
                      std::optional<std::string> _location_key) : zone(_zone),
diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc
index 72a94c019ec..2261a31f20e 100644
--- a/src/rgw/driver/rados/rgw_rados.cc
+++ b/src/rgw/driver/rados/rgw_rados.cc
@@ -3792,7 +3792,7 @@ int RGWRados::stat_remote_obj(const DoutPrefixProvider *dpp,
   int ret = conn->get_obj(dpp, user_id, info, src_obj, pmod, unmod_ptr,
                       dest_mtime_weight.zone_short_id, dest_mtime_weight.pg_ver,
                       prepend_meta, get_op, rgwx_stat,
-                      sync_manifest, skip_decrypt, sync_cloudtiered,
+                      sync_manifest, skip_decrypt, nullptr, sync_cloudtiered,
                       true, &cb, &in_stream_req);
   if (ret < 0) {
     return ret;
@@ -3989,6 +3989,7 @@ int RGWRados::fetch_remote_obj(RGWObjectCtx& obj_ctx,
   const real_time *pmod = mod_ptr;
 
   obj_time_weight dest_mtime_weight;
+  rgw_zone_set_entry dst_zone_trace(svc.zone->get_zone().id, dest_bucket_info.bucket.get_key());
 
   if (copy_if_newer) {
     /* need to get mtime for destination */
@@ -4011,8 +4012,8 @@ int RGWRados::fetch_remote_obj(RGWObjectCtx& obj_ctx,
   ret = conn->get_obj(dpp, user_id, info, src_obj, pmod, unmod_ptr,
                       dest_mtime_weight.zone_short_id, dest_mtime_weight.pg_ver,
                       prepend_meta, get_op, rgwx_stat,
-                      sync_manifest, skip_decrypt, sync_cloudtiered,
-                      true,
+                      sync_manifest, skip_decrypt, &dst_zone_trace,
+                      sync_cloudtiered, true,
                       &cb, &in_stream_req);
   if (ret < 0) {
     goto set_err_state;
diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index d605458b44f..98b11108b7f 100644
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -2308,6 +2308,23 @@ void RGWGetObj::execute(optional_yield y)
       filter = &*decompress;
   }
 
+  attr_iter = attrs.find(RGW_ATTR_OBJ_REPLICATION_TRACE);
+  if (attr_iter != attrs.end()) {
+    try {
+      std::vector<rgw_zone_set_entry> zones;
+      auto p = attr_iter->second.cbegin();
+      decode(zones, p);
+      for (const auto& zone: zones) {
+        if (zone == dst_zone_trace) {
+          op_ret = -ERR_NOT_MODIFIED;
+          ldpp_dout(this, 4) << "Object already has been copied to this destination. Returning "
+            << op_ret << dendl;
+          goto done_err;
+        }
+      }
+    } catch (const buffer::error&) {}
+  }
+
   if (get_type() == RGW_OP_GET_OBJ && get_data) {
     op_ret = handle_cloudtier_obj(attrs, sync_cloudtiered);
     if (op_ret < 0) {
diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h
index a2d39c83a7f..63d5f673ac1 100644
--- a/src/rgw/rgw_op.h
+++ b/src/rgw/rgw_op.h
@@ -359,6 +359,7 @@ protected:
   std::string lo_etag;
   bool rgwx_stat; /* extended rgw stat operation */
   std::string version_id;
+  rgw_zone_set_entry dst_zone_trace;
 
   // compression attrs
   RGWCompressionInfo cs_info;
diff --git a/src/rgw/rgw_rest_conn.cc b/src/rgw/rgw_rest_conn.cc
index 21ddf6b718f..ffb536ed977 100644
--- a/src/rgw/rgw_rest_conn.cc
+++ b/src/rgw/rgw_rest_conn.cc
@@ -203,7 +203,7 @@ int RGWRESTConn::get_obj(const DoutPrefixProvider *dpp, const rgw_user& uid, req
                          uint32_t mod_zone_id, uint64_t mod_pg_ver,
                          bool prepend_metadata, bool get_op, bool rgwx_stat,
                          bool sync_manifest, bool skip_decrypt,
-                         bool sync_cloudtiered,
+                         rgw_zone_set_entry *dst_zone_trace, bool sync_cloudtiered,
                          bool send, RGWHTTPStreamRWRequest::ReceiveCB *cb, RGWRESTStreamRWRequest **req)
 {
   get_obj_params params;
@@ -217,6 +217,7 @@ int RGWRESTConn::get_obj(const DoutPrefixProvider *dpp, const rgw_user& uid, req
   params.sync_manifest = sync_manifest;
   params.skip_decrypt = skip_decrypt;
   params.sync_cloudtiered = sync_cloudtiered;
+  params.dst_zone_trace = dst_zone_trace;
   params.cb = cb;
   return get_obj(dpp, obj, params, send, req);
 }
@@ -245,6 +246,9 @@ int RGWRESTConn::get_obj(const DoutPrefixProvider *dpp, const rgw_obj& obj, cons
   if (in_params.skip_decrypt) {
     params.push_back(param_pair_t(RGW_SYS_PARAM_PREFIX "skip-decrypt", ""));
   }
+  if (in_params.dst_zone_trace) {
+    params.push_back(param_pair_t(RGW_SYS_PARAM_PREFIX "if-not-replicated-to", in_params.dst_zone_trace->to_str()));
+  }
   if (!obj.key.instance.empty()) {
     params.push_back(param_pair_t("versionId", obj.key.instance));
   }
diff --git a/src/rgw/rgw_rest_conn.h b/src/rgw/rgw_rest_conn.h
index bb0d5a8bbf8..81f839f49a2 100644
--- a/src/rgw/rgw_rest_conn.h
+++ b/src/rgw/rgw_rest_conn.h
@@ -162,6 +162,7 @@ public:
     bool range_is_set{false};
     uint64_t range_start{0};
     uint64_t range_end{0};
+    rgw_zone_set_entry *dst_zone_trace{nullptr};
   };
 
   int get_obj(const DoutPrefixProvider *dpp, const rgw_obj& obj, const get_obj_params& params, bool send, RGWRESTStreamRWRequest **req);
@@ -170,7 +171,7 @@ public:
               const ceph::real_time *mod_ptr, const ceph::real_time *unmod_ptr,
               uint32_t mod_zone_id, uint64_t mod_pg_ver,
               bool prepend_metadata, bool get_op, bool rgwx_stat, bool sync_manifest,
-              bool skip_decrypt, bool sync_cloudtiered,
+              bool skip_decrypt, rgw_zone_set_entry *dst_zone_trace, bool sync_cloudtiered,
               bool send, RGWHTTPStreamRWRequest::ReceiveCB *cb, RGWRESTStreamRWRequest **req);
   int complete_request(RGWRESTStreamRWRequest *req,
                        std::string *etag,
diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc
index cc65f815769..1344f247d28 100644
--- a/src/rgw/rgw_rest_s3.cc
+++ b/src/rgw/rgw_rest_s3.cc
@@ -302,6 +302,7 @@ int RGWGetObj_ObjStore_S3::get_params(optional_yield y)
   // multisite sync requests should fetch cloudtiered objects
   sync_cloudtiered = s->info.args.exists(RGW_SYS_PARAM_PREFIX "sync-cloudtiered");
 
+  dst_zone_trace = s->info.args.get(RGW_SYS_PARAM_PREFIX "if-not-replicated-to");
   get_torrent = s->info.args.exists("torrent");
 
   return RGWGetObj_ObjStore::get_params(y);