rgw_file: introduce fast S3 Unix stats (immutable)

For objects originating in S3/Swift, it is frequently acceptable
to synthesize c/mtime and size stats from meta attributes in the RGW
bucket index entry.

Synthetic stat behavior is not compatible with mutation of file
ownership and permissions via SETATTR.  In future, current xattr-based,
mutable, Unix stats may be moved to the bucket index.  For now, use
synthetic stats only when a new option rgw_nfs_s3_fast_attrs is present.

Fixes: http://tracker.ceph.com/issues/40456

Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>

11 files changed, 163 insertions, 49 deletions

diff --git a/src/common/legacy_config_opts.h b/src/common/legacy_config_opts.h
index 6f18e45c6b2..5156abd00c2 100644
--- a/src/common/legacy_config_opts.h
+++ b/src/common/legacy_config_opts.h
@@ -1387,6 +1387,10 @@ OPTION(rgw_nfs_max_gc, OPT_INT) /* max gc events per cycle */
 OPTION(rgw_nfs_write_completion_interval_s, OPT_INT) /* stateless (V3)
 							  * commit
 							  * delay */
+OPTION(rgw_nfs_s3_fast_attrs, OPT_BOOL) /* use fast S3 attrs from
+					 * bucket index--currently
+					 * assumes NFS mounts are
+					 * immutable */
 
 OPTION(rgw_zone, OPT_STR) // zone name
 OPTION(rgw_zone_root_pool, OPT_STR)    // pool where zone specific info is stored
diff --git a/src/common/options.cc b/src/common/options.cc
index d967c934bf6..82c71b3a1f9 100644
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -6063,6 +6063,12 @@ std::vector<Option> get_rgw_options() {
     .set_default(10)
     .set_description(""),
 
+    Option("rgw_nfs_s3_fast_attrs", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
+    .set_default(false)
+    .set_description("use fast S3 attrs from bucket index (immutable only)")
+    .set_long_description("use fast S3 attrs from bucket index (assumes NFS "
+			  "mounts are immutable)"),
+
     Option("rgw_rados_pool_autoscale_bias", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
     .set_default(4.0)
     .set_min_max(0.01, 100000.0)
diff --git a/src/include/rados/rgw_file.h b/src/include/rados/rgw_file.h
index c1c31955093..66cf627aafa 100644
--- a/src/include/rados/rgw_file.h
+++ b/src/include/rados/rgw_file.h
@@ -102,7 +102,8 @@ void rgwfile_version(int *major, int *minor, int *extra);
 
 int rgw_lookup(struct rgw_fs *rgw_fs,
 	      struct rgw_file_handle *parent_fh, const char *path,
-	      struct rgw_file_handle **fh, uint32_t flags);
+	      struct rgw_file_handle **fh,
+	      struct stat *st, uint32_t mask, uint32_t flags);
 
 /*
   lookup object by handle (NFS style)
@@ -221,6 +222,7 @@ int rgw_unlink(struct rgw_fs *rgw_fs,
     read  directory content
 */
 typedef bool (*rgw_readdir_cb)(const char *name, void *arg, uint64_t offset,
+			       struct stat *st, uint32_t mask,
 			       uint32_t flags);
 
 #define RGW_READDIR_FLAG_NONE      0x0000
diff --git a/src/pybind/rgw/rgw.pyx b/src/pybind/rgw/rgw.pyx
index d6eaa75e3d1..d5db7f5d93c 100644
--- a/src/pybind/rgw/rgw.pyx
+++ b/src/pybind/rgw/rgw.pyx
@@ -107,7 +107,8 @@ cdef extern from "rados/rgw_file.h" nogil:
 
     int rgw_lookup(rgw_fs *fs,
                    rgw_file_handle *parent_fh, const char *path,
-                   rgw_file_handle **fh, uint32_t flags)
+                   rgw_file_handle **fh, stat* st, uint32_t st_mask,
+		   uint32_t flags)
 
     int rgw_lookup_handle(rgw_fs *fs, rgw_fh_hk *fh_hk,
                           rgw_file_handle **fh, uint32_t flags)
@@ -144,7 +145,7 @@ cdef extern from "rados/rgw_file.h" nogil:
 
     int rgw_readdir(rgw_fs *fs,
                     rgw_file_handle *parent_fh, uint64_t *offset,
-                    bool (*cb)(const char *name, void *arg, uint64_t offset, uint32_t flags) nogil except? -9000,
+                    bool (*cb)(const char *name, void *arg, uint64_t offset, stat *st, uint32_t st_mask, uint32_t flags) nogil except? -9000,
                     void *cb_arg, bool *eof, uint32_t flags) except? -9000
 
     int rgw_getattr(rgw_fs *fs,
@@ -314,7 +315,7 @@ cdef make_ex(ret, msg):
         return Error(msg + (": error code %d" % ret))
 
 
-cdef bool readdir_cb(const char *name, void *arg, uint64_t offset, uint32_t flags) \
+cdef bool readdir_cb(const char *name, void *arg, uint64_t offset, stat *st, uint32_t st_mask, uint32_t flags) \
 except? -9000 with gil:
     if exc.PyErr_Occurred():
         return False
@@ -566,10 +567,12 @@ cdef class LibRGWFS(object):
             rgw_file_handle *_file_handler
             int _flags = flags
             char* _dirname = dirname
+            stat st
+            uint32_t st_mask = 0
 
         with nogil:
             ret = rgw_lookup(self.fs, _dir_handler, _dirname,
-                             &_file_handler, _flags)
+                             &_file_handler, &st, st_mask, _flags)
         if ret < 0:
             raise make_ex(ret, "error in open '%s'" % dirname)
 
@@ -590,10 +593,12 @@ cdef class LibRGWFS(object):
             rgw_file_handle *_file_handler
             int _flags = flags
             char* _filename = filename
+            stat st
+            uint32_t st_mask = 0
 
         with nogil:
             ret = rgw_lookup(self.fs, _dir_handler, _filename,
-                             &_file_handler, _flags)
+                             &_file_handler, &st, st_mask, _flags)
         if ret < 0:
             raise make_ex(ret, "error in open '%s'" % filename)
         with nogil:
diff --git a/src/rgw/rgw_file.cc b/src/rgw/rgw_file.cc
index 4c4926db6ba..3bc1b659af2 100644
--- a/src/rgw/rgw_file.cc
+++ b/src/rgw/rgw_file.cc
@@ -107,6 +107,42 @@ namespace rgw {
     return fhr;
   }
 
+  LookupFHResult RGWLibFS::fake_leaf(RGWFileHandle* parent,
+				     const char *path,
+				     enum rgw_fh_type type,
+				     struct stat *st, uint32_t st_mask,
+				     uint32_t flags)
+  {
+    /* synthesize a minimal handle from parent, path, type, and st */
+    using std::get;
+
+    flags |= RGWFileHandle::FLAG_CREATE;
+
+    switch (type) {
+    case RGW_FS_TYPE_DIRECTORY:
+      flags |= RGWFileHandle::FLAG_DIRECTORY;
+      break;
+    default:
+      /* file */
+      break;
+    };
+
+    LookupFHResult fhr = lookup_fh(parent, path, flags);
+    if (get<0>(fhr)) {
+      RGWFileHandle* rgw_fh = get<0>(fhr);
+      if (st) {	
+	lock_guard guard(rgw_fh->mtx);
+	if (st_mask & RGW_SETATTR_SIZE) {
+	  rgw_fh->set_size(st->st_size);
+	}
+	if (st_mask & RGW_SETATTR_MTIME) {
+	  rgw_fh->set_times(st->st_mtim);
+	}
+      } /* st */
+    } /* rgw_fh */
+    return fhr;
+  } /* RGWLibFS::fake_leaf */
+
   LookupFHResult RGWLibFS::stat_leaf(RGWFileHandle* parent,
 				     const char *path,
 				     enum rgw_fh_type type,
@@ -355,6 +391,7 @@ namespace rgw {
 	 * atomicity at this endpoint */
 	struct rgw_file_handle *fh;
 	rc = rgw_lookup(get_fs(), parent->get_fh(), name, &fh,
+			nullptr /* st */, 0 /* mask */,
 			RGW_LOOKUP_FLAG_NONE);
 	if (!! rc)
 	  return rc;
@@ -528,6 +565,7 @@ namespace rgw {
     rgw_file_handle *lfh;
 
     rc = rgw_lookup(get_fs(), parent->get_fh(), name, &lfh,
+		    nullptr /* st */, 0 /* mask */,
 		    RGW_LOOKUP_FLAG_NONE);
     if (! rc) {
       /* conflict! */
@@ -642,6 +680,7 @@ namespace rgw {
 
     rgw_file_handle *lfh;
     rc = rgw_lookup(get_fs(), parent->get_fh(), name, &lfh,
+		    nullptr /* st */, 0 /* mask */,
 		    RGW_LOOKUP_FLAG_NONE);
     if (! rc) {
       /* conflict! */
@@ -709,7 +748,8 @@ namespace rgw {
     using std::get;
 
     rgw_file_handle *lfh;
-    rc = rgw_lookup(get_fs(), parent->get_fh(), name, &lfh, 
+    rc = rgw_lookup(get_fs(), parent->get_fh(), name, &lfh,
+		    nullptr /* st */, 0 /* mask */,
                     RGW_LOOKUP_FLAG_NONE);
     if (! rc) {
       /* conflict! */
@@ -1912,7 +1952,8 @@ int rgw_unlink(struct rgw_fs *rgw_fs, struct rgw_file_handle *parent_fh,
 */
 int rgw_lookup(struct rgw_fs *rgw_fs,
 	      struct rgw_file_handle *parent_fh, const char* path,
-	      struct rgw_file_handle **fh, uint32_t flags)
+	      struct rgw_file_handle **fh,
+	      struct stat *st, uint32_t mask, uint32_t flags)
 {
   //CephContext* cct = static_cast<CephContext*>(rgw_fs->rgw);
   RGWLibFS *fs = static_cast<RGWLibFS*>(rgw_fs->fs_private);
@@ -1948,14 +1989,29 @@ int rgw_lookup(struct rgw_fs *rgw_fs,
 	<< dendl;
       fs->ref(rgw_fh);
     } else {
-      /* lookup in a readdir callback */
       enum rgw_fh_type fh_type = fh_type_of(flags);
 
       uint32_t sl_flags = (flags & RGW_LOOKUP_FLAG_RCB)
 	? RGWFileHandle::FLAG_NONE
 	: RGWFileHandle::FLAG_EXACT_MATCH;
 
-      fhr = fs->stat_leaf(parent, path, fh_type, sl_flags);
+      bool fast_attrs= fs->get_context()->_conf->rgw_nfs_s3_fast_attrs;
+
+      if ((flags & RGW_LOOKUP_FLAG_RCB) && fast_attrs) {
+	/* FAKE STAT--this should mean, interpolate special
+	 * owner, group, and perms masks */
+	fhr = fs->fake_leaf(parent, path, fh_type, st, mask, sl_flags);
+      } else {
+	if ((fh_type == RGW_FS_TYPE_DIRECTORY) && fast_attrs) {
+	  /* trust cached dir, if present */
+	  fhr = fs->lookup_fh(parent, path, RGWFileHandle::FLAG_DIRECTORY);
+	  if (get<0>(fhr)) {
+	    rgw_fh = get<0>(fhr);
+	    goto done;
+	  }
+	}
+	fhr = fs->stat_leaf(parent, path, fh_type, sl_flags);
+      }
       if (! get<0>(fhr)) {
 	if (! (flags & RGW_LOOKUP_FLAG_CREATE))
 	  return -ENOENT;
@@ -1966,6 +2022,7 @@ int rgw_lookup(struct rgw_fs *rgw_fs,
     }
   } /* !root */
 
+done:
   struct rgw_file_handle *rfh = rgw_fh->get_fh();
   *fh = rfh;
 
@@ -2100,8 +2157,8 @@ int rgw_readdir(struct rgw_fs *rgw_fs,
   if ((*offset == 0) &&
       (flags & RGW_READDIR_FLAG_DOTDOT)) {
     /* send '.' and '..' with their NFS-defined offsets */
-    rcb(".", cb_arg, 1, RGW_LOOKUP_FLAG_DIR);
-    rcb("..", cb_arg, 2, RGW_LOOKUP_FLAG_DIR);
+    rcb(".", cb_arg, 1, nullptr, 0, RGW_LOOKUP_FLAG_DIR);
+    rcb("..", cb_arg, 2, nullptr, 0, RGW_LOOKUP_FLAG_DIR);
   }
 
   int rc = parent->readdir(rcb, cb_arg, offset, eof, flags);
@@ -2128,8 +2185,8 @@ int rgw_readdir2(struct rgw_fs *rgw_fs,
   if ((! name) &&
       (flags & RGW_READDIR_FLAG_DOTDOT)) {
     /* send '.' and '..' with their NFS-defined offsets */
-    rcb(".", cb_arg, 1, RGW_LOOKUP_FLAG_DIR);
-    rcb("..", cb_arg, 2, RGW_LOOKUP_FLAG_DIR);
+    rcb(".", cb_arg, 1, nullptr, 0, RGW_LOOKUP_FLAG_DIR);
+    rcb("..", cb_arg, 2, nullptr, 0, RGW_LOOKUP_FLAG_DIR);
   }
 
   int rc = parent->readdir(rcb, cb_arg, name, eof, flags);
diff --git a/src/rgw/rgw_file.h b/src/rgw/rgw_file.h
index 9af81ed322f..1b849ba5674 100644
--- a/src/rgw/rgw_file.h
+++ b/src/rgw/rgw_file.h
@@ -280,6 +280,9 @@ namespace rgw {
 #define CREATE_FLAGS(x) \
     ((x) & ~(RGWFileHandle::FLAG_CREATE|RGWFileHandle::FLAG_LOCK))
 
+    static constexpr uint32_t RCB_MASK = \
+      RGW_SETATTR_MTIME|RGW_SETATTR_CTIME|RGW_SETATTR_ATIME|RGW_SETATTR_SIZE;
+
     friend class RGWLibFS;
 
   private:
@@ -644,12 +647,16 @@ namespace rgw {
       state.size = size;
     }
 
-    void set_times(real_time t) {
-      state.ctime = real_clock::to_timespec(t);
+    void set_times(const struct timespec &ts) {
+      state.ctime = ts;
       state.mtime = state.ctime;
       state.atime = state.ctime;
     }
 
+    void set_times(real_time t) {
+      set_times(real_clock::to_timespec(t));
+    }
+
     void set_ctime(const struct timespec &ts) {
       state.ctime = ts;
     }
@@ -1181,6 +1188,11 @@ namespace rgw {
 			       RGWLibFS::BucketStats& bs,
 			       uint32_t flags);
 
+    LookupFHResult fake_leaf(RGWFileHandle* parent, const char *path,
+			     enum rgw_fh_type type = RGW_FS_TYPE_NIL,
+			     struct stat *st = nullptr, uint32_t mask = 0,
+			     uint32_t flags = RGWFileHandle::FLAG_NONE);
+
     LookupFHResult stat_leaf(RGWFileHandle* parent, const char *path,
 			     enum rgw_fh_type type = RGW_FS_TYPE_NIL,
 			     uint32_t flags = RGWFileHandle::FLAG_NONE);
@@ -1407,7 +1419,7 @@ public:
     rgw_fh->add_marker(off, rgw_obj_key{marker.data(), ""},
 		       RGW_FS_TYPE_DIRECTORY);
     ++d_count;
-    return rcb(name.data(), cb_arg, off, RGW_LOOKUP_FLAG_DIR);
+    return rcb(name.data(), cb_arg, off, nullptr, 0, RGW_LOOKUP_FLAG_DIR);
   }
 
   bool eof() {
@@ -1510,7 +1522,7 @@ public:
   }
 
   int operator()(const boost::string_ref name, const rgw_obj_key& marker,
-		uint8_t type) {
+		 const ceph::real_time& t, const uint64_t fsz, uint8_t type) {
 
     assert(name.length() > 0); // all cases handled in callers
 
@@ -1519,10 +1531,25 @@ public:
     if (unlikely(!! ioff)) {
       *ioff = off;
     }
+
     /* update traversal cache */
     rgw_fh->add_marker(off, marker, type);
     ++d_count;
-    return rcb(name.data(), cb_arg, off,
+
+    /* set c/mtime and size from bucket index entry */
+    struct stat st = {};
+#ifdef HAVE_STAT_ST_MTIMESPEC_TV_NSEC
+    st.st_atimespec = ceph::real_clock::to_timespec(t);
+    st.st_mtimespec = st.st_atimespec;
+    st.st_ctimespec = st.st_atimespec;
+#else
+    st.st_atim = ceph::real_clock::to_timespec(t);
+    st.st_mtim = st.st_atim;
+    st.st_ctim = st.st_atim;
+#endif
+    st.st_size = fsz;
+
+    return rcb(name.data(), cb_arg, off, &st, RGWFileHandle::RCB_MASK,
 	       (type == RGW_FS_TYPE_DIRECTORY) ?
 	       RGW_LOOKUP_FLAG_DIR :
 	       RGW_LOOKUP_FLAG_FILE);
@@ -1556,9 +1583,13 @@ public:
 			     << " prefix=" << prefix << " "
 			     << " obj path=" << iter.key.name
 			     << " (" << sref << ")" << ""
+			     << " mtime="
+			     << real_clock::to_time_t(iter.meta.mtime)
+			     << " size=" << iter.meta.accounted_size
 			     << dendl;
 
-      if (! this->operator()(sref, next_marker, RGW_FS_TYPE_FILE)) {
+      if (! this->operator()(sref, next_marker, iter.meta.mtime,
+			     iter.meta.accounted_size, RGW_FS_TYPE_FILE)) {
 	/* caller cannot accept more */
 	lsubdout(cct, rgw, 5) << "readdir rcb failed"
 			      << " dirent=" << sref.data()
@@ -1569,6 +1600,8 @@ public:
       }
       ++ix;
     }
+
+    auto cnow = real_clock::now();
     for (auto& iter : common_prefixes) {
 
       lsubdout(cct, rgw, 15) << "readdir common prefixes prefix: " << prefix
@@ -1605,7 +1638,8 @@ public:
 	return;
       }
 
-      if (! this->operator()(sref, next_marker, RGW_FS_TYPE_DIRECTORY)) {
+      if (! this->operator()(sref, next_marker, cnow, 0,
+			     RGW_FS_TYPE_DIRECTORY)) {
 	/* caller cannot accept more */
 	lsubdout(cct, rgw, 5) << "readdir rcb failed"
 			      << " dirent=" << sref.data()
diff --git a/src/test/librgw_file.cc b/src/test/librgw_file.cc
index 3689a052903..fea8a061191 100644
--- a/src/test/librgw_file.cc
+++ b/src/test/librgw_file.cc
@@ -72,6 +72,7 @@ TEST(LibRGW, GETATTR_ROOT) {
 
 extern "C" {
   static bool r1_cb(const char* name, void *arg, uint64_t offset,
+		    struct stat* st, uint32_t st_mask,
 		    uint32_t flags) {
     // don't need arg--it would point to fids1
     fids1.push_back(fid_type(name, offset, nullptr /* handle */));
@@ -110,7 +111,7 @@ TEST(LibRGW, LOOKUP_BUCKETS) {
     // auto& obj_vector = get<1>(fid_row);
     struct rgw_file_handle *rgw_fh = nullptr;
     ASSERT_EQ(0, rgw_lookup(fs, fs->root_fh, get<0>(fid).c_str(), &rgw_fh,
-			    0 /* flags */));
+			    nullptr /* stat */, 0 /* mask */, 0 /* flags */));
     get<2>(fid) = rgw_fh;
     ASSERT_NE(get<2>(fid), nullptr);
   }
@@ -135,6 +136,7 @@ TEST(LibRGW, GETATTR_BUCKETS) {
 
 extern "C" {
   static bool r2_cb(const char* name, void *arg, uint64_t offset,
+		    struct stat* st, uint32_t st_mask,
 		    uint32_t flags) {
     std::vector<fid_type>& obj_vector = *(static_cast<std::vector<fid_type>*>(arg));
     obj_vector.push_back(fid_type(name, offset, nullptr));
@@ -188,7 +190,7 @@ TEST(LibRGW, GETATTR_OBJECTS) {
 	struct rgw_file_handle *obj_fh = nullptr;
 	std::string object_name = get<0>(obj);
 	ret = rgw_lookup(fs, bucket_fh, get<0>(obj).c_str(), &obj_fh,
-			0 /* flags */);
+			 nullptr /* stat */, 0 /* mask */, 0 /* flags */);
 	ASSERT_EQ(ret, 0);
 	get<2>(obj) = obj_fh; // stash obj_fh for cleanup
 	ASSERT_NE(get<2>(obj), nullptr);
diff --git a/src/test/librgw_file_aw.cc b/src/test/librgw_file_aw.cc
index 859cd14074d..ed51610cecf 100644
--- a/src/test/librgw_file_aw.cc
+++ b/src/test/librgw_file_aw.cc
@@ -199,13 +199,13 @@ TEST(LibRGW, CREATE_BUCKET) {
 
 TEST(LibRGW, LOOKUP_BUCKET) {
   int ret = rgw_lookup(fs, fs->root_fh, bucket_name.c_str(), &bucket_fh,
-		      RGW_LOOKUP_FLAG_NONE);
+		       nullptr, 0, RGW_LOOKUP_FLAG_NONE);
   ASSERT_EQ(ret, 0);
 }
 
 TEST(LibRGW, LOOKUP_OBJECT) {
   int ret = rgw_lookup(fs, bucket_fh, object_name.c_str(), &object_fh,
-		       RGW_LOOKUP_FLAG_CREATE);
+		       nullptr, 0, RGW_LOOKUP_FLAG_CREATE);
   ASSERT_EQ(ret, 0);
 }
 
diff --git a/src/test/librgw_file_gp.cc b/src/test/librgw_file_gp.cc
index 80a165418a4..1ad41cfd30f 100644
--- a/src/test/librgw_file_gp.cc
+++ b/src/test/librgw_file_gp.cc
@@ -187,12 +187,13 @@ TEST(LibRGW, MOUNT) {
 
 TEST(LibRGW, LOOKUP_BUCKET) {
   int ret = rgw_lookup(fs, fs->root_fh, bucket_name.c_str(), &bucket_fh,
-		       RGW_LOOKUP_FLAG_NONE);
+		       nullptr, 0, RGW_LOOKUP_FLAG_NONE);
   ASSERT_EQ(ret, 0);
 }
 
 extern "C" {
   static bool r2_cb(const char* name, void *arg, uint64_t offset,
+		    struct stat *st, uint32_t st_mask,
 		    uint32_t flags) {
     // don't need arg--it would point to fids
     fids.push_back(fid_type(name, offset, nullptr));
@@ -222,7 +223,7 @@ TEST(LibRGW, LIST_OBJECTS) {
 TEST(LibRGW, LOOKUP_OBJECT) {
   if (do_get || do_stat || do_put || do_bulk || do_readv || do_writev) {
     int ret = rgw_lookup(fs, bucket_fh, object_name.c_str(), &object_fh,
-			RGW_LOOKUP_FLAG_CREATE);
+			 nullptr, 0, RGW_LOOKUP_FLAG_CREATE);
     ASSERT_EQ(ret, 0);
   }
 }
diff --git a/src/test/librgw_file_marker.cc b/src/test/librgw_file_marker.cc
index 105ab90ee2a..9aaa6b3256b 100644
--- a/src/test/librgw_file_marker.cc
+++ b/src/test/librgw_file_marker.cc
@@ -216,7 +216,7 @@ TEST(LibRGW, MARKER1_SETUP_BUCKET) {
   st.st_mode = 755;
 
   (void) rgw_lookup(fs, fs->root_fh, bucket_name.c_str(), &bucket_fh,
-		    RGW_LOOKUP_FLAG_NONE);
+		    nullptr, 0, RGW_LOOKUP_FLAG_NONE);
   if (! bucket_fh) {
     if (do_create) {
       struct stat st;
@@ -234,7 +234,7 @@ TEST(LibRGW, MARKER1_SETUP_BUCKET) {
   ASSERT_NE(bucket_fh, nullptr);
 
   (void) rgw_lookup(fs, bucket_fh, marker_dir.c_str(), &marker_fh,
-		    RGW_LOOKUP_FLAG_NONE);
+		    nullptr, 0, RGW_LOOKUP_FLAG_NONE);
   if (! marker_fh) {
     if (do_create) {
       ret = rgw_mkdir(fs, bucket_fh, marker_dir.c_str(), &st, create_mask,
@@ -259,7 +259,7 @@ TEST(LibRGW, MARKER1_SETUP_OBJECTS)
       obj_rec obj{object_name, nullptr, marker_fh, nullptr};
       // lookup object--all operations are by handle
       ret = rgw_lookup(fs, marker_fh, obj.name.c_str(), &obj.fh,
-		       RGW_LOOKUP_FLAG_CREATE);
+		       nullptr, 0, RGW_LOOKUP_FLAG_CREATE);
       ASSERT_EQ(ret, 0);
       obj.rgw_fh = get_rgwfh(obj.fh);
       // open object--open transaction
@@ -285,6 +285,7 @@ TEST(LibRGW, MARKER1_SETUP_OBJECTS)
 
 extern "C" {
   static bool r2_cb(const char* name, void *arg, uint64_t offset,
+		    struct stat* st, uint32_t st_mask,
 		    uint32_t flags) {
     dirent_vec& dvec =
       *(static_cast<dirent_vec*>(arg));
diff --git a/src/test/librgw_file_nfsns.cc b/src/test/librgw_file_nfsns.cc
index 38a76053e14..9e023c28af0 100644
--- a/src/test/librgw_file_nfsns.cc
+++ b/src/test/librgw_file_nfsns.cc
@@ -201,7 +201,7 @@ TEST(LibRGW, SETUP_HIER1)
 {
   if (do_hier1) {
     (void) rgw_lookup(fs, fs->root_fh, bucket_name.c_str(), &bucket_fh,
-		      RGW_LOOKUP_FLAG_NONE);
+		      nullptr, 0, RGW_LOOKUP_FLAG_NONE);
     if (! bucket_fh) {
       if (do_create) {
 	struct stat st;
@@ -265,7 +265,7 @@ TEST(LibRGW, SETUP_DIRS1) {
     dirs1_b.parent_fh = fs->root_fh;
 
     (void) rgw_lookup(fs, dirs1_b.parent_fh, dirs1_bucket_name.c_str(),
-		      &dirs1_b.fh, RGW_LOOKUP_FLAG_NONE);
+		      &dirs1_b.fh, nullptr, 0, RGW_LOOKUP_FLAG_NONE);
 
     if (! dirs1_b.fh) {
       if (do_create) {
@@ -289,7 +289,7 @@ TEST(LibRGW, SETUP_DIRS1) {
       ovec.clear();
 
       (void) rgw_lookup(fs, dir.parent_fh, dir.name.c_str(), &dir.fh,
-			RGW_LOOKUP_FLAG_NONE);
+			nullptr, 0, RGW_LOOKUP_FLAG_NONE);
       if (! dir.fh) {
 	if (do_create) {
 	  rc = rgw_mkdir(fs, dir.parent_fh, dir.name.c_str(), &st, create_mask,
@@ -311,7 +311,7 @@ TEST(LibRGW, SETUP_DIRS1) {
 	obj_rec sdir{sdname, nullptr, dir.fh, nullptr};
 
 	(void) rgw_lookup(fs, sdir.parent_fh, sdir.name.c_str(), &sdir.fh,
-			  RGW_LOOKUP_FLAG_NONE);
+			  nullptr, 0, RGW_LOOKUP_FLAG_NONE);
 
 	if (! sdir.fh) {
 	  if (do_create) {
@@ -334,13 +334,13 @@ TEST(LibRGW, SETUP_DIRS1) {
 	obj_rec sf{sfname, nullptr, dir.fh, nullptr};
 
 	(void) rgw_lookup(fs, sf.parent_fh, sf.name.c_str(), &sf.fh,
-			  RGW_LOOKUP_FLAG_NONE);
+			  nullptr, 0, RGW_LOOKUP_FLAG_NONE);
 
 	if (! sf.fh) {
 	  if (do_create) {
 	    /* make a new file object (the hard way) */
 	    rc = rgw_lookup(fs, sf.parent_fh, sf.name.c_str(), &sf.fh,
-			    RGW_LOOKUP_FLAG_CREATE);
+			    nullptr, 0, RGW_LOOKUP_FLAG_CREATE);
 	    ASSERT_EQ(rc, 0);
 	    sf.sync();
 	    ASSERT_TRUE(sf.rgw_fh->is_file());
@@ -396,7 +396,7 @@ TEST(LibRGW, SETATTR) {
 
       /* dir_0 MUST exist and MUST be resident */
       (void) rgw_lookup(fs, dir.parent_fh, dir.name.c_str(), &dir.fh,
-			RGW_LOOKUP_FLAG_NONE);
+			nullptr, 0, RGW_LOOKUP_FLAG_NONE);
 
       ASSERT_NE(dir.fh, nullptr);
       dir.sync();
@@ -408,12 +408,12 @@ TEST(LibRGW, SETATTR) {
       obj_rec sf{sfname, nullptr, dir.fh, nullptr};
 
       (void) rgw_lookup(fs, sf.parent_fh, sf.name.c_str(), &sf.fh,
-			RGW_LOOKUP_FLAG_NONE);
+			nullptr, 0, RGW_LOOKUP_FLAG_NONE);
 
       if (! sf.fh) {
 	/* make a new file object (the hard way) */
 	rc = rgw_lookup(fs, sf.parent_fh, sf.name.c_str(), &sf.fh,
-			RGW_LOOKUP_FLAG_CREATE);
+			nullptr, 0, RGW_LOOKUP_FLAG_CREATE);
 	ASSERT_EQ(rc, 0);
 	sf.sync();
 	ASSERT_TRUE(sf.rgw_fh->is_file());
@@ -457,7 +457,7 @@ TEST(LibRGW, SETATTR) {
 
       /* revalidate -- expect magic uid and gid */
       (void) rgw_lookup(fs, sf.parent_fh, sf.name.c_str(), &sf.fh,
-			RGW_LOOKUP_FLAG_NONE);
+			nullptr, 0, RGW_LOOKUP_FLAG_NONE);
       sf.sync();
       ASSERT_NE(sf.fh, nullptr);
 
@@ -495,7 +495,7 @@ TEST(LibRGW, RGW_CREATE_DIRS1) {
 	std::string sfname{"sfile_" + to_string(n_dirs1_objs)};
 	obj_rec sf{sfname, nullptr, dir.fh, nullptr};
 	(void) rgw_lookup(fs, sf.parent_fh, sf.name.c_str(), &sf.fh,
-			  RGW_LOOKUP_FLAG_NONE);
+			  nullptr, 0, RGW_LOOKUP_FLAG_NONE);
 	if (! sf.fh) {
 	  rc = rgw_create(fs, sf.parent_fh, sf.name.c_str(), &st, create_mask,
 			  &sf.fh, 0 /* posix flags */, RGW_CREATE_FLAG_NONE);
@@ -523,7 +523,7 @@ TEST(LibRGW, RGW_SETUP_RENAME1) {
       std::string bname{"brename_" + to_string(b_ix)};
       obj_rec brec{bname, nullptr, nullptr, nullptr};
       (void) rgw_lookup(fs, fs->root_fh, brec.name.c_str(), &brec.fh,
-			RGW_LOOKUP_FLAG_NONE);
+			nullptr, 0, RGW_LOOKUP_FLAG_NONE);
       if (! brec.fh) {
 	if (do_create) {
 	  struct stat st;
@@ -542,7 +542,7 @@ TEST(LibRGW, RGW_SETUP_RENAME1) {
 	rfname += to_string(f_ix);
 	obj_rec rf{rfname, nullptr, brec.fh, nullptr};
 	(void) rgw_lookup(fs, rf.parent_fh, rf.name.c_str(), &rf.fh,
-			  RGW_LOOKUP_FLAG_NONE);
+			  nullptr, 0, RGW_LOOKUP_FLAG_NONE);
 	if (! rf.fh) {
 	  rc = rgw_create(fs, rf.parent_fh, rf.name.c_str(), &st, create_mask,
 			  &rf.fh, 0 /* posix flags */, RGW_CREATE_FLAG_NONE);
@@ -729,7 +729,7 @@ TEST(LibRGW, READF_DIRS1) {
       obj_rec fobj{readf_name, nullptr, dirs1_b.fh, nullptr};
 
       int rc = rgw_lookup(fs, dirs1_b.fh, fobj.name.c_str(), &fobj.fh,
-			  RGW_LOOKUP_FLAG_NONE);
+			  nullptr, 0, RGW_LOOKUP_FLAG_NONE);
       ASSERT_EQ(rc, 0);
       ASSERT_NE(fobj.fh, nullptr);
       fobj.sync();
@@ -768,12 +768,12 @@ TEST(LibRGW, WRITEF_DIRS1) {
       obj_rec fobj{writef_name, nullptr, dirs1_b.fh, nullptr};
 
       (void) rgw_lookup(fs, fobj.parent_fh, fobj.name.c_str(), &fobj.fh,
-			RGW_LOOKUP_FLAG_NONE);
+			nullptr, 0, RGW_LOOKUP_FLAG_NONE);
       if (! fobj.fh) {
 	if (do_create) {
 	  /* make a new file object (the hard way) */
 	  rc = rgw_lookup(fs, fobj.parent_fh, fobj.name.c_str(), &fobj.fh,
-			  RGW_LOOKUP_FLAG_CREATE);
+			  nullptr, 0, RGW_LOOKUP_FLAG_CREATE);
 	  ASSERT_EQ(rc, 0);
 	}
       }
@@ -853,6 +853,7 @@ TEST(LibRGW, RELEASE_DIRS1) {
 
 extern "C" {
   static bool r1_cb(const char* name, void *arg, uint64_t offset,
+		    struct stat* st, uint32_t st_mask,
 		    uint32_t flags) {
     struct rgw_file_handle* parent_fh
       = static_cast<struct rgw_file_handle*>(arg);
@@ -892,7 +893,7 @@ TEST(LibRGW, HIER1) {
 	  << " elt.name=" << elt.name
 	  << dendl;
 	rc = rgw_lookup(fs, parent_fh, elt.name.c_str(), &elt.fh,
-			RGW_LOOKUP_FLAG_NONE);
+			nullptr, 0, RGW_LOOKUP_FLAG_NONE);
 	ASSERT_EQ(rc, 0);
 	// XXXX
 	RGWFileHandle* efh = get_rgwfh(elt.fh);
@@ -965,7 +966,7 @@ TEST(LibRGW, MARKER1_SETUP_BUCKET) {
 		      &marker_fh, RGW_MKDIR_FLAG_NONE);
     } else {
       ret = rgw_lookup(fs, bucket_fh, marker_dir.c_str(), &marker_fh,
-		       RGW_LOOKUP_FLAG_NONE);
+		       nullptr, 0, RGW_LOOKUP_FLAG_NONE);
     }
     ASSERT_EQ(ret, 0);
   }
@@ -984,7 +985,7 @@ TEST(LibRGW, MARKER1_SETUP_OBJECTS)
       obj_rec obj{object_name, nullptr, marker_fh, nullptr};
       // lookup object--all operations are by handle
       ret = rgw_lookup(fs, marker_fh, obj.name.c_str(), &obj.fh,
-		       RGW_LOOKUP_FLAG_CREATE);
+		       nullptr, 0, RGW_LOOKUP_FLAG_CREATE);
       ASSERT_EQ(ret, 0);
       obj.rgw_fh = get_rgwfh(obj.fh);
       // open object--open transaction
@@ -1010,6 +1011,7 @@ TEST(LibRGW, MARKER1_SETUP_OBJECTS)
 
 extern "C" {
   static bool r2_cb(const char* name, void *arg, uint64_t offset,
+		    struct stat* st, uint32_t st_mask,
 		    uint32_t flags) {
     dirent_vec& dvec =
       *(static_cast<dirent_vec*>(arg));