qa: enhancement for subvol creation and mounting

Fixes: https://tracker.ceph.com/issues/54317
Signed-off-by: Milind Changire <mchangir@redhat.com>

7 files changed, 78 insertions, 4 deletions

diff --git a/qa/tasks/ceph.py b/qa/tasks/ceph.py
index db86d6d53cc..b10bc672310 100644
--- a/qa/tasks/ceph.py
+++ b/qa/tasks/ceph.py
@@ -444,6 +444,9 @@ def cephfs_setup(ctx, config):
             name = fs_config.pop('name')
             temp = deepcopy(cephfs_config)
             teuthology.deep_merge(temp, fs_config)
+            subvols = config.get('subvols', None)
+            if subvols:
+                teuthology.deep_merge(temp, {'subvols': subvols})
             fs = Filesystem(ctx, fs_config=temp, name=name, create=True)
             fss.append(fs)
 
diff --git a/qa/tasks/ceph_fuse.py b/qa/tasks/ceph_fuse.py
index 2712229f972..b443516230c 100644
--- a/qa/tasks/ceph_fuse.py
+++ b/qa/tasks/ceph_fuse.py
@@ -72,6 +72,20 @@ def task(ctx, config):
               mount_timeout: 120 # default is 30, give up if /sys/ is not populated
         - interactive:
 
+    Example that creates and mounts a subvol:
+
+        overrides:
+          ceph:
+            subvols:
+              create: 2
+              subvol_options: "--namespace-isolated --size 25000000000"
+          ceph-fuse:
+            client.0:
+              mount_subvol_num: 0
+          kclient:
+            client.1:
+              mount_subvol_num: 1
+
     :param ctx: Context
     :param config: Configuration
     """
diff --git a/qa/tasks/cephadm.py b/qa/tasks/cephadm.py
index ae0fd02c8ab..bd900733b46 100644
--- a/qa/tasks/cephadm.py
+++ b/qa/tasks/cephadm.py
@@ -890,6 +890,9 @@ def cephfs_setup(ctx, config):
             name = fs_config.pop('name')
             temp = deepcopy(cephfs_config)
             teuthology.deep_merge(temp, fs_config)
+            subvols = config.get('subvols', None)
+            if subvols:
+                teuthology.deep_merge(temp, {'subvols': subvols})
             fs = Filesystem(ctx, fs_config=temp, name=name, create=True)
             if set_allow_multifs:
                 fs.set_allow_multifs()
diff --git a/qa/tasks/cephfs/filesystem.py b/qa/tasks/cephfs/filesystem.py
index 65af4adaa2f..9319cd7d85a 100644
--- a/qa/tasks/cephfs/filesystem.py
+++ b/qa/tasks/cephfs/filesystem.py
@@ -699,6 +699,7 @@ class Filesystem(MDSCluster):
                 raise
 
         if self.fs_config is not None:
+            log.debug(f"fs_config: {self.fs_config}")
             max_mds = self.fs_config.get('max_mds', 1)
             if max_mds > 1:
                 self.set_max_mds(max_mds)
@@ -711,6 +712,34 @@ class Filesystem(MDSCluster):
             if session_timeout != 60:
                 self.set_session_timeout(session_timeout)
 
+            if self.fs_config.get('subvols', None) is not None:
+                log.debug(f"Creating {self.fs_config.get('subvols')} subvols "
+                          f"for filesystem '{self.name}'")
+                if not hasattr(self._ctx, "created_subvols"):
+                    self._ctx.created_subvols = dict()
+
+                subvols = self.fs_config.get('subvols')
+                assert(isinstance(subvols, dict))
+                assert(isinstance(subvols['create'], int))
+                assert(subvols['create'] > 0)
+
+                for sv in range(0, subvols['create']):
+                    sv_name = f'sv_{sv}'
+                    self.mon_manager.raw_cluster_cmd(
+                        'fs', 'subvolume', 'create', self.name, sv_name,
+                        self.fs_config.get('subvol_options', ''))
+
+                    if self.name not in self._ctx.created_subvols:
+                        self._ctx.created_subvols[self.name] = []
+                    
+                    subvol_path = self.mon_manager.raw_cluster_cmd(
+                        'fs', 'subvolume', 'getpath', self.name, sv_name)
+                    subvol_path = subvol_path.strip()
+                    self._ctx.created_subvols[self.name].append(subvol_path)
+            else:
+                log.debug(f"Not Creating any subvols for filesystem '{self.name}'")
+
+
         self.getinfo(refresh = True)
 
         # wait pgs to be clean
diff --git a/qa/tasks/cephfs/fuse_mount.py b/qa/tasks/cephfs/fuse_mount.py
index 6c82379e26e..c3b204537ab 100644
--- a/qa/tasks/cephfs/fuse_mount.py
+++ b/qa/tasks/cephfs/fuse_mount.py
@@ -113,8 +113,12 @@ class FuseMount(CephFSMount):
             mount_cmd += ['--id', self.client_id]
         if self.client_keyring_path and self.client_id:
             mount_cmd += ['-k', self.client_keyring_path]
-        if self.cephfs_mntpt:
-            mount_cmd += ["--client_mountpoint=" + self.cephfs_mntpt]
+
+        self.validate_subvol_options()
+
+        assert(self.cephfs_mntpt)
+        mount_cmd += ["--client_mountpoint=" + self.cephfs_mntpt]
+
         if self.cephfs_name:
             mount_cmd += ["--client_fs=" + self.cephfs_name]
         if mntopts:
diff --git a/qa/tasks/cephfs/kernel_mount.py b/qa/tasks/cephfs/kernel_mount.py
index 9eef819633d..26f40ab45b8 100644
--- a/qa/tasks/cephfs/kernel_mount.py
+++ b/qa/tasks/cephfs/kernel_mount.py
@@ -44,8 +44,6 @@ class KernelMount(CephFSMount):
 
         self.setup_netns()
 
-        if not self.cephfs_mntpt:
-            self.cephfs_mntpt = '/'
         if not self.cephfs_name:
             self.cephfs_name = 'cephfs'
 
@@ -85,6 +83,10 @@ class KernelMount(CephFSMount):
     def _make_mount_cmd_old_or_new_style(self):
         optd = {}
         mnt_stx = ''
+
+        self.validate_subvol_options()
+
+        assert(self.cephfs_mntpt)
         if self.syntax_style == 'v1':
             mnt_stx = f':{self.cephfs_mntpt}'
             if self.client_id:
diff --git a/qa/tasks/cephfs/mount.py b/qa/tasks/cephfs/mount.py
index 41ebcba8587..6f754e18fb3 100644
--- a/qa/tasks/cephfs/mount.py
+++ b/qa/tasks/cephfs/mount.py
@@ -1351,3 +1351,22 @@ class CephFSMount(object):
         checksum_text = self.run_shell(cmd).stdout.getvalue().strip()
         checksum_sorted = sorted(checksum_text.split('\n'), key=lambda v: v.split()[1])
         return hashlib.md5(('\n'.join(checksum_sorted)).encode('utf-8')).hexdigest()
+
+    def validate_subvol_options(self):
+        mount_subvol_num = self.client_config.get('mount_subvol_num', None)
+        if self.cephfs_mntpt and mount_subvol_num is not None:
+            log.warning("You cannot specify both: cephfs_mntpt and mount_subvol_num")
+            log.info(f"Mounting subvol {mount_subvol_num} for now")
+
+        if mount_subvol_num is not None:
+            # mount_subvol must be an index into the subvol path array for the fs
+            if not self.cephfs_name:
+                self.cephfs_name = 'cephfs'
+            assert(hasattr(self.ctx, "created_subvols"))
+            # mount_subvol must be specified under client.[0-9] yaml section
+            subvol_paths = self.ctx.created_subvols[self.cephfs_name]
+            path_to_mount = subvol_paths[mount_subvol_num]
+            self.cephfs_mntpt = path_to_mount
+        elif not self.cephfs_mntpt:
+            # default to the "/" path
+            self.cephfs_mntpt = "/"