Merge pull request #32828 from leseb/raw-db-wal

ceph-volume: add db and wal support to raw mode

7 files changed, 96 insertions, 126 deletions

diff --git a/src/ceph-volume/ceph_volume/devices/raw/activate.py b/src/ceph-volume/ceph_volume/devices/raw/activate.py
index 44164c844df..94f54f1a20b 100644
--- a/src/ceph-volume/ceph_volume/devices/raw/activate.py
+++ b/src/ceph-volume/ceph_volume/devices/raw/activate.py
@@ -11,9 +11,10 @@ from .list import direct_report
 
 logger = logging.getLogger(__name__)
 
-def activate_bluestore(meta, tmpfs, systemd):
+def activate_bluestore(meta, tmpfs, systemd, block_wal=None, block_db=None):
     # find the osd
     osd_id = meta['osd_id']
+    osd_uuid = meta['osd_uuid']
 
     # mount on tmpfs the osd directory
     osd_path = '/var/lib/ceph/osd/%s-%s' % (conf.cluster, osd_id)
@@ -42,18 +43,15 @@ def activate_bluestore(meta, tmpfs, systemd):
     # always re-do the symlink regardless if it exists, so that the block,
     # block.wal, and block.db devices that may have changed can be mapped
     # correctly every time
-    process.run(['ln', '-snf', meta['device'], os.path.join(osd_path, 'block')])
-    system.chown(os.path.join(osd_path, 'block'))
-    system.chown(osd_path)
+    prepare_utils.link_block( meta['device'], osd_id)
 
-#    if systemd:
-        # write me
-        # enable the OSD
-        #systemctl.enable_osd(osd_id)
+    if block_wal:
+        prepare_utils.link_wal(block_wal, osd_id, osd_uuid)
 
-        # start the OSD
-        #systemctl.start_osd(osd_id)
+    if block_db:
+        prepare_utils.link_db(block_db, osd_id, osd_uuid)
 
+    system.chown(osd_path)
     terminal.success("ceph-volume raw activate successful for osd ID: %s" % osd_id)
 
 
@@ -66,7 +64,7 @@ class Activate(object):
         self.args = None
 
     @decorators.needs_root
-    def activate(self, devices, tmpfs, systemd):
+    def activate(self, devices, tmpfs, systemd, block_wal, block_db):
         """
         :param args: The parsed arguments coming from the CLI
         """
@@ -75,18 +73,19 @@ class Activate(object):
 
         for osd_id, meta in found.items():
             logger.info('Activating osd.%s uuid %s cluster %s' % (
-                    osd_id, meta['osd_uuid'], meta['ceph_fsid']))
+                        osd_id, meta['osd_uuid'], meta['ceph_fsid']))
             activate_bluestore(meta,
                                tmpfs=tmpfs,
-                               systemd=systemd)
+                               systemd=systemd,
+                               block_wal=block_wal,
+                               block_db=block_db)
 
     def main(self):
         sub_command_help = dedent("""
-        Activate (BlueStore) OSD on a raw block device based on the 
+        Activate (BlueStore) OSD on a raw block device based on the
         device label (normally the first block of the device).
 
             ceph-volume raw activate --device /dev/sdb
-            ceph-volume raw activate --osd-id 1 --osd-fsid f0327efd-c28e-40bb-9199-f2e61e54c12a
 
         The device(s) associated with the OSD needs to have been prepared
         previously, so that all needed tags and metadata exist.
@@ -96,22 +95,34 @@ class Activate(object):
             formatter_class=argparse.RawDescriptionHelpFormatter,
             description=sub_command_help,
         )
-
         parser.add_argument(
             '--device',
             nargs='+',
-            help='The device(s) for the OSD to start')
+            help='The device for the OSD to start'
+            )
         parser.add_argument(
             '--no-systemd',
             dest='no_systemd',
             action='store_true',
-            help='Skip creating and enabling systemd units and starting OSD services',
+            help='Skip creating and enabling systemd units and starting OSD services'
+        )
+        parser.add_argument(
+            '--block.db',
+            dest='block_db',
+            help='Path to bluestore block.db block device'
+        )
+        parser.add_argument(
+            '--block.wal',
+            dest='block_wal',
+            help='Path to bluestore block.wal block device'
         )
         parser.add_argument(
             '--no-tmpfs',
             action='store_true',
-            help='Do not use a tmpfs mount for OSD data dir')
-        if len(self.argv) == 0:
+            help='Do not use a tmpfs mount for OSD data dir'
+            )
+
+        if not self.argv:
             print(sub_command_help)
             return
         args = parser.parse_args(self.argv)
@@ -121,4 +132,6 @@ class Activate(object):
             raise SystemExit(1)
         self.activate(args.device,
                       tmpfs=not args.no_tmpfs,
-                      systemd=not self.args.no_systemd)
+                      systemd=not self.args.no_systemd,
+                      block_wal=self.args.block_wal,
+                      block_db=self.args.block_db)
diff --git a/src/ceph-volume/ceph_volume/devices/raw/common.py b/src/ceph-volume/ceph_volume/devices/raw/common.py
index 03b6cd690ed..d34a2941d16 100644
--- a/src/ceph-volume/ceph_volume/devices/raw/common.py
+++ b/src/ceph-volume/ceph_volume/devices/raw/common.py
@@ -27,19 +27,22 @@ def create_parser(prog, description):
         help='Crush device class to assign this OSD to',
     )
     parser.add_argument(
-        '--osd-id',
-        help='Reuse an existing OSD id',
-    )
-    parser.add_argument(
-        '--osd-fsid',
-        help='Reuse an existing OSD UUID',
-    )
-    parser.add_argument(
         '--cluster-fsid',
         help='Specify the cluster fsid, useful when no ceph.conf is available',
     )
     parser.add_argument(
         '--no-tmpfs',
         action='store_true',
-        help='Do not use a tmpfs mount for OSD data dir')
+        help='Do not use a tmpfs mount for OSD data dir'
+    )
+    parser.add_argument(
+        '--block.db',
+        dest='block_db',
+        help='Path to bluestore block.db block device'
+    )
+    parser.add_argument(
+        '--block.wal',
+        dest='block_wal',
+        help='Path to bluestore block.wal block device'
+    )
     return parser
diff --git a/src/ceph-volume/ceph_volume/devices/raw/create.py b/src/ceph-volume/ceph_volume/devices/raw/create.py
deleted file mode 100644
index fb633c7ffca..00000000000
--- a/src/ceph-volume/ceph_volume/devices/raw/create.py
+++ /dev/null
@@ -1,81 +0,0 @@
-from __future__ import print_function
-from textwrap import dedent
-import logging
-from ceph_volume.util import system
-from ceph_volume import decorators, terminal
-from .prepare import Prepare
-from .activate import Activate
-from .common import create_parser
-from ceph_volume.devices.lvm.common import rollback_osd
-
-logger = logging.getLogger(__name__)
-
-
-class Create(object):
-
-    help = 'Create a new (BlueStore) OSD on a raw device'
-
-    def __init__(self, argv):
-        self.argv = argv
-        self.args = None
-
-    @decorators.needs_root
-    def create(self, args):
-        if not args.osd_fsid:
-            args.osd_fsid = system.generate_uuid()
-        prepare_step = Prepare([])
-        prepare_step.safe_prepare(args)
-        osd_id = prepare_step.osd_id
-        try:
-            # we try this for activate only when 'creating' an OSD,
-            # because a rollback should not happen when doing normal
-            # activation. For example when starting an OSD, systemd
-            # will call activate, which would never need to be rolled
-            # back.
-            a = Activate([])
-            a.args = self.args
-            a.activate([args.data],
-                       tmpfs=not args.no_tmpfs,
-                       systemd=not args.no_systemd)
-        except Exception:
-            logger.exception('raw activate was unable to complete, while creating the OSD')
-            logger.info('will rollback OSD ID creation')
-            rollback_osd(args, osd_id)
-            raise
-        terminal.success("ceph-volume raw create successful for: %s" % args.data)
-
-    def main(self):
-        sub_command_help = dedent("""
-        Create an OSD by assigning an ID and FSID, registering them with the
-        cluster with an ID and FSID, formatting and mounting the volume, and
-        starting the OSD daemon. This is a convinience command that combines
-        the prepare and activate steps.
-
-        Encryption is not supported.
-
-        Separate DB and WAL devices are not supported.
-
-            ceph-volume raw create --data /dev/sdb
-
-        """)
-        parser = create_parser(
-            prog='ceph-volume raw create',
-            description=sub_command_help,
-        )
-        parser.add_argument(
-            '--no-systemd',
-            dest='no_systemd',
-            action='store_true',
-            help='Skip creating and enabling systemd units and starting OSD services',
-        )
-        if len(self.argv) == 0:
-            print(sub_command_help)
-            return
-        self.args = parser.parse_args(self.argv)
-        if not self.args.bluestore:
-            terminal.error('must specify --bluestore (currently the only supported backend)')
-            raise SystemExit(1)
-        if not self.args.no_systemd:
-            terminal.error('systemd support not yet implemented')
-            raise SystemExit(1)
-        self.create(self.args)
diff --git a/src/ceph-volume/ceph_volume/devices/raw/list.py b/src/ceph-volume/ceph_volume/devices/raw/list.py
index 5d610c7a024..f72c135ed89 100644
--- a/src/ceph-volume/ceph_volume/devices/raw/list.py
+++ b/src/ceph-volume/ceph_volume/devices/raw/list.py
@@ -34,9 +34,7 @@ class List(object):
                 'lsblk', '--paths', '--nodeps', '--output=NAME', '--noheadings'
             ])
             assert not ret
-            r = json.loads(''.join(out))
-            for dev in r.get('blockdevices', []):
-                devs.append('/dev/' + dev['name'])
+            devs = out
         result = {}
         for dev in devs:
             logger.debug('Examining %s' % dev)
diff --git a/src/ceph-volume/ceph_volume/devices/raw/main.py b/src/ceph-volume/ceph_volume/devices/raw/main.py
index 61286acad1e..efa25109096 100644
--- a/src/ceph-volume/ceph_volume/devices/raw/main.py
+++ b/src/ceph-volume/ceph_volume/devices/raw/main.py
@@ -2,7 +2,6 @@ import argparse
 from textwrap import dedent
 from ceph_volume import terminal
 from . import list
-from . import create
 from . import prepare
 from . import activate
 
@@ -19,7 +18,6 @@ class Raw(object):
 
     mapper = {
         'list': list.List,
-        'create': create.Create,
         'prepare': prepare.Prepare,
         'activate': activate.Activate,
     }
diff --git a/src/ceph-volume/ceph_volume/devices/raw/prepare.py b/src/ceph-volume/ceph_volume/devices/raw/prepare.py
index 75163056373..cb5c59ce40f 100644
--- a/src/ceph-volume/ceph_volume/devices/raw/prepare.py
+++ b/src/ceph-volume/ceph_volume/devices/raw/prepare.py
@@ -11,7 +11,7 @@ from .common import create_parser
 logger = logging.getLogger(__name__)
 
 
-def prepare_bluestore(block, secrets, osd_id, fsid, tmpfs):
+def prepare_bluestore(block, wal, db, secrets, osd_id, fsid, tmpfs):
     """
     :param block: The name of the logical volume for the bluestore data
     :param wal: a regular/plain disk or logical volume, to be used for block.wal
@@ -34,6 +34,8 @@ def prepare_bluestore(block, secrets, osd_id, fsid, tmpfs):
     prepare_utils.osd_mkfs_bluestore(
         osd_id, fsid,
         keyring=cephx_secret,
+        wal=wal,
+        db=db
     )
 
 
@@ -71,22 +73,31 @@ class Prepare(object):
         """
         if self.args.cluster_fsid:
             return self.args.cluster_fsid
-        else:
-            return conf.ceph.get('global', 'fsid')
+
+        return conf.ceph.get('global', 'fsid')
 
     @decorators.needs_root
     def prepare(self):
         secrets = {'cephx_secret': prepare_utils.create_key()}
-        osd_fsid = self.args.osd_fsid or system.generate_uuid()
+        osd_fsid = system.generate_uuid()
         crush_device_class = self.args.crush_device_class
         if crush_device_class:
             secrets['crush_device_class'] = crush_device_class
         tmpfs = not self.args.no_tmpfs
+        wal = ""
+        db = ""
+        if self.args.block_wal:
+            wal = self.args.block_wal
+        if self.args.block_db:
+            db = self.args.block_db
+
         # reuse a given ID if it exists, otherwise create a new ID
         self.osd_id = prepare_utils.create_id(
-            osd_fsid, json.dumps(secrets), osd_id=self.args.osd_id)
+            osd_fsid, json.dumps(secrets))
         prepare_bluestore(
             self.args.data,
+            wal,
+            db,
             secrets,
             self.osd_id,
             osd_fsid,
@@ -103,16 +114,18 @@ class Prepare(object):
 
         Encryption is not supported.
 
-        DB and WAL devices are not supported.
-
             ceph-volume raw prepare --bluestore --data {device}
 
+        DB and WAL devices are supported.
+
+            ceph-volume raw prepare --bluestore --data {device} --block.db {device} --block.wal {device}
+
         """)
         parser = create_parser(
             prog='ceph-volume raw prepare',
             description=sub_command_help,
         )
-        if len(self.argv) == 0:
+        if not self.argv:
             print(sub_command_help)
             return
         self.args = parser.parse_args(self.argv)
diff --git a/src/ceph-volume/ceph_volume/util/prepare.py b/src/ceph-volume/ceph_volume/util/prepare.py
index c258dc4a359..867e2e2ec10 100644
--- a/src/ceph-volume/ceph_volume/util/prepare.py
+++ b/src/ceph-volume/ceph_volume/util/prepare.py
@@ -334,6 +334,30 @@ def _link_device(device, device_type, osd_id):
 
     process.run(command)
 
+def _validate_bluestore_device(device, excepted_device_type, osd_uuid):
+    """
+    Validate whether the given device is truly what it is supposed to be
+    """
+
+    out, err, ret = process.call(['ceph-bluestore-tool', 'show-label', '--dev', device])
+    if err:
+        terminal.error('ceph-bluestore-tool failed to run. %s'% err)
+        raise SystemExit(1)
+    if ret:
+        terminal.error('no label on %s'% device)
+        raise SystemExit(1)
+    oj = json.loads(''.join(out))
+    if device not in oj:
+        terminal.error('%s not in the output of ceph-bluestore-tool, buggy?'% device)
+        raise SystemExit(1)
+    current_device_type = oj[device]['description']
+    if current_device_type != excepted_device_type:
+        terminal.error('%s is not a %s device but %s'% (device, excepted_device_type, current_device_type))
+        raise SystemExit(1)
+    current_osd_uuid = oj[device]['osd_uuid']
+    if current_osd_uuid != osd_uuid:
+        terminal.error('device %s is used by another osd %s as %s, should be %s'% (device, current_osd_uuid, current_device_type, osd_uuid))
+        raise SystemExit(1)
 
 def link_journal(journal_device, osd_id):
     _link_device(journal_device, 'journal', osd_id)
@@ -343,11 +367,13 @@ def link_block(block_device, osd_id):
     _link_device(block_device, 'block', osd_id)
 
 
-def link_wal(wal_device, osd_id):
+def link_wal(wal_device, osd_id, osd_uuid=None):
+    _validate_bluestore_device(wal_device, 'bluefs wal', osd_uuid)
     _link_device(wal_device, 'block.wal', osd_id)
 
 
-def link_db(db_device, osd_id):
+def link_db(db_device, osd_id, osd_uuid=None):
+    _validate_bluestore_device(db_device, 'bluefs db', osd_uuid)
     _link_device(db_device, 'block.db', osd_id)