diff options
190 files changed, 1901 insertions, 869 deletions
diff --git a/qa/cephfs/clusters/3-mds.yaml b/qa/cephfs/clusters/3-mds.yaml index c655b90c81c..ff35ed1a65b 100644 --- a/qa/cephfs/clusters/3-mds.yaml +++ b/qa/cephfs/clusters/3-mds.yaml @@ -1,4 +1,4 @@ roles: -- [mon.a, mon.c, mds.a, osd.0, osd.1, osd.2] -- [mon.b, mds.b, mds.c, osd.3, osd.4, osd.5] +- [mon.a, mon.c, mgr.y, mds.a, osd.0, osd.1, osd.2] +- [mon.b, mgr.x, mds.b, mds.c, osd.3, osd.4, osd.5] - [client.0] diff --git a/qa/cephfs/clusters/9-mds.yaml b/qa/cephfs/clusters/9-mds.yaml index ed554c9fe3c..c1228b3a1c7 100644 --- a/qa/cephfs/clusters/9-mds.yaml +++ b/qa/cephfs/clusters/9-mds.yaml @@ -1,4 +1,4 @@ roles: -- [mon.a, mon.c, mds.a, mds.b, mds.c, mds.d, osd.0, osd.1, osd.2] -- [mon.b, mds.e, mds.f, mds.g, mds.h, mds.i, osd.3, osd.4, osd.5] +- [mon.a, mon.c, mgr.y, mds.a, mds.b, mds.c, mds.d, osd.0, osd.1, osd.2] +- [mon.b, mgr.x, mds.e, mds.f, mds.g, mds.h, mds.i, osd.3, osd.4, osd.5] - [client.0] diff --git a/qa/cephfs/clusters/fixed-2-ucephfs.yaml b/qa/cephfs/clusters/fixed-2-ucephfs.yaml index bc120ad0b29..5f4773fc5ae 100644 --- a/qa/cephfs/clusters/fixed-2-ucephfs.yaml +++ b/qa/cephfs/clusters/fixed-2-ucephfs.yaml @@ -1,6 +1,6 @@ roles: -- [mon.a, mds.a, osd.0, osd.1, client.0] -- [mon.b, mds.a-s, mon.c, osd.2, osd.3] +- [mon.a, mds.a, mgr.x, osd.0, osd.1, client.0] +- [mon.b, mds.a-s, mon.c, mgr.y, osd.2, osd.3] openstack: - volumes: # attached to each instance count: 2 diff --git a/qa/clusters/extra-client.yaml b/qa/clusters/extra-client.yaml index 349439cb331..2b495374f3d 100644 --- a/qa/clusters/extra-client.yaml +++ b/qa/clusters/extra-client.yaml @@ -1,6 +1,6 @@ roles: - [mon.a, mon.c, osd.0, osd.1, osd.2] -- [mon.b, mds.a, osd.3, osd.4, osd.5] +- [mon.b, mgr.x, mds.a, osd.3, osd.4, osd.5] - [client.0] - [client.1] openstack: diff --git a/qa/clusters/fixed-1.yaml b/qa/clusters/fixed-1.yaml index 37686073308..38725b08642 100644 --- a/qa/clusters/fixed-1.yaml +++ b/qa/clusters/fixed-1.yaml @@ -7,4 +7,4 @@ overrides: osd pool default pg num: 128 osd pool default pgp num: 128 roles: -- [mon.a, osd.0, osd.1, osd.2, client.0] +- [mon.a, mgr.x, osd.0, osd.1, osd.2, client.0] diff --git a/qa/clusters/fixed-2.yaml b/qa/clusters/fixed-2.yaml index 861e5091931..6886d1701b8 100644 --- a/qa/clusters/fixed-2.yaml +++ b/qa/clusters/fixed-2.yaml @@ -1,6 +1,6 @@ roles: -- [mon.a, mon.c, osd.0, osd.1, osd.2, client.0] -- [mon.b, osd.3, osd.4, osd.5, client.1] +- [mon.a, mon.c, mgr.y, osd.0, osd.1, osd.2, client.0] +- [mon.b, mgr.x, osd.3, osd.4, osd.5, client.1] openstack: - volumes: # attached to each instance count: 3 diff --git a/qa/clusters/fixed-3-cephfs.yaml b/qa/clusters/fixed-3-cephfs.yaml index 499c84c3718..7e2e5f05c76 100644 --- a/qa/clusters/fixed-3-cephfs.yaml +++ b/qa/clusters/fixed-3-cephfs.yaml @@ -1,6 +1,6 @@ roles: -- [mon.a, mds.a, osd.0, osd.1] -- [mon.b, mds.a-s, mon.c, osd.2, osd.3] +- [mon.a, mds.a, mgr.x, osd.0, osd.1] +- [mon.b, mds.a-s, mon.c, mgr.y, osd.2, osd.3] - [client.0] openstack: - volumes: # attached to each instance diff --git a/qa/clusters/fixed-3.yaml b/qa/clusters/fixed-3.yaml index 8e622d2db94..3ed30b94ff0 100644 --- a/qa/clusters/fixed-3.yaml +++ b/qa/clusters/fixed-3.yaml @@ -1,6 +1,6 @@ roles: -- [mon.a, mon.c, osd.0, osd.1, osd.2] -- [mon.b, osd.3, osd.4, osd.5] +- [mon.a, mon.c, mgr.x, osd.0, osd.1, osd.2] +- [mon.b, mgr.y, osd.3, osd.4, osd.5] - [client.0] openstack: - volumes: # attached to each instance diff --git a/qa/clusters/fixed-4.yaml b/qa/clusters/fixed-4.yaml index 7f051dc1fed..86eb2c63439 100644 --- a/qa/clusters/fixed-4.yaml +++ b/qa/clusters/fixed-4.yaml @@ -1,5 +1,5 @@ roles: -- [mon.a, osd.0, osd.4, osd.8, osd.12] +- [mon.a, mgr.y, osd.0, osd.4, osd.8, osd.12] - [mon.b, osd.1, osd.5, osd.9, osd.13] - [mon.c, osd.2, osd.6, osd.10, osd.14] -- [osd.3, osd.7, osd.11, osd.15, client.0] +- [mgr.x, osd.3, osd.7, osd.11, osd.15, client.0] diff --git a/qa/erasure-code/ec-feature-plugins-v2.yaml b/qa/erasure-code/ec-feature-plugins-v2.yaml index 102a4528924..f2d374dd98a 100644 --- a/qa/erasure-code/ec-feature-plugins-v2.yaml +++ b/qa/erasure-code/ec-feature-plugins-v2.yaml @@ -12,6 +12,7 @@ roles: - osd.1 - - osd.2 - mon.c + - mgr.x tasks: # # Install firefly diff --git a/qa/erasure-code/ec-feature-plugins-v3.yaml b/qa/erasure-code/ec-feature-plugins-v3.yaml index 66a5726d12b..332b9440728 100644 --- a/qa/erasure-code/ec-feature-plugins-v3.yaml +++ b/qa/erasure-code/ec-feature-plugins-v3.yaml @@ -12,6 +12,7 @@ roles: - osd.1 - - osd.2 - mon.c + - mgr.x tasks: # # Install hammer diff --git a/qa/releases/luminous.yaml b/qa/releases/luminous.yaml index 849bd824427..b7ff0dc53b6 100644 --- a/qa/releases/luminous.yaml +++ b/qa/releases/luminous.yaml @@ -1,5 +1,13 @@ tasks: - exec: + mgr.x: + - mkdir -p /var/lib/ceph/mgr/ceph-x + - ceph auth get-or-create-key mgr.x mon 'allow profile mgr' + - ceph auth export mgr.x > /var/lib/ceph/mgr/ceph-x/keyring +- ceph.restart: + daemons: [mgr.x] + wait-for-healthy: false +- exec: osd.0: - ceph osd set require_luminous_osds - ceph.healthy: @@ -8,3 +16,5 @@ overrides: conf: mon: mon warn on osd down out interval zero: false + log-whitelist: + - no active mgr diff --git a/qa/suites/big/rados-thrash/clusters/big.yaml b/qa/suites/big/rados-thrash/clusters/big.yaml index 18197ad8571..fd8c2174118 100644 --- a/qa/suites/big/rados-thrash/clusters/big.yaml +++ b/qa/suites/big/rados-thrash/clusters/big.yaml @@ -1,7 +1,7 @@ roles: -- [osd.0, osd.1, osd.2, client.0, mon.a] -- [osd.3, osd.4, osd.5, client.1, mon.b] -- [osd.6, osd.7, osd.8, client.2, mon.c] +- [osd.0, osd.1, osd.2, client.0, mon.a, mgr.x] +- [osd.3, osd.4, osd.5, client.1, mon.b, mgr.y] +- [osd.6, osd.7, osd.8, client.2, mon.c, mgr.z] - [osd.9, osd.10, osd.11, client.3, mon.d] - [osd.12, osd.13, osd.14, client.4, mon.e] - [osd.15, osd.16, osd.17, client.5] diff --git a/qa/suites/big/rados-thrash/clusters/medium.yaml b/qa/suites/big/rados-thrash/clusters/medium.yaml index 48b66dd5ca3..ecded01b4ff 100644 --- a/qa/suites/big/rados-thrash/clusters/medium.yaml +++ b/qa/suites/big/rados-thrash/clusters/medium.yaml @@ -1,7 +1,7 @@ roles: -- [osd.0, osd.1, osd.2, client.0, mon.a] -- [osd.3, osd.4, osd.5, client.1, mon.b] -- [osd.6, osd.7, osd.8, client.2, mon.c] +- [osd.0, osd.1, osd.2, client.0, mon.a, mgr.x] +- [osd.3, osd.4, osd.5, client.1, mon.b, mgr.y] +- [osd.6, osd.7, osd.8, client.2, mon.c, mgr.z] - [osd.9, osd.10, osd.11, client.3, mon.d] - [osd.12, osd.13, osd.14, client.4, mon.e] - [osd.15, osd.16, osd.17, client.5] diff --git a/qa/suites/big/rados-thrash/clusters/small.yaml b/qa/suites/big/rados-thrash/clusters/small.yaml index b5a79906c69..d0aecd00875 100644 --- a/qa/suites/big/rados-thrash/clusters/small.yaml +++ b/qa/suites/big/rados-thrash/clusters/small.yaml @@ -1,6 +1,6 @@ roles: -- [osd.0, osd.1, osd.2, client.0, mon.a] -- [osd.3, osd.4, osd.5, client.1, mon.b] -- [osd.6, osd.7, osd.8, client.2, mon.c] +- [osd.0, osd.1, osd.2, client.0, mon.a, mgr.x] +- [osd.3, osd.4, osd.5, client.1, mon.b, mgr.y] +- [osd.6, osd.7, osd.8, client.2, mon.c, mgr.z] - [osd.9, osd.10, osd.11, client.3, mon.d] - [osd.12, osd.13, osd.14, client.4, mon.e] diff --git a/qa/suites/ceph-ansible/smoke/basic/0-clusters/3-node.yaml b/qa/suites/ceph-ansible/smoke/basic/0-clusters/3-node.yaml index c67a5cb63b9..0b4b6b15713 100644 --- a/qa/suites/ceph-ansible/smoke/basic/0-clusters/3-node.yaml +++ b/qa/suites/ceph-ansible/smoke/basic/0-clusters/3-node.yaml @@ -2,8 +2,8 @@ meta: - desc: "3-node cluster" roles: - [mon.a, mds.a, osd.0, osd.1, osd.2] -- [mon.b, osd.3, osd.4, osd.5] -- [mon.c, osd.6, osd.7, osd.8, client.0] +- [mon.b, mgr.x, osd.3, osd.4, osd.5] +- [mon.c, mgr.y, osd.6, osd.7, osd.8, client.0] openstack: - volumes: # attached to each instance count: 3 diff --git a/qa/suites/ceph-deploy/basic/tasks/ceph-admin-commands.yaml b/qa/suites/ceph-deploy/basic/tasks/ceph-admin-commands.yaml index 0d651ca7641..fc4873ceed1 100644 --- a/qa/suites/ceph-deploy/basic/tasks/ceph-admin-commands.yaml +++ b/qa/suites/ceph-deploy/basic/tasks/ceph-admin-commands.yaml @@ -1,5 +1,6 @@ roles: - - mon.a + - mgr.x - mds.0 - osd.0 - - osd.1 diff --git a/qa/suites/ceph-disk/basic/tasks/ceph-disk.yaml b/qa/suites/ceph-disk/basic/tasks/ceph-disk.yaml index b7655a2fc54..98adab33e57 100644 --- a/qa/suites/ceph-disk/basic/tasks/ceph-disk.yaml +++ b/qa/suites/ceph-disk/basic/tasks/ceph-disk.yaml @@ -1,5 +1,6 @@ roles: - - mon.a + - mgr.x - client.0 - - osd.0 - osd.1 diff --git a/qa/suites/dummy/all/nop.yaml b/qa/suites/dummy/all/nop.yaml index e027e553395..0f00ffc8870 100644 --- a/qa/suites/dummy/all/nop.yaml +++ b/qa/suites/dummy/all/nop.yaml @@ -1,5 +1,5 @@ roles: - - [mon.a, mds.a, osd.0, osd.1, client.0] + - [mon.a, mgr.x, mds.a, osd.0, osd.1, client.0] tasks: - nop: diff --git a/qa/suites/experimental/multimds/clusters/7-multimds.yaml b/qa/suites/experimental/multimds/clusters/7-multimds.yaml index 17cfd7b3d79..d4bb14143d3 100644 --- a/qa/suites/experimental/multimds/clusters/7-multimds.yaml +++ b/qa/suites/experimental/multimds/clusters/7-multimds.yaml @@ -1,7 +1,7 @@ roles: -- [mon.a, mds.a, mds.a-s] -- [mon.b, mds.b, mds.b-s] -- [mon.c, mds.c, mds.c-s] +- [mon.a, mgr.x, mds.a, mds.a-s] +- [mon.b, mgr.y, mds.b, mds.b-s] +- [mon.c, mgr.z, mds.c, mds.c-s] - [osd.0] - [osd.1] - [osd.2] diff --git a/qa/suites/fs/multiclient/clusters/three_clients.yaml b/qa/suites/fs/multiclient/clusters/three_clients.yaml index 1fc1b445440..78b912f7252 100644 --- a/qa/suites/fs/multiclient/clusters/three_clients.yaml +++ b/qa/suites/fs/multiclient/clusters/three_clients.yaml @@ -1,5 +1,5 @@ roles: -- [mon.a, mon.b, mon.c, mds.a, osd.0, osd.1, osd.2] +- [mon.a, mon.b, mon.c, mgr.x, mds.a, osd.0, osd.1, osd.2] - [client.2] - [client.1] - [client.0] diff --git a/qa/suites/fs/multiclient/clusters/two_clients.yaml b/qa/suites/fs/multiclient/clusters/two_clients.yaml index e6d82601cfd..9586e6c8ffd 100644 --- a/qa/suites/fs/multiclient/clusters/two_clients.yaml +++ b/qa/suites/fs/multiclient/clusters/two_clients.yaml @@ -1,5 +1,5 @@ roles: -- [mon.a, mon.b, mon.c, mds.a, osd.0, osd.1, osd.2] +- [mon.a, mon.b, mon.c, mgr.x, mds.a, osd.0, osd.1, osd.2] - [client.1] - [client.0] diff --git a/qa/suites/fs/multifs/clusters/2-remote-clients.yaml b/qa/suites/fs/multifs/clusters/2-remote-clients.yaml index b5ff712cb47..52c5d7e0171 100644 --- a/qa/suites/fs/multifs/clusters/2-remote-clients.yaml +++ b/qa/suites/fs/multifs/clusters/2-remote-clients.yaml @@ -1,5 +1,5 @@ roles: -- [mon.a, osd.0, mon.b, mds.a, mds.b, client.1] +- [mon.a, mgr.x, osd.0, mon.b, mds.a, mds.b, client.1] - [mds.c, mds.d, mon.c, client.0, osd.1, osd.2] openstack: - volumes: # attached to each instance diff --git a/qa/suites/fs/recovery/clusters/4-remote-clients.yaml b/qa/suites/fs/recovery/clusters/4-remote-clients.yaml index 6b5e23b6c1f..f9e423e2fc3 100644 --- a/qa/suites/fs/recovery/clusters/4-remote-clients.yaml +++ b/qa/suites/fs/recovery/clusters/4-remote-clients.yaml @@ -1,5 +1,5 @@ roles: -- [mon.a, osd.0, mds.a, mds.b, client.1, client.2, client.3] +- [mon.a, mgr.x, osd.0, mds.a, mds.b, client.1, client.2, client.3] - [client.0, osd.1, osd.2] openstack: - volumes: # attached to each instance diff --git a/qa/suites/fs/standbyreplay/clusters/standby-replay.yaml b/qa/suites/fs/standbyreplay/clusters/standby-replay.yaml index ba2442b27ca..a4bafed29a4 100644 --- a/qa/suites/fs/standbyreplay/clusters/standby-replay.yaml +++ b/qa/suites/fs/standbyreplay/clusters/standby-replay.yaml @@ -6,7 +6,7 @@ overrides: mds standby replay: true roles: -- [mon.a, mds.a, mds.b-s-0, osd.0, osd.1, client.0] +- [mon.a, mgr.x, mds.a, mds.b-s-0, osd.0, osd.1, client.0] - [mon.b, mds.c-s-0, mds.d-s-0, mon.c, osd.2, osd.3] openstack: - volumes: # attached to each instance diff --git a/qa/suites/fs/thrash/clusters/mds-1active-1standby.yaml b/qa/suites/fs/thrash/clusters/mds-1active-1standby.yaml index cceea69e263..d02524866c3 100644 --- a/qa/suites/fs/thrash/clusters/mds-1active-1standby.yaml +++ b/qa/suites/fs/thrash/clusters/mds-1active-1standby.yaml @@ -1,6 +1,6 @@ roles: - [mon.a, mon.c, osd.0, osd.1, osd.2, mds.b-s-a] -- [mon.b, mds.a, osd.3, osd.4, osd.5, client.0] +- [mon.b, mgr.x, mds.a, osd.3, osd.4, osd.5, client.0] openstack: - volumes: # attached to each instance count: 3 diff --git a/qa/suites/hadoop/basic/clusters/fixed-3.yaml b/qa/suites/hadoop/basic/clusters/fixed-3.yaml index 52efc693074..6fc0af68f4b 100644 --- a/qa/suites/hadoop/basic/clusters/fixed-3.yaml +++ b/qa/suites/hadoop/basic/clusters/fixed-3.yaml @@ -9,8 +9,8 @@ overrides: client permissions: false roles: - [mon.0, mds.0, osd.0, hadoop.master.0] -- [mon.1, osd.1, hadoop.slave.0] -- [mon.2, hadoop.slave.1, client.0] +- [mon.1, mgr.x, osd.1, hadoop.slave.0] +- [mon.2, mgr.y, hadoop.slave.1, client.0] openstack: - volumes: # attached to each instance count: 1 diff --git a/qa/suites/kcephfs/mixed-clients/clusters/2-clients.yaml b/qa/suites/kcephfs/mixed-clients/clusters/2-clients.yaml index 26e029d0dbe..90b6cf626c8 100644 --- a/qa/suites/kcephfs/mixed-clients/clusters/2-clients.yaml +++ b/qa/suites/kcephfs/mixed-clients/clusters/2-clients.yaml @@ -1,5 +1,5 @@ roles: -- [mon.a, mds.a, osd.0, osd.1] +- [mon.a, mgr.x, mds.a, osd.0, osd.1] - [mon.b, mon.c, osd.2, osd.3] - [client.0] - [client.1] diff --git a/qa/suites/kcephfs/recovery/clusters/4-remote-clients.yaml b/qa/suites/kcephfs/recovery/clusters/4-remote-clients.yaml index 7bb4e3255be..1d432addb4a 100644 --- a/qa/suites/kcephfs/recovery/clusters/4-remote-clients.yaml +++ b/qa/suites/kcephfs/recovery/clusters/4-remote-clients.yaml @@ -1,6 +1,6 @@ roles: - [mon.a, osd.0, mds.a, mds.c, client.2] -- [osd.1, osd.2, mds.b, mds.d, client.3] +- [mgr.x, osd.1, osd.2, mds.b, mds.d, client.3] - [client.0] - [client.1] openstack: diff --git a/qa/suites/krbd/singleton/tasks/rbd_xfstests.yaml b/qa/suites/krbd/singleton/tasks/rbd_xfstests.yaml index 237a4947f2a..d27054a8482 100644 --- a/qa/suites/krbd/singleton/tasks/rbd_xfstests.yaml +++ b/qa/suites/krbd/singleton/tasks/rbd_xfstests.yaml @@ -1,6 +1,6 @@ roles: - [mon.a, mon.c, osd.0, osd.1, osd.2] -- [mon.b, mds.a, osd.3, osd.4, osd.5] +- [mon.b, mgr.x, mds.a, osd.3, osd.4, osd.5] - [client.0] - [client.1] openstack: diff --git a/qa/suites/krbd/unmap/clusters/separate-client.yaml b/qa/suites/krbd/unmap/clusters/separate-client.yaml index a2184eefc5d..be13431890f 100644 --- a/qa/suites/krbd/unmap/clusters/separate-client.yaml +++ b/qa/suites/krbd/unmap/clusters/separate-client.yaml @@ -8,7 +8,7 @@ overrides: osd pool default pg num: 128 osd pool default pgp num: 128 roles: -- [mon.a, osd.0, osd.1, osd.2] +- [mon.a, mgr.x, osd.0, osd.1, osd.2] - [client.0] openstack: - volumes: # attached to each instance diff --git a/qa/suites/marginal/basic/clusters/fixed-3.yaml b/qa/suites/marginal/basic/clusters/fixed-3.yaml index 0038432afa7..5e23c9e4f44 100644 --- a/qa/suites/marginal/basic/clusters/fixed-3.yaml +++ b/qa/suites/marginal/basic/clusters/fixed-3.yaml @@ -1,4 +1,4 @@ roles: - [mon.a, mon.c, osd.0, osd.1, osd.2] -- [mon.b, mds.a, osd.3, osd.4, osd.5] +- [mon.b, mgr.x, mds.a, osd.3, osd.4, osd.5] - [client.0] diff --git a/qa/suites/marginal/fs-misc/clusters/two_clients.yaml b/qa/suites/marginal/fs-misc/clusters/two_clients.yaml index 2258befd8bf..19d312dce57 100644 --- a/qa/suites/marginal/fs-misc/clusters/two_clients.yaml +++ b/qa/suites/marginal/fs-misc/clusters/two_clients.yaml @@ -1,4 +1,4 @@ roles: -- [mon.a, mon.b, mon.c, mds.a, osd.0, osd.1, osd.2] +- [mon.a, mon.b, mon.c, mgr.x, mds.a, osd.0, osd.1, osd.2] - [client.1] - [client.0] diff --git a/qa/suites/marginal/mds_restart/clusters/one_mds.yaml b/qa/suites/marginal/mds_restart/clusters/one_mds.yaml index 9e11c02a36c..45c3e80e57d 100644 --- a/qa/suites/marginal/mds_restart/clusters/one_mds.yaml +++ b/qa/suites/marginal/mds_restart/clusters/one_mds.yaml @@ -1,4 +1,4 @@ roles: -- [mon.a, mon.b, mon.c, osd.0, osd.1, osd.2] +- [mon.a, mon.b, mon.c, mgr.x, osd.0, osd.1, osd.2] - [mds.a] - [client.0] diff --git a/qa/suites/marginal/multimds/clusters/3-node-3-mds.yaml b/qa/suites/marginal/multimds/clusters/3-node-3-mds.yaml index 088d9f0d31d..2995ea9f358 100644 --- a/qa/suites/marginal/multimds/clusters/3-node-3-mds.yaml +++ b/qa/suites/marginal/multimds/clusters/3-node-3-mds.yaml @@ -1,5 +1,5 @@ roles: - [mon.a, mon.c, mds.a, osd.0, osd.1, osd.2] -- [mon.b, mds.b, mds.c, osd.3, osd.4, osd.5] +- [mon.b, mgr.x, mds.b, mds.c, osd.3, osd.4, osd.5] - [client.0] - [client.1] diff --git a/qa/suites/marginal/multimds/clusters/3-node-9-mds.yaml b/qa/suites/marginal/multimds/clusters/3-node-9-mds.yaml index be824f0f554..083a07c25e1 100644 --- a/qa/suites/marginal/multimds/clusters/3-node-9-mds.yaml +++ b/qa/suites/marginal/multimds/clusters/3-node-9-mds.yaml @@ -1,5 +1,5 @@ roles: - [mon.a, mon.c, mds.a, mds.b, mds.c, mds.d, osd.0, osd.1, osd.2] -- [mon.b, mds.e, mds.f, mds.g, mds.h, mds.i, osd.3, osd.4, osd.5] +- [mon.b, mgr.x, mds.e, mds.f, mds.g, mds.h, mds.i, osd.3, osd.4, osd.5] - [client.0] - [client.1] diff --git a/qa/suites/mixed-clients/basic/clusters/fixed-3.yaml b/qa/suites/mixed-clients/basic/clusters/fixed-3.yaml index e1d3c7b7932..134bca1b9a3 100644 --- a/qa/suites/mixed-clients/basic/clusters/fixed-3.yaml +++ b/qa/suites/mixed-clients/basic/clusters/fixed-3.yaml @@ -1,4 +1,4 @@ roles: -- [mon.a, mds.a, osd.0, osd.1] +- [mon.a, mgr.x, mds.a, osd.0, osd.1] - [mon.b, mon.c, osd.2, osd.3, client.0] - [client.1] diff --git a/qa/suites/multimds/thrash/clusters/3-mds-2-standby.yaml b/qa/suites/multimds/thrash/clusters/3-mds-2-standby.yaml index acf5deeee8f..1a415ef249e 100644 --- a/qa/suites/multimds/thrash/clusters/3-mds-2-standby.yaml +++ b/qa/suites/multimds/thrash/clusters/3-mds-2-standby.yaml @@ -1,4 +1,4 @@ roles: - [mon.a, mon.c, mds.a, osd.0, osd.1, osd.2, mds.d-s] -- [mon.b, mds.b, mds.c, osd.3, osd.4, osd.5, mds.e-s] +- [mon.b, mgr.x, mds.b, mds.c, osd.3, osd.4, osd.5, mds.e-s] - [client.0] diff --git a/qa/suites/multimds/thrash/clusters/9-mds-3-standby.yaml b/qa/suites/multimds/thrash/clusters/9-mds-3-standby.yaml index c51bc16dffd..4a8334c66a2 100644 --- a/qa/suites/multimds/thrash/clusters/9-mds-3-standby.yaml +++ b/qa/suites/multimds/thrash/clusters/9-mds-3-standby.yaml @@ -1,4 +1,4 @@ roles: - [mon.a, mon.c, mds.a, mds.b, mds.c, mds.d, osd.0, osd.1, osd.2, mds.j-s, mds.k-s] -- [mon.b, mds.e, mds.f, mds.g, mds.h, mds.i, osd.3, osd.4, osd.5, mds.l-s] +- [mon.b, mgr.x, mds.e, mds.f, mds.g, mds.h, mds.i, osd.3, osd.4, osd.5, mds.l-s] - [client.0] diff --git a/qa/suites/powercycle/osd/clusters/3osd-1per-target.yaml b/qa/suites/powercycle/osd/clusters/3osd-1per-target.yaml index d5503a40c86..8ba3c691e1a 100644 --- a/qa/suites/powercycle/osd/clusters/3osd-1per-target.yaml +++ b/qa/suites/powercycle/osd/clusters/3osd-1per-target.yaml @@ -1,5 +1,5 @@ roles: -- [mon.0, mon.1, mon.2, mds.0, client.0] +- [mon.a, mon.b, mon.c, mgr.x, mgr.y, mds.0, client.0] - [osd.0] - [osd.1] - [osd.2] diff --git a/qa/suites/rados/monthrash/clusters/3-mons.yaml b/qa/suites/rados/monthrash/clusters/3-mons.yaml index b3a2caf7212..4b721ef80c7 100644 --- a/qa/suites/rados/monthrash/clusters/3-mons.yaml +++ b/qa/suites/rados/monthrash/clusters/3-mons.yaml @@ -1,6 +1,6 @@ roles: - [mon.a, mon.c, osd.0, osd.1, osd.2] -- [mon.b, osd.3, osd.4, osd.5, client.0] +- [mon.b, mgr.x, osd.3, osd.4, osd.5, client.0] openstack: - volumes: # attached to each instance count: 3 diff --git a/qa/suites/rados/monthrash/clusters/9-mons.yaml b/qa/suites/rados/monthrash/clusters/9-mons.yaml index 54332118434..a2874c1d0e5 100644 --- a/qa/suites/rados/monthrash/clusters/9-mons.yaml +++ b/qa/suites/rados/monthrash/clusters/9-mons.yaml @@ -1,6 +1,6 @@ roles: - [mon.a, mon.b, mon.c, mon.d, mon.e, osd.0, osd.1, osd.2] -- [mon.f, mon.g, mon.h, mon.i, osd.3, osd.4, osd.5, client.0] +- [mon.f, mon.g, mon.h, mon.i, mgr.x, osd.3, osd.4, osd.5, client.0] openstack: - volumes: # attached to each instance count: 3 diff --git a/qa/suites/rados/multimon/clusters/21.yaml b/qa/suites/rados/multimon/clusters/21.yaml index 646ff15d45e..b6d04a022d8 100644 --- a/qa/suites/rados/multimon/clusters/21.yaml +++ b/qa/suites/rados/multimon/clusters/21.yaml @@ -1,6 +1,6 @@ roles: - [mon.a, mon.d, mon.g, mon.j, mon.m, mon.p, mon.s, osd.0] -- [mon.b, mon.e, mon.h, mon.k, mon.n, mon.q, mon.t] +- [mon.b, mon.e, mon.h, mon.k, mon.n, mon.q, mon.t, mgr.x] - [mon.c, mon.f, mon.i, mon.l, mon.o, mon.r, mon.u, osd.1] openstack: - volumes: # attached to each instance diff --git a/qa/suites/rados/multimon/clusters/3.yaml b/qa/suites/rados/multimon/clusters/3.yaml index e663324b5d6..7b8c62645dd 100644 --- a/qa/suites/rados/multimon/clusters/3.yaml +++ b/qa/suites/rados/multimon/clusters/3.yaml @@ -1,6 +1,6 @@ roles: - [mon.a, mon.c, osd.0] -- [mon.b, osd.1] +- [mon.b, mgr.x, osd.1] openstack: - volumes: # attached to each instance count: 2 diff --git a/qa/suites/rados/multimon/clusters/6.yaml b/qa/suites/rados/multimon/clusters/6.yaml index b16e3267c06..715cbe855ae 100644 --- a/qa/suites/rados/multimon/clusters/6.yaml +++ b/qa/suites/rados/multimon/clusters/6.yaml @@ -1,6 +1,6 @@ roles: -- [mon.a, mon.c, mon.e, osd.0] -- [mon.b, mon.d, mon.f, osd.1] +- [mon.a, mon.c, mon.e, mgr.x, osd.0] +- [mon.b, mon.d, mon.f, mgr.y, osd.1] openstack: - volumes: # attached to each instance count: 1 diff --git a/qa/suites/rados/multimon/clusters/9.yaml b/qa/suites/rados/multimon/clusters/9.yaml index c2c7b494ed8..d029c4c594d 100644 --- a/qa/suites/rados/multimon/clusters/9.yaml +++ b/qa/suites/rados/multimon/clusters/9.yaml @@ -1,6 +1,6 @@ roles: - [mon.a, mon.d, mon.g, osd.0] -- [mon.b, mon.e, mon.h] +- [mon.b, mon.e, mon.h, mgr.x] - [mon.c, mon.f, mon.i, osd.1] openstack: - volumes: # attached to each instance diff --git a/qa/suites/rados/objectstore/alloc-hint.yaml b/qa/suites/rados/objectstore/alloc-hint.yaml index cc608663c54..d40143c17a7 100644 --- a/qa/suites/rados/objectstore/alloc-hint.yaml +++ b/qa/suites/rados/objectstore/alloc-hint.yaml @@ -1,5 +1,5 @@ roles: -- [mon.a, osd.0, osd.1, osd.2, client.0] +- [mon.a, mgr.x, osd.0, osd.1, osd.2, client.0] openstack: - volumes: # attached to each instance count: 3 diff --git a/qa/suites/rados/objectstore/ceph_objectstore_tool.yaml b/qa/suites/rados/objectstore/ceph_objectstore_tool.yaml index e062459c691..215d0f08f9b 100644 --- a/qa/suites/rados/objectstore/ceph_objectstore_tool.yaml +++ b/qa/suites/rados/objectstore/ceph_objectstore_tool.yaml @@ -1,5 +1,5 @@ roles: -- [mon.0, osd.0, osd.1, osd.2, osd.3, osd.4, osd.5, client.0] +- [mon.a, mgr.x, osd.0, osd.1, osd.2, osd.3, osd.4, osd.5, client.0] openstack: - volumes: # attached to each instance count: 6 diff --git a/qa/suites/rados/objectstore/filejournal.yaml b/qa/suites/rados/objectstore/filejournal.yaml index 7e401d168f5..b0af80084f8 100644 --- a/qa/suites/rados/objectstore/filejournal.yaml +++ b/qa/suites/rados/objectstore/filejournal.yaml @@ -1,5 +1,5 @@ roles: -- [mon.0, osd.0, osd.1, client.0] +- [mon.a, mgr.x, osd.0, osd.1, client.0] openstack: - volumes: # attached to each instance count: 2 diff --git a/qa/suites/rados/objectstore/filestore-idempotent-aio-journal.yaml b/qa/suites/rados/objectstore/filestore-idempotent-aio-journal.yaml index 53a93f67f44..58b5197d61b 100644 --- a/qa/suites/rados/objectstore/filestore-idempotent-aio-journal.yaml +++ b/qa/suites/rados/objectstore/filestore-idempotent-aio-journal.yaml @@ -1,5 +1,5 @@ roles: -- [mon.0, osd.0, osd.1, client.0] +- [mon.a, mgr.x, osd.0, osd.1, client.0] openstack: - volumes: # attached to each instance count: 2 diff --git a/qa/suites/rados/objectstore/filestore-idempotent.yaml b/qa/suites/rados/objectstore/filestore-idempotent.yaml index 64dd3a7352d..2d3f3c69212 100644 --- a/qa/suites/rados/objectstore/filestore-idempotent.yaml +++ b/qa/suites/rados/objectstore/filestore-idempotent.yaml @@ -1,5 +1,5 @@ roles: -- [mon.0, osd.0, osd.1, client.0] +- [mon.a, mgr.x, osd.0, osd.1, client.0] openstack: - volumes: # attached to each instance count: 2 diff --git a/qa/suites/rados/objectstore/fusestore.yaml b/qa/suites/rados/objectstore/fusestore.yaml index 78f61c2ad4c..1c34fcaea2d 100644 --- a/qa/suites/rados/objectstore/fusestore.yaml +++ b/qa/suites/rados/objectstore/fusestore.yaml @@ -1,5 +1,5 @@ roles: -- [mon.0, osd.0, osd.1, client.0] +- [mon.a, mgr.x, osd.0, osd.1, client.0] tasks: - install: - workunit: diff --git a/qa/suites/rados/objectstore/keyvaluedb.yaml b/qa/suites/rados/objectstore/keyvaluedb.yaml index 6921c5d572e..efff8d37930 100644 --- a/qa/suites/rados/objectstore/keyvaluedb.yaml +++ b/qa/suites/rados/objectstore/keyvaluedb.yaml @@ -1,5 +1,5 @@ roles: -- [mon.0, osd.0, osd.1, client.0] +- [mon.a, mgr.x, osd.0, osd.1, client.0] tasks: - install: - exec: diff --git a/qa/suites/rados/objectstore/objectcacher-stress.yaml b/qa/suites/rados/objectstore/objectcacher-stress.yaml index 386d12c098e..e407a391807 100644 --- a/qa/suites/rados/objectstore/objectcacher-stress.yaml +++ b/qa/suites/rados/objectstore/objectcacher-stress.yaml @@ -1,5 +1,5 @@ roles: -- [mon.0, osd.0, osd.1, client.0] +- [mon.a, mgr.x, osd.0, osd.1, client.0] openstack: - volumes: # attached to each instance count: 2 diff --git a/qa/suites/rados/objectstore/objectstore.yaml b/qa/suites/rados/objectstore/objectstore.yaml index 910e374ece9..b5442342b18 100644 --- a/qa/suites/rados/objectstore/objectstore.yaml +++ b/qa/suites/rados/objectstore/objectstore.yaml @@ -1,5 +1,5 @@ roles: -- [mon.0, osd.0, osd.1, client.0] +- [mon.a, mgr.x, osd.0, osd.1, client.0] openstack: - volumes: # attached to each instance count: 2 diff --git a/qa/suites/rados/singleton-nomsgr/all/cache-fs-trunc.yaml b/qa/suites/rados/singleton-nomsgr/all/cache-fs-trunc.yaml index 8cd36b8342e..5009ee61703 100644 --- a/qa/suites/rados/singleton-nomsgr/all/cache-fs-trunc.yaml +++ b/qa/suites/rados/singleton-nomsgr/all/cache-fs-trunc.yaml @@ -1,5 +1,5 @@ roles: -- [mon.0, mds.a, osd.0, osd.1, osd.2, client.0, client.1] +- [mon.a, mgr.x, mds.a, osd.0, osd.1, osd.2, client.0, client.1] tasks: - install: - ceph: diff --git a/qa/suites/rados/singleton-nomsgr/all/ceph-post-file.yaml b/qa/suites/rados/singleton-nomsgr/all/ceph-post-file.yaml index d0a4db067cf..8634362339f 100644 --- a/qa/suites/rados/singleton-nomsgr/all/ceph-post-file.yaml +++ b/qa/suites/rados/singleton-nomsgr/all/ceph-post-file.yaml @@ -1,5 +1,5 @@ roles: -- [mon.a, osd.0, osd.1, osd.2, client.0] +- [mon.a, mgr.x, osd.0, osd.1, osd.2, client.0] tasks: - install: - workunit: diff --git a/qa/suites/rados/singleton-nomsgr/all/export-after-evict.yaml b/qa/suites/rados/singleton-nomsgr/all/export-after-evict.yaml index 4a9de25e180..e0badd4d3af 100644 --- a/qa/suites/rados/singleton-nomsgr/all/export-after-evict.yaml +++ b/qa/suites/rados/singleton-nomsgr/all/export-after-evict.yaml @@ -1,5 +1,6 @@ roles: - - mon.a + - mgr.x - osd.0 - osd.1 - osd.2 diff --git a/qa/suites/rados/singleton-nomsgr/all/full-tiering.yaml b/qa/suites/rados/singleton-nomsgr/all/full-tiering.yaml index 6b46bc5bf3b..e21086a6be8 100644 --- a/qa/suites/rados/singleton-nomsgr/all/full-tiering.yaml +++ b/qa/suites/rados/singleton-nomsgr/all/full-tiering.yaml @@ -1,6 +1,6 @@ # verify #13098 fix roles: -- [mon.a, osd.0, osd.1, osd.2, client.0] +- [mon.a, mgr.x, osd.0, osd.1, osd.2, client.0] overrides: ceph: log-whitelist: diff --git a/qa/suites/rados/singleton-nomsgr/all/msgr.yaml b/qa/suites/rados/singleton-nomsgr/all/msgr.yaml index 1f9a695adab..98b509526e6 100644 --- a/qa/suites/rados/singleton-nomsgr/all/msgr.yaml +++ b/qa/suites/rados/singleton-nomsgr/all/msgr.yaml @@ -1,5 +1,5 @@ roles: -- [mon.0, osd.0, osd.1, client.0] +- [mon.a, mgr.x, osd.0, osd.1, client.0] tasks: - install: - exec: diff --git a/qa/suites/rados/singleton-nomsgr/all/multi-backfill-reject.yaml b/qa/suites/rados/singleton-nomsgr/all/multi-backfill-reject.yaml index 78f6e96c9db..0afc444a7c4 100644 --- a/qa/suites/rados/singleton-nomsgr/all/multi-backfill-reject.yaml +++ b/qa/suites/rados/singleton-nomsgr/all/multi-backfill-reject.yaml @@ -1,5 +1,6 @@ roles: - - mon.a + - mgr.x - osd.0 - osd.1 - osd.2 diff --git a/qa/suites/rados/singleton-nomsgr/all/valgrind-leaks.yaml b/qa/suites/rados/singleton-nomsgr/all/valgrind-leaks.yaml index 776a6a921dd..c05ad133be3 100644 --- a/qa/suites/rados/singleton-nomsgr/all/valgrind-leaks.yaml +++ b/qa/suites/rados/singleton-nomsgr/all/valgrind-leaks.yaml @@ -15,7 +15,7 @@ overrides: mon: [--tool=memcheck, --leak-check=full, --show-reachable=yes] osd: [--tool=memcheck] roles: -- [mon.0, osd.0, osd.1, client.0] +- [mon.a, mgr.x, osd.0, osd.1, client.0] tasks: - install: - ceph: diff --git a/qa/suites/rados/singleton/all/admin-socket.yaml b/qa/suites/rados/singleton/all/admin-socket.yaml index 5a61e21e90c..afbe9702e58 100644 --- a/qa/suites/rados/singleton/all/admin-socket.yaml +++ b/qa/suites/rados/singleton/all/admin-socket.yaml @@ -1,5 +1,6 @@ roles: - - mon.a + - mgr.x - osd.0 - osd.1 - client.a diff --git a/qa/suites/rados/singleton/all/cephtool.yaml b/qa/suites/rados/singleton/all/cephtool.yaml index b0a886339a8..55f8822f2b1 100644 --- a/qa/suites/rados/singleton/all/cephtool.yaml +++ b/qa/suites/rados/singleton/all/cephtool.yaml @@ -2,6 +2,7 @@ roles: - - mon.a - mon.b - mon.c + - mgr.x - osd.0 - osd.1 - osd.2 diff --git a/qa/suites/rados/singleton/all/divergent_priors.yaml b/qa/suites/rados/singleton/all/divergent_priors.yaml index 3103703edf4..bb7c2b57f50 100644 --- a/qa/suites/rados/singleton/all/divergent_priors.yaml +++ b/qa/suites/rados/singleton/all/divergent_priors.yaml @@ -1,5 +1,6 @@ roles: -- - mon.0 +- - mon.a + - mgr.x - osd.0 - osd.1 - osd.2 diff --git a/qa/suites/rados/singleton/all/divergent_priors2.yaml b/qa/suites/rados/singleton/all/divergent_priors2.yaml index dd00b87003a..ab749f1b516 100644 --- a/qa/suites/rados/singleton/all/divergent_priors2.yaml +++ b/qa/suites/rados/singleton/all/divergent_priors2.yaml @@ -1,5 +1,6 @@ roles: -- - mon.0 +- - mon.a + - mgr.x - osd.0 - osd.1 - osd.2 diff --git a/qa/suites/rados/singleton/all/dump-stuck.yaml b/qa/suites/rados/singleton/all/dump-stuck.yaml index f153de62954..7d3b443021b 100644 --- a/qa/suites/rados/singleton/all/dump-stuck.yaml +++ b/qa/suites/rados/singleton/all/dump-stuck.yaml @@ -1,5 +1,6 @@ roles: - - mon.a + - mgr.x - osd.0 - osd.1 openstack: diff --git a/qa/suites/rados/singleton/all/ec-lost-unfound.yaml b/qa/suites/rados/singleton/all/ec-lost-unfound.yaml index 7c4b5f0e039..6ceefe1222e 100644 --- a/qa/suites/rados/singleton/all/ec-lost-unfound.yaml +++ b/qa/suites/rados/singleton/all/ec-lost-unfound.yaml @@ -2,6 +2,7 @@ roles: - - mon.a - mon.b - mon.c + - mgr.x - osd.0 - osd.1 - osd.2 diff --git a/qa/suites/rados/singleton/all/lost-unfound-delete.yaml b/qa/suites/rados/singleton/all/lost-unfound-delete.yaml index fbc24e0d17e..15f4710bd9b 100644 --- a/qa/suites/rados/singleton/all/lost-unfound-delete.yaml +++ b/qa/suites/rados/singleton/all/lost-unfound-delete.yaml @@ -2,6 +2,7 @@ roles: - - mon.a - mon.b - mon.c + - mgr.x - osd.0 - osd.1 - osd.2 diff --git a/qa/suites/rados/singleton/all/lost-unfound.yaml b/qa/suites/rados/singleton/all/lost-unfound.yaml index 31649dbb0da..3f22ba3c013 100644 --- a/qa/suites/rados/singleton/all/lost-unfound.yaml +++ b/qa/suites/rados/singleton/all/lost-unfound.yaml @@ -2,6 +2,7 @@ roles: - - mon.a - mon.b - mon.c + - mgr.x - osd.0 - osd.1 - osd.2 diff --git a/qa/suites/rados/singleton/all/mon-config-keys.yaml b/qa/suites/rados/singleton/all/mon-config-keys.yaml index b905691a9f4..7bb4f65030e 100644 --- a/qa/suites/rados/singleton/all/mon-config-keys.yaml +++ b/qa/suites/rados/singleton/all/mon-config-keys.yaml @@ -1,7 +1,8 @@ roles: -- - mon.0 - - mon.1 - - mon.2 +- - mon.a + - mon.b + - mon.c + - mgr.x - osd.0 - osd.1 - osd.2 diff --git a/qa/suites/rados/singleton/all/mon-seesaw.yaml b/qa/suites/rados/singleton/all/mon-seesaw.yaml index a5cd937791b..ccd980fded7 100644 --- a/qa/suites/rados/singleton/all/mon-seesaw.yaml +++ b/qa/suites/rados/singleton/all/mon-seesaw.yaml @@ -1,5 +1,6 @@ roles: - - mon.a + - mgr.x - osd.0 - osd.1 - osd.2 diff --git a/qa/suites/rados/singleton/all/mon-thrasher.yaml b/qa/suites/rados/singleton/all/mon-thrasher.yaml index 5cadddabe6b..1b4622998f9 100644 --- a/qa/suites/rados/singleton/all/mon-thrasher.yaml +++ b/qa/suites/rados/singleton/all/mon-thrasher.yaml @@ -2,6 +2,7 @@ roles: - - mon.a - mon.b - mon.c + - mgr.x - osd.0 - osd.1 - client.0 diff --git a/qa/suites/rados/singleton/all/osd-backfill.yaml b/qa/suites/rados/singleton/all/osd-backfill.yaml index c9263db00ec..f84a0df8d66 100644 --- a/qa/suites/rados/singleton/all/osd-backfill.yaml +++ b/qa/suites/rados/singleton/all/osd-backfill.yaml @@ -2,6 +2,7 @@ roles: - - mon.a - mon.b - mon.c + - mgr.x - osd.0 - osd.1 - osd.2 diff --git a/qa/suites/rados/singleton/all/osd-recovery-incomplete.yaml b/qa/suites/rados/singleton/all/osd-recovery-incomplete.yaml index c7f053c40f3..773cb2480a6 100644 --- a/qa/suites/rados/singleton/all/osd-recovery-incomplete.yaml +++ b/qa/suites/rados/singleton/all/osd-recovery-incomplete.yaml @@ -2,6 +2,7 @@ roles: - - mon.a - mon.b - mon.c + - mgr.x - osd.0 - osd.1 - osd.2 diff --git a/qa/suites/rados/singleton/all/osd-recovery.yaml b/qa/suites/rados/singleton/all/osd-recovery.yaml index c86dc872067..214d7f20cc7 100644 --- a/qa/suites/rados/singleton/all/osd-recovery.yaml +++ b/qa/suites/rados/singleton/all/osd-recovery.yaml @@ -2,6 +2,7 @@ roles: - - mon.a - mon.b - mon.c + - mgr.x - osd.0 - osd.1 - osd.2 diff --git a/qa/suites/rados/singleton/all/peer.yaml b/qa/suites/rados/singleton/all/peer.yaml index 9eaaa946e83..6e22b44563e 100644 --- a/qa/suites/rados/singleton/all/peer.yaml +++ b/qa/suites/rados/singleton/all/peer.yaml @@ -1,7 +1,8 @@ roles: -- - mon.0 - - mon.1 - - mon.2 +- - mon.a + - mon.b + - mon.c + - mgr.x - osd.0 - osd.1 - osd.2 diff --git a/qa/suites/rados/singleton/all/pg-removal-interruption.yaml b/qa/suites/rados/singleton/all/pg-removal-interruption.yaml index 552ed26c28b..f7e61c96263 100644 --- a/qa/suites/rados/singleton/all/pg-removal-interruption.yaml +++ b/qa/suites/rados/singleton/all/pg-removal-interruption.yaml @@ -1,5 +1,6 @@ roles: - - mon.a + - mgr.x - osd.0 - osd.1 - osd.2 diff --git a/qa/suites/rados/singleton/all/radostool.yaml b/qa/suites/rados/singleton/all/radostool.yaml index 1df40794970..8bc9dbdcd0f 100644 --- a/qa/suites/rados/singleton/all/radostool.yaml +++ b/qa/suites/rados/singleton/all/radostool.yaml @@ -1,5 +1,6 @@ roles: - - mon.a + - mgr.x - osd.0 - osd.1 - osd.2 diff --git a/qa/suites/rados/singleton/all/rebuild-mondb.yaml b/qa/suites/rados/singleton/all/rebuild-mondb.yaml index 5ce1cd0a893..c3be13ae6a2 100644 --- a/qa/suites/rados/singleton/all/rebuild-mondb.yaml +++ b/qa/suites/rados/singleton/all/rebuild-mondb.yaml @@ -1,7 +1,8 @@ roles: -- - mon.0 - - mon.1 - - mon.2 +- - mon.a + - mon.b + - mon.c + - mgr.x - osd.0 - osd.1 - osd.2 diff --git a/qa/suites/rados/singleton/all/reg11184.yaml b/qa/suites/rados/singleton/all/reg11184.yaml index 8bc502a1456..5521a27f04f 100644 --- a/qa/suites/rados/singleton/all/reg11184.yaml +++ b/qa/suites/rados/singleton/all/reg11184.yaml @@ -1,5 +1,6 @@ roles: -- - mon.0 +- - mon.a + - mgr.x - osd.0 - osd.1 - osd.2 diff --git a/qa/suites/rados/singleton/all/resolve_stuck_peering.yaml b/qa/suites/rados/singleton/all/resolve_stuck_peering.yaml index 278ff1aa5ba..c64593212a5 100644 --- a/qa/suites/rados/singleton/all/resolve_stuck_peering.yaml +++ b/qa/suites/rados/singleton/all/resolve_stuck_peering.yaml @@ -1,5 +1,5 @@ roles: -- [mon.0] +- [mon.a, mgr.x] - [osd.0, osd.1, osd.2, client.0] tasks: diff --git a/qa/suites/rados/singleton/all/rest-api.yaml b/qa/suites/rados/singleton/all/rest-api.yaml index ce39a6b2cd4..cbd90e4097d 100644 --- a/qa/suites/rados/singleton/all/rest-api.yaml +++ b/qa/suites/rados/singleton/all/rest-api.yaml @@ -1,7 +1,8 @@ roles: -- - mon.0 - - mon.1 - - mon.2 +- - mon.a + - mon.b + - mon.c + - mgr.x - osd.0 - osd.1 - osd.2 diff --git a/qa/suites/rados/singleton/all/test_envlibrados_for_rocksdb.yaml b/qa/suites/rados/singleton/all/test_envlibrados_for_rocksdb.yaml index 65c875a9a19..0835d22bfa6 100644 --- a/qa/suites/rados/singleton/all/test_envlibrados_for_rocksdb.yaml +++ b/qa/suites/rados/singleton/all/test_envlibrados_for_rocksdb.yaml @@ -6,7 +6,7 @@ overrides: osd max object name len: 460 osd max object namespace len: 64 roles: -- [mon.a, osd.0, osd.1, osd.2, client.0] +- [mon.a, mgr.x, osd.0, osd.1, osd.2, client.0] tasks: - install: - ceph: diff --git a/qa/suites/rados/singleton/all/thrash-rados.yaml b/qa/suites/rados/singleton/all/thrash-rados.yaml index af963106a2a..49e3e8799c1 100644 --- a/qa/suites/rados/singleton/all/thrash-rados.yaml +++ b/qa/suites/rados/singleton/all/thrash-rados.yaml @@ -1,5 +1,6 @@ roles: - - mon.a + - mgr.x - osd.0 - osd.1 - osd.2 diff --git a/qa/suites/rados/singleton/all/thrash_cache_writeback_proxy_none.yaml b/qa/suites/rados/singleton/all/thrash_cache_writeback_proxy_none.yaml index 02c228d7793..1875da409a2 100644 --- a/qa/suites/rados/singleton/all/thrash_cache_writeback_proxy_none.yaml +++ b/qa/suites/rados/singleton/all/thrash_cache_writeback_proxy_none.yaml @@ -1,5 +1,6 @@ roles: - - mon.a + - mgr.x - osd.0 - osd.1 - osd.2 diff --git a/qa/suites/rados/singleton/all/watch-notify-same-primary.yaml b/qa/suites/rados/singleton/all/watch-notify-same-primary.yaml index 48900f392df..ad1fd17d5f5 100644 --- a/qa/suites/rados/singleton/all/watch-notify-same-primary.yaml +++ b/qa/suites/rados/singleton/all/watch-notify-same-primary.yaml @@ -1,7 +1,8 @@ roles: -- - mon.0 - - mon.1 - - mon.2 +- - mon.a + - mon.b + - mon.c + - mgr.x - osd.0 - osd.1 - osd.2 diff --git a/qa/suites/rados/thrash-erasure-code-big/cluster/12-osds.yaml b/qa/suites/rados/thrash-erasure-code-big/cluster/12-osds.yaml index 11317e6ee74..db86583f22c 100644 --- a/qa/suites/rados/thrash-erasure-code-big/cluster/12-osds.yaml +++ b/qa/suites/rados/thrash-erasure-code-big/cluster/12-osds.yaml @@ -2,4 +2,4 @@ roles: - [osd.0, osd.1, osd.2, client.0, mon.a] - [osd.3, osd.4, osd.5, mon.b] - [osd.6, osd.7, osd.8, mon.c] -- [osd.9, osd.10, osd.11] +- [osd.9, osd.10, osd.11, mgr.x] diff --git a/qa/suites/rados/upgrade/jewel-x-singleton/0-cluster/start.yaml b/qa/suites/rados/upgrade/jewel-x-singleton/0-cluster/start.yaml index fd200d752a2..2872225e993 100644 --- a/qa/suites/rados/upgrade/jewel-x-singleton/0-cluster/start.yaml +++ b/qa/suites/rados/upgrade/jewel-x-singleton/0-cluster/start.yaml @@ -13,6 +13,8 @@ roles: - - mon.a - mon.b - mon.c + - mgr.x + - mgr.y - mds.a - osd.0 - osd.1 diff --git a/qa/suites/rados/upgrade/jewel-x-singleton/1-jewel-install/jewel.yaml b/qa/suites/rados/upgrade/jewel-x-singleton/1-jewel-install/jewel.yaml index 87bc11f451c..a617f91d19a 100644 --- a/qa/suites/rados/upgrade/jewel-x-singleton/1-jewel-install/jewel.yaml +++ b/qa/suites/rados/upgrade/jewel-x-singleton/1-jewel-install/jewel.yaml @@ -6,4 +6,5 @@ tasks: exclude_packages: ['ceph-mgr','libcephfs2','libcephfs-devel','libcephfs-dev'] - print: "**** done install jewel" - ceph: + skip_mgr_daemons: true - print: "**** done ceph" diff --git a/qa/suites/rbd/mirror/cluster/2-node.yaml b/qa/suites/rbd/mirror/cluster/2-node.yaml index e88b12b1381..8eb30c45e6f 100644 --- a/qa/suites/rbd/mirror/cluster/2-node.yaml +++ b/qa/suites/rbd/mirror/cluster/2-node.yaml @@ -3,6 +3,7 @@ meta: roles: - - cluster1.mon.a - cluster1.mon.b + - cluster1.mgr.x - cluster1.osd.0 - cluster1.osd.1 - cluster1.osd.2 @@ -12,6 +13,7 @@ roles: - - cluster1.mon.c - cluster2.mon.a - cluster2.mon.b + - cluster2.mgr.x - cluster2.osd.0 - cluster2.osd.1 - cluster2.osd.2 diff --git a/qa/suites/rbd/nbd/cluster/fixed-3.yaml b/qa/suites/rbd/nbd/cluster/fixed-3.yaml index 455de4f9831..182589152d2 100644 --- a/qa/suites/rbd/nbd/cluster/fixed-3.yaml +++ b/qa/suites/rbd/nbd/cluster/fixed-3.yaml @@ -1,4 +1,4 @@ roles: - [mon.a, mon.c, osd.0, osd.1, osd.2] -- [mon.b, osd.3, osd.4, osd.5] +- [mon.b, mgr.x, osd.3, osd.4, osd.5] - [client.0] diff --git a/qa/suites/rbd/openstack/clusters/fixed-2.yaml b/qa/suites/rbd/openstack/clusters/fixed-2.yaml index 93be2fce61a..473c205411f 100644 --- a/qa/suites/rbd/openstack/clusters/fixed-2.yaml +++ b/qa/suites/rbd/openstack/clusters/fixed-2.yaml @@ -5,5 +5,5 @@ overrides: osd pool default size: 2 osd crush chooseleaf type: 0 roles: -- [mon.a, osd.0, osd.1, osd.2] +- [mon.a, mgr.x, osd.0, osd.1, osd.2] - [client.0] diff --git a/qa/suites/rbd/singleton/all/admin_socket.yaml b/qa/suites/rbd/singleton/all/admin_socket.yaml index 6394cc7ef00..22dbd8c0334 100644 --- a/qa/suites/rbd/singleton/all/admin_socket.yaml +++ b/qa/suites/rbd/singleton/all/admin_socket.yaml @@ -1,5 +1,5 @@ roles: -- [mon.a, osd.0, osd.1, client.0] +- [mon.a, mgr.x, osd.0, osd.1, client.0] tasks: - install: - ceph: diff --git a/qa/suites/rbd/singleton/all/formatted-output.yaml b/qa/suites/rbd/singleton/all/formatted-output.yaml index 8bed75c1889..f6a19910c12 100644 --- a/qa/suites/rbd/singleton/all/formatted-output.yaml +++ b/qa/suites/rbd/singleton/all/formatted-output.yaml @@ -1,5 +1,5 @@ roles: -- [mon.a, osd.0, osd.1, client.0] +- [mon.a, mgr.x, osd.0, osd.1, client.0] tasks: - install: - ceph: diff --git a/qa/suites/rbd/singleton/all/merge_diff.yaml b/qa/suites/rbd/singleton/all/merge_diff.yaml index 76a999db8f3..31b269d6309 100644 --- a/qa/suites/rbd/singleton/all/merge_diff.yaml +++ b/qa/suites/rbd/singleton/all/merge_diff.yaml @@ -1,5 +1,5 @@ roles: -- [mon.a, osd.0, osd.1, client.0] +- [mon.a, mgr.x, osd.0, osd.1, client.0] tasks: - install: - ceph: diff --git a/qa/suites/rbd/singleton/all/permissions.yaml b/qa/suites/rbd/singleton/all/permissions.yaml index bb7e12b7968..c00a5c9b8fb 100644 --- a/qa/suites/rbd/singleton/all/permissions.yaml +++ b/qa/suites/rbd/singleton/all/permissions.yaml @@ -1,5 +1,5 @@ roles: -- [mon.a, osd.0, osd.1, client.0] +- [mon.a, mgr.x, osd.0, osd.1, client.0] tasks: - install: - ceph: diff --git a/qa/suites/rbd/singleton/all/qemu-iotests-no-cache.yaml b/qa/suites/rbd/singleton/all/qemu-iotests-no-cache.yaml index bf67dbab3d1..bfb20390418 100644 --- a/qa/suites/rbd/singleton/all/qemu-iotests-no-cache.yaml +++ b/qa/suites/rbd/singleton/all/qemu-iotests-no-cache.yaml @@ -1,6 +1,6 @@ exclude_arch: armv7l roles: -- [mon.a, osd.0, osd.1, client.0] +- [mon.a, mgr.x, osd.0, osd.1, client.0] tasks: - install: - ceph: diff --git a/qa/suites/rbd/singleton/all/qemu-iotests-writeback.yaml b/qa/suites/rbd/singleton/all/qemu-iotests-writeback.yaml index 823e67202c3..bf1b4befe03 100644 --- a/qa/suites/rbd/singleton/all/qemu-iotests-writeback.yaml +++ b/qa/suites/rbd/singleton/all/qemu-iotests-writeback.yaml @@ -1,6 +1,6 @@ exclude_arch: armv7l roles: -- [mon.a, osd.0, osd.1, client.0] +- [mon.a, mgr.x, osd.0, osd.1, client.0] tasks: - install: - ceph: diff --git a/qa/suites/rbd/singleton/all/qemu-iotests-writethrough.yaml b/qa/suites/rbd/singleton/all/qemu-iotests-writethrough.yaml index 64667d9df29..908a6780403 100644 --- a/qa/suites/rbd/singleton/all/qemu-iotests-writethrough.yaml +++ b/qa/suites/rbd/singleton/all/qemu-iotests-writethrough.yaml @@ -1,6 +1,6 @@ exclude_arch: armv7l roles: -- [mon.a, osd.0, osd.1, client.0] +- [mon.a, mgr.x, osd.0, osd.1, client.0] tasks: - install: - ceph: diff --git a/qa/suites/rbd/singleton/all/rbd-vs-unmanaged-snaps.yaml b/qa/suites/rbd/singleton/all/rbd-vs-unmanaged-snaps.yaml index 434575d382d..f14bd7431b2 100644 --- a/qa/suites/rbd/singleton/all/rbd-vs-unmanaged-snaps.yaml +++ b/qa/suites/rbd/singleton/all/rbd-vs-unmanaged-snaps.yaml @@ -1,5 +1,5 @@ roles: -- [mon.a, osd.0, osd.1, client.0] +- [mon.a, mgr.x, osd.0, osd.1, client.0] tasks: - install: - ceph: diff --git a/qa/suites/rbd/singleton/all/rbd_mirror.yaml b/qa/suites/rbd/singleton/all/rbd_mirror.yaml index 9caf64464ad..21624164beb 100644 --- a/qa/suites/rbd/singleton/all/rbd_mirror.yaml +++ b/qa/suites/rbd/singleton/all/rbd_mirror.yaml @@ -1,5 +1,5 @@ roles: -- [mon.a, osd.0, osd.1, client.0] +- [mon.a, mgr.x, osd.0, osd.1, client.0] tasks: - install: - ceph: diff --git a/qa/suites/rbd/singleton/all/read-flags-no-cache.yaml b/qa/suites/rbd/singleton/all/read-flags-no-cache.yaml index a28875cc563..cf602cbb154 100644 --- a/qa/suites/rbd/singleton/all/read-flags-no-cache.yaml +++ b/qa/suites/rbd/singleton/all/read-flags-no-cache.yaml @@ -1,5 +1,5 @@ roles: -- [mon.a, osd.0, osd.1, client.0] +- [mon.a, mgr.x, osd.0, osd.1, client.0] tasks: - install: - ceph: diff --git a/qa/suites/rbd/singleton/all/read-flags-writeback.yaml b/qa/suites/rbd/singleton/all/read-flags-writeback.yaml index 4f2bd19a92e..e763bcc34e9 100644 --- a/qa/suites/rbd/singleton/all/read-flags-writeback.yaml +++ b/qa/suites/rbd/singleton/all/read-flags-writeback.yaml @@ -1,5 +1,5 @@ roles: -- [mon.a, osd.0, osd.1, client.0] +- [mon.a, mgr.x, osd.0, osd.1, client.0] tasks: - install: - ceph: diff --git a/qa/suites/rbd/singleton/all/read-flags-writethrough.yaml b/qa/suites/rbd/singleton/all/read-flags-writethrough.yaml index 543a84958de..fc499d49506 100644 --- a/qa/suites/rbd/singleton/all/read-flags-writethrough.yaml +++ b/qa/suites/rbd/singleton/all/read-flags-writethrough.yaml @@ -1,5 +1,5 @@ roles: -- [mon.a, osd.0, osd.1, client.0] +- [mon.a, mgr.x, osd.0, osd.1, client.0] tasks: - install: - ceph: diff --git a/qa/suites/rbd/singleton/all/verify_pool.yaml b/qa/suites/rbd/singleton/all/verify_pool.yaml index 92b110277ed..5ab06f74944 100644 --- a/qa/suites/rbd/singleton/all/verify_pool.yaml +++ b/qa/suites/rbd/singleton/all/verify_pool.yaml @@ -1,5 +1,5 @@ roles: -- [mon.a, osd.0, osd.1, client.0] +- [mon.a, mgr.x, osd.0, osd.1, client.0] tasks: - install: - ceph: diff --git a/qa/suites/rest/basic/tasks/rest_test.yaml b/qa/suites/rest/basic/tasks/rest_test.yaml index 1adc95390ae..bf7f5aa3a5e 100644 --- a/qa/suites/rest/basic/tasks/rest_test.yaml +++ b/qa/suites/rest/basic/tasks/rest_test.yaml @@ -1,5 +1,6 @@ roles: - - mon.a + - mgr.x - mds.a - osd.0 - osd.1 diff --git a/qa/suites/rgw/singleton/all/radosgw-admin-data-sync.yaml b/qa/suites/rgw/singleton/all/radosgw-admin-data-sync.yaml index 4885bf78203..03d7c612c4a 100644 --- a/qa/suites/rgw/singleton/all/radosgw-admin-data-sync.yaml +++ b/qa/suites/rgw/singleton/all/radosgw-admin-data-sync.yaml @@ -1,6 +1,6 @@ roles: - [mon.a, osd.0, client.0] -- [osd.1, osd.2, osd.3, client.1] +- [mgr.x, osd.1, osd.2, osd.3, client.1] openstack: - volumes: # attached to each instance count: 3 diff --git a/qa/suites/rgw/singleton/all/radosgw-admin-multi-region.yaml b/qa/suites/rgw/singleton/all/radosgw-admin-multi-region.yaml index 93b75517898..d28dec9e7a1 100644 --- a/qa/suites/rgw/singleton/all/radosgw-admin-multi-region.yaml +++ b/qa/suites/rgw/singleton/all/radosgw-admin-multi-region.yaml @@ -1,5 +1,5 @@ roles: -- [mon.a, osd.0, osd.1, osd.2, client.0] +- [mon.a, mgr.x, osd.0, osd.1, osd.2, client.0] - [mon.b, mon.c, osd.3, osd.4, osd.5, client.1] openstack: - volumes: # attached to each instance diff --git a/qa/suites/rgw/singleton/all/radosgw-admin.yaml b/qa/suites/rgw/singleton/all/radosgw-admin.yaml index ccdafad2069..aada29b5b33 100644 --- a/qa/suites/rgw/singleton/all/radosgw-admin.yaml +++ b/qa/suites/rgw/singleton/all/radosgw-admin.yaml @@ -1,6 +1,6 @@ roles: - [mon.a, osd.0] -- [client.0, osd.1, osd.2, osd.3] +- [mgr.x, client.0, osd.1, osd.2, osd.3] openstack: - volumes: # attached to each instance count: 3 diff --git a/qa/suites/rgw/singleton/all/radosgw-convert-to-region.yaml b/qa/suites/rgw/singleton/all/radosgw-convert-to-region.yaml index 5da11351a20..8e81240e25a 100644 --- a/qa/suites/rgw/singleton/all/radosgw-convert-to-region.yaml +++ b/qa/suites/rgw/singleton/all/radosgw-convert-to-region.yaml @@ -15,7 +15,7 @@ overrides: size: 2000 stddev: 500 roles: -- [mon.a, osd.0, osd.1, osd.2, client.0] +- [mon.a, mgr.x, osd.0, osd.1, osd.2, client.0] - [mon.b, mon.c, osd.3, osd.4, osd.5, client.1] openstack: - volumes: # attached to each instance diff --git a/qa/suites/samba/clusters/samba-basic.yaml b/qa/suites/samba/clusters/samba-basic.yaml index 86c71b81daf..af432f61038 100644 --- a/qa/suites/samba/clusters/samba-basic.yaml +++ b/qa/suites/samba/clusters/samba-basic.yaml @@ -1,5 +1,5 @@ roles: -- [mon.a, mon.b, mon.c, mds.a, osd.0, osd.1] +- [mon.a, mon.b, mon.c, mgr.x, mds.a, osd.0, osd.1] - [samba.0, client.0, client.1] openstack: - volumes: # attached to each instance diff --git a/qa/suites/smoke/systemd/clusters/fixed-4.yaml b/qa/suites/smoke/systemd/clusters/fixed-4.yaml index 8e5a9874205..43b4de755cb 100644 --- a/qa/suites/smoke/systemd/clusters/fixed-4.yaml +++ b/qa/suites/smoke/systemd/clusters/fixed-4.yaml @@ -1,5 +1,5 @@ roles: -- [mon.a, osd.0] +- [mon.a, mgr.x, osd.0] - [osd.1, osd.2] - [mds.a, osd.3] - [mon.b, client.0] diff --git a/qa/suites/stress/thrash/clusters/16-osd.yaml b/qa/suites/stress/thrash/clusters/16-osd.yaml index 373dd4052c3..76232339bcf 100644 --- a/qa/suites/stress/thrash/clusters/16-osd.yaml +++ b/qa/suites/stress/thrash/clusters/16-osd.yaml @@ -1,7 +1,7 @@ roles: -- [mon.0, mds.a, osd.0] -- [mon.1, osd.1] -- [mon.2, osd.2] +- [mon.a, mds.a, osd.0] +- [mon.b, mgr.x, osd.1] +- [mon.c, mgr.y, osd.2] - [osd.3] - [osd.4] - [osd.5] diff --git a/qa/suites/stress/thrash/clusters/3-osd-1-machine.yaml b/qa/suites/stress/thrash/clusters/3-osd-1-machine.yaml index d8ff594b95d..8c3556ae926 100644 --- a/qa/suites/stress/thrash/clusters/3-osd-1-machine.yaml +++ b/qa/suites/stress/thrash/clusters/3-osd-1-machine.yaml @@ -1,3 +1,3 @@ roles: -- [mon.0, mds.a, osd.0, osd.1, osd.2] -- [mon.1, mon.2, client.0] +- [mon.a, mgr.x, mds.a, osd.0, osd.1, osd.2] +- [mon.b, mon.c, client.0] diff --git a/qa/suites/stress/thrash/clusters/8-osd.yaml b/qa/suites/stress/thrash/clusters/8-osd.yaml index 3b131054e95..9f51c6bad78 100644 --- a/qa/suites/stress/thrash/clusters/8-osd.yaml +++ b/qa/suites/stress/thrash/clusters/8-osd.yaml @@ -1,7 +1,7 @@ roles: -- [mon.0, mds.a, osd.0] -- [mon.1, osd.1] -- [mon.2, osd.2] +- [mon.a, mds.a, osd.0] +- [mon.b, mgr.x, osd.1] +- [mon.c, osd.2] - [osd.3] - [osd.4] - [osd.5] diff --git a/qa/suites/teuthology/buildpackages/tasks/branch.yaml b/qa/suites/teuthology/buildpackages/tasks/branch.yaml index b93c5d07c4a..1dad96f3a3a 100644 --- a/qa/suites/teuthology/buildpackages/tasks/branch.yaml +++ b/qa/suites/teuthology/buildpackages/tasks/branch.yaml @@ -1,5 +1,5 @@ roles: - - [mon.0, client.0] + - [mon.a, mgr.x, client.0] tasks: - install: # branch has precedence over sha1 diff --git a/qa/suites/teuthology/buildpackages/tasks/tag.yaml b/qa/suites/teuthology/buildpackages/tasks/tag.yaml index 126749c9bb0..2bfb8a995db 100644 --- a/qa/suites/teuthology/buildpackages/tasks/tag.yaml +++ b/qa/suites/teuthology/buildpackages/tasks/tag.yaml @@ -1,5 +1,5 @@ roles: - - [mon.0, client.0] + - [mon.a, mgr.x, client.0] tasks: - install: # tag has precedence over branch and sha1 diff --git a/qa/suites/teuthology/ceph/clusters/single.yaml b/qa/suites/teuthology/ceph/clusters/single.yaml index beba3972ce9..0c6a40d0bde 100644 --- a/qa/suites/teuthology/ceph/clusters/single.yaml +++ b/qa/suites/teuthology/ceph/clusters/single.yaml @@ -1,2 +1,2 @@ roles: - - [mon.0, client.0] + - [mon.a, mgr.x, client.0] diff --git a/qa/suites/teuthology/multi-cluster/all/ceph.yaml b/qa/suites/teuthology/multi-cluster/all/ceph.yaml index 6a93b9a9a48..4659ef3d1f4 100644 --- a/qa/suites/teuthology/multi-cluster/all/ceph.yaml +++ b/qa/suites/teuthology/multi-cluster/all/ceph.yaml @@ -1,11 +1,13 @@ roles: - - ceph.mon.a - ceph.mon.b + - ceph.mgr.x - backup.osd.0 - backup.osd.1 - backup.osd.2 - backup.client.0 - - backup.mon.a + - backup.mgr.x - ceph.osd.0 - ceph.osd.1 - ceph.osd.2 diff --git a/qa/suites/teuthology/multi-cluster/all/thrashosds.yaml b/qa/suites/teuthology/multi-cluster/all/thrashosds.yaml index c25a2141c19..52002f57f92 100644 --- a/qa/suites/teuthology/multi-cluster/all/thrashosds.yaml +++ b/qa/suites/teuthology/multi-cluster/all/thrashosds.yaml @@ -1,6 +1,7 @@ roles: - - backup.mon.a - backup.mon.b + - backup.mgr.x - backup.osd.0 - backup.osd.1 - backup.osd.2 diff --git a/qa/suites/teuthology/multi-cluster/all/upgrade.yaml b/qa/suites/teuthology/multi-cluster/all/upgrade.yaml index c8a742ffefc..42cd93b2c17 100644 --- a/qa/suites/teuthology/multi-cluster/all/upgrade.yaml +++ b/qa/suites/teuthology/multi-cluster/all/upgrade.yaml @@ -8,11 +8,13 @@ overrides: roles: - - ceph.mon.a - ceph.mon.b + - ceph.mgr.x - backup.osd.0 - backup.osd.1 - backup.osd.2 - backup.client.0 - - backup.mon.a + - backup.mgr.x - ceph.osd.0 - ceph.osd.1 - ceph.osd.2 diff --git a/qa/suites/teuthology/multi-cluster/all/workunit.yaml b/qa/suites/teuthology/multi-cluster/all/workunit.yaml index 4c32d9af6a0..b1288e38efb 100644 --- a/qa/suites/teuthology/multi-cluster/all/workunit.yaml +++ b/qa/suites/teuthology/multi-cluster/all/workunit.yaml @@ -1,11 +1,13 @@ roles: - - backup.mon.a + - backup.mgr.x - osd.0 - osd.1 - osd.2 - client.0 - backup.client.0 - - mon.a + - mgr.x - backup.osd.0 - backup.osd.1 - backup.osd.2 diff --git a/qa/suites/teuthology/no-ceph/clusters/single.yaml b/qa/suites/teuthology/no-ceph/clusters/single.yaml index beba3972ce9..0c6a40d0bde 100644 --- a/qa/suites/teuthology/no-ceph/clusters/single.yaml +++ b/qa/suites/teuthology/no-ceph/clusters/single.yaml @@ -1,2 +1,2 @@ roles: - - [mon.0, client.0] + - [mon.a, mgr.x, client.0] diff --git a/qa/suites/teuthology/rgw/tasks/s3tests-civetweb.yaml b/qa/suites/teuthology/rgw/tasks/s3tests-civetweb.yaml index e817dad2373..01580e38f06 100644 --- a/qa/suites/teuthology/rgw/tasks/s3tests-civetweb.yaml +++ b/qa/suites/teuthology/rgw/tasks/s3tests-civetweb.yaml @@ -1,7 +1,7 @@ # this runs s3tests against rgw, using civetweb roles: - [mon.a, mon.c, osd.0, osd.1, osd.2, client.0] -- [mon.b, osd.3, osd.4, osd.5, client.1] +- [mon.b, mgr.x, osd.3, osd.4, osd.5, client.1] tasks: - install: diff --git a/qa/suites/teuthology/rgw/tasks/s3tests-fastcgi.yaml b/qa/suites/teuthology/rgw/tasks/s3tests-fastcgi.yaml index 1f34a712131..d8a5050385c 100644 --- a/qa/suites/teuthology/rgw/tasks/s3tests-fastcgi.yaml +++ b/qa/suites/teuthology/rgw/tasks/s3tests-fastcgi.yaml @@ -1,7 +1,7 @@ # this runs s3tests against rgw, using mod_fastcgi roles: - [mon.a, mon.c, osd.0, osd.1, osd.2, client.0] -- [mon.b, osd.3, osd.4, osd.5, client.1] +- [mon.b, mgr.x, osd.3, osd.4, osd.5, client.1] tasks: - install: diff --git a/qa/suites/teuthology/rgw/tasks/s3tests-fcgi.yaml b/qa/suites/teuthology/rgw/tasks/s3tests-fcgi.yaml index 065f6e2b973..1def7b042bd 100644 --- a/qa/suites/teuthology/rgw/tasks/s3tests-fcgi.yaml +++ b/qa/suites/teuthology/rgw/tasks/s3tests-fcgi.yaml @@ -2,7 +2,7 @@ # the choice between uds or tcp with mod_proxy_fcgi depends on the distro roles: - [mon.a, mon.c, osd.0, osd.1, osd.2, client.0] -- [mon.b, osd.3, osd.4, osd.5, client.1] +- [mon.b, mgr.x, osd.3, osd.4, osd.5, client.1] tasks: - install: diff --git a/qa/suites/tgt/basic/clusters/fixed-3.yaml b/qa/suites/tgt/basic/clusters/fixed-3.yaml index 0038432afa7..5e23c9e4f44 100644 --- a/qa/suites/tgt/basic/clusters/fixed-3.yaml +++ b/qa/suites/tgt/basic/clusters/fixed-3.yaml @@ -1,4 +1,4 @@ roles: - [mon.a, mon.c, osd.0, osd.1, osd.2] -- [mon.b, mds.a, osd.3, osd.4, osd.5] +- [mon.b, mgr.x, mds.a, osd.3, osd.4, osd.5] - [client.0] diff --git a/qa/suites/upgrade/jewel-x/parallel/1-jewel-install/jewel.yaml b/qa/suites/upgrade/jewel-x/parallel/1-jewel-install/jewel.yaml index 6fba9555c6f..f5eec07e51d 100644 --- a/qa/suites/upgrade/jewel-x/parallel/1-jewel-install/jewel.yaml +++ b/qa/suites/upgrade/jewel-x/parallel/1-jewel-install/jewel.yaml @@ -9,6 +9,7 @@ tasks: exclude_packages: ['ceph-mgr','libcephfs2','libcephfs-devel','libcephfs-dev'] - print: "**** done installing jewel" - ceph: + skip_mgr_daemons: true - print: "**** done ceph" - install.upgrade: mon.a: diff --git a/qa/suites/upgrade/jewel-x/stress-split/1-jewel-install/jewel.yaml b/qa/suites/upgrade/jewel-x/stress-split/1-jewel-install/jewel.yaml index 87bc11f451c..a617f91d19a 100644 --- a/qa/suites/upgrade/jewel-x/stress-split/1-jewel-install/jewel.yaml +++ b/qa/suites/upgrade/jewel-x/stress-split/1-jewel-install/jewel.yaml @@ -6,4 +6,5 @@ tasks: exclude_packages: ['ceph-mgr','libcephfs2','libcephfs-devel','libcephfs-dev'] - print: "**** done install jewel" - ceph: + skip_mgr_daemons: true - print: "**** done ceph" diff --git a/qa/tasks/ceph.py b/qa/tasks/ceph.py index 831df9f0345..181429b0716 100644 --- a/qa/tasks/ceph.py +++ b/qa/tasks/ceph.py @@ -39,6 +39,7 @@ def generate_caps(type_): defaults = dict( osd=dict( mon='allow *', + mgr='allow *', osd='allow *', ), mgr=dict( @@ -46,11 +47,13 @@ def generate_caps(type_): ), mds=dict( mon='allow *', + mgr='allow *', osd='allow *', mds='allow', ), client=dict( mon='allow rw', + mgr='allow r', osd='allow rwx', mds='allow', ), @@ -542,6 +545,7 @@ def cluster(ctx, config): '--cap', 'mon', 'allow *', '--cap', 'osd', 'allow *', '--cap', 'mds', 'allow *', + '--cap', 'mgr', 'allow *', keyring_path, ], ) @@ -596,34 +600,35 @@ def cluster(ctx, config): ), ) - log.info('Setting up mgr nodes...') - mgrs = ctx.cluster.only(teuthology.is_type('mgr', cluster_name)) - for remote, roles_for_host in mgrs.remotes.iteritems(): - for role in teuthology.cluster_roles_of_type(roles_for_host, 'mgr', - cluster_name): - _, _, id_ = teuthology.split_role(role) - mgr_dir = '/var/lib/ceph/mgr/{cluster}-{id}'.format( - cluster=cluster_name, - id=id_, - ) - remote.run( - args=[ - 'sudo', - 'mkdir', - '-p', - mgr_dir, - run.Raw('&&'), - 'sudo', - 'adjust-ulimits', - 'ceph-coverage', - coverage_dir, - 'ceph-authtool', - '--create-keyring', - '--gen-key', - '--name=mgr.{id}'.format(id=id_), - mgr_dir + '/keyring', - ], - ) + if not config.get('skip_mgr_daemons', False): + log.info('Setting up mgr nodes...') + mgrs = ctx.cluster.only(teuthology.is_type('mgr', cluster_name)) + for remote, roles_for_host in mgrs.remotes.iteritems(): + for role in teuthology.cluster_roles_of_type(roles_for_host, 'mgr', + cluster_name): + _, _, id_ = teuthology.split_role(role) + mgr_dir = '/var/lib/ceph/mgr/{cluster}-{id}'.format( + cluster=cluster_name, + id=id_, + ) + remote.run( + args=[ + 'sudo', + 'mkdir', + '-p', + mgr_dir, + run.Raw('&&'), + 'sudo', + 'adjust-ulimits', + 'ceph-coverage', + coverage_dir, + 'ceph-authtool', + '--create-keyring', + '--gen-key', + '--name=mgr.{id}'.format(id=id_), + mgr_dir + '/keyring', + ], + ) log.info('Setting up mds nodes...') mdss = ctx.cluster.only(teuthology.is_type('mds', cluster_name)) @@ -787,6 +792,8 @@ def cluster(ctx, config): keys = [] for remote, roles_for_host in ctx.cluster.remotes.iteritems(): for type_ in ['mgr', 'mds', 'osd']: + if type_ == 'mgr' and config.get('skip_mgr_daemons', False): + continue for role in teuthology.cluster_roles_of_type(roles_for_host, type_, cluster_name): _, _, id_ = teuthology.split_role(role) data = teuthology.get_file( @@ -1137,13 +1144,18 @@ def run_daemon(ctx, config, type_): run_cmd.extend(run_cmd_tail) - ctx.daemons.add_daemon(remote, type_, id_, - cluster=cluster_name, - args=run_cmd, - logger=log.getChild(role), - stdin=run.PIPE, - wait=False, - ) + # always register mgr; don't necessarily start + ctx.daemons.register_daemon( + remote, type_, id_, + cluster=cluster_name, + args=run_cmd, + logger=log.getChild(role), + stdin=run.PIPE, + wait=False + ) + if type_ != 'mgr' or not config.get('skip_mgr_daemons', False): + role = cluster_name + '.' + type_ + ctx.daemons.get_daemon(type_, id_, cluster_name).restart() try: yield @@ -1547,6 +1559,7 @@ def task(ctx, config): mount_options=config.get('mount_options', None), block_journal=config.get('block_journal', None), tmpfs_journal=config.get('tmpfs_journal', None), + skip_mgr_daemons=config.get('skip_mgr_daemons', False), log_whitelist=config.get('log-whitelist', []), cpu_profile=set(config.get('cpu_profile', []),), cluster=config['cluster'], diff --git a/qa/tasks/dump_stuck.py b/qa/tasks/dump_stuck.py index be29c7c2110..86e04a9ca88 100644 --- a/qa/tasks/dump_stuck.py +++ b/qa/tasks/dump_stuck.py @@ -26,7 +26,6 @@ def check_stuck(manager, num_inactive, num_unclean, num_stale, timeout=10): inactive = manager.get_stuck_pgs('inactive', timeout) unclean = manager.get_stuck_pgs('unclean', timeout) stale = manager.get_stuck_pgs('stale', timeout) - log.info('hi mom') log.info('inactive %s / %d, unclean %s / %d, stale %s / %d', len(inactive), num_inactive, len(unclean), num_unclean, @@ -35,19 +34,6 @@ def check_stuck(manager, num_inactive, num_unclean, num_stale, timeout=10): assert len(unclean) == num_unclean assert len(stale) == num_stale - # check health output as well - health = manager.raw_cluster_cmd('health') - log.debug('ceph health is: %s', health) - if num_inactive > 0: - m = re.search('(\d+) pgs stuck inactive', health) - assert int(m.group(1)) == num_inactive - if num_unclean > 0: - m = re.search('(\d+) pgs stuck unclean', health) - assert int(m.group(1)) == num_unclean - if num_stale > 0: - m = re.search('(\d+) pgs stuck stale', health) - assert int(m.group(1)) == num_stale - def task(ctx, config): """ Test the dump_stuck command. diff --git a/qa/workunits/ceph-helpers.sh b/qa/workunits/ceph-helpers.sh index 2e23c4a4508..316e826f803 100755 --- a/qa/workunits/ceph-helpers.sh +++ b/qa/workunits/ceph-helpers.sh @@ -174,6 +174,7 @@ function test_teardown() { # @param delays sequence of sleep times before failure # function kill_daemon() { + set -x local pid=$(cat $1) local send_signal=$2 local delays=${3:-0.1 0.2 1 1 1 2 3 5 5 5 10 10 20 60 60 60 120} @@ -195,6 +196,7 @@ function test_kill_daemon() { local dir=$1 setup $dir || return 1 run_mon $dir a --osd_pool_default_size=1 || return 1 + run_mgr $dir x || return 1 run_osd $dir 0 || return 1 name_prefix=osd @@ -214,6 +216,14 @@ function test_kill_daemon() { ceph osd dump | grep "osd.0 down" || return 1 + name_prefix=mgr + for pidfile in $(find $dir 2>/dev/null | grep $name_prefix'[^/]*\.pid') ; do + # + # kill the mgr + # + kill_daemon $pidfile TERM || return 1 + done + name_prefix=mon for pidfile in $(find $dir 2>/dev/null | grep $name_prefix'[^/]*\.pid') ; do # @@ -279,6 +289,7 @@ function test_kill_daemons() { local dir=$1 setup $dir || return 1 run_mon $dir a --osd_pool_default_size=1 || return 1 + run_mgr $dir x || return 1 run_osd $dir 0 || return 1 # # sending signal 0 won't kill the daemon @@ -293,6 +304,10 @@ function test_kill_daemons() { kill_daemons $dir TERM osd || return 1 ceph osd dump | grep "osd.0 down" || return 1 # + # kill the mgr + # + kill_daemons $dir TERM mgr || return 1 + # # kill the mon and verify it cannot be reached # kill_daemons $dir TERM || return 1 @@ -428,6 +443,32 @@ function test_run_mon() { ####################################################################### +function run_mgr() { + local dir=$1 + shift + local id=$1 + shift + local data=$dir/$id + + ceph-mgr \ + --id $id \ + --erasure-code-dir=$CEPH_LIB \ + --plugin-dir=$CEPH_LIB \ + --debug-mgr 20 \ + --debug-objecter 20 \ + --debug-ms 20 \ + --debug-paxos 20 \ + --chdir= \ + --mgr-data=$data \ + --log-file=$dir/\$name.log \ + --admin-socket=$dir/\$cluster-\$name.asok \ + --run-dir=$dir \ + --pid-file=$dir/\$name.pid \ + "$@" || return 1 +} + +####################################################################### + ## # Create (prepare) and run (activate) an osd by the name osd.**id** # with data in **dir**/**id**. The logs can be found in @@ -484,6 +525,7 @@ function test_run_osd() { setup $dir || return 1 run_mon $dir a || return 1 + run_mgr $dir x || return 1 run_osd $dir 0 || return 1 local backfills=$(CEPH_ARGS='' ceph --format=json daemon $dir//ceph-osd.0.asok \ @@ -534,6 +576,7 @@ function test_destroy_osd() { setup $dir || return 1 run_mon $dir a || return 1 + run_mgr $dir x || return 1 run_osd $dir 0 || return 1 destroy_osd $dir 0 || return 1 ! ceph osd dump | grep "osd.$id " || return 1 @@ -625,6 +668,7 @@ function test_activate_osd() { setup $dir || return 1 run_mon $dir a || return 1 + run_mgr $dir x || return 1 run_osd $dir 0 || return 1 local backfills=$(CEPH_ARGS='' ceph --format=json daemon $dir//ceph-osd.0.asok \ @@ -672,6 +716,7 @@ function test_wait_for_osd() { local dir=$1 setup $dir || return 1 run_mon $dir a --osd_pool_default_size=1 || return 1 + run_mgr $dir x || return 1 run_osd $dir 0 || return 1 wait_for_osd up 0 || return 1 kill_daemons $dir TERM osd || return 1 @@ -706,6 +751,7 @@ function test_get_osds() { setup $dir || return 1 run_mon $dir a --osd_pool_default_size=2 || return 1 + run_mgr $dir x || return 1 run_osd $dir 0 || return 1 run_osd $dir 1 || return 1 wait_for_clean || return 1 @@ -773,6 +819,7 @@ function test_get_pg() { setup $dir || return 1 run_mon $dir a --osd_pool_default_size=1 || return 1 + run_mgr $dir x || return 1 run_osd $dir 0 || return 1 wait_for_clean || return 1 get_pg rbd GROUP | grep --quiet '^[0-9]\.[0-9a-f][0-9a-f]*$' || return 1 @@ -809,6 +856,7 @@ function test_get_config() { setup $dir || return 1 run_mon $dir a --osd_pool_default_size=1 || return 1 test $(get_config mon a osd_pool_default_size) = 1 || return 1 + run_mgr $dir x || return 1 run_osd $dir 0 --osd_max_scrubs=3 || return 1 test $(get_config osd 0 osd_max_scrubs) = 3 || return 1 teardown $dir || return 1 @@ -876,6 +924,7 @@ function test_get_primary() { setup $dir || return 1 run_mon $dir a --osd_pool_default_size=1 || return 1 local osd=0 + run_mgr $dir x || return 1 run_osd $dir $osd || return 1 wait_for_clean || return 1 test $(get_primary rbd GROUP) = $osd || return 1 @@ -908,6 +957,7 @@ function test_get_not_primary() { setup $dir || return 1 run_mon $dir a --osd_pool_default_size=2 || return 1 + run_mgr $dir x || return 1 run_osd $dir 0 || return 1 run_osd $dir 1 || return 1 wait_for_clean || return 1 @@ -958,6 +1008,7 @@ function test_objectstore_tool() { setup $dir || return 1 run_mon $dir a --osd_pool_default_size=1 || return 1 local osd=0 + run_mgr $dir x || return 1 run_osd $dir $osd || return 1 wait_for_clean || return 1 rados --pool rbd put GROUP /etc/group || return 1 @@ -991,6 +1042,7 @@ function test_get_is_making_recovery_progress() { setup $dir || return 1 run_mon $dir a || return 1 + run_mgr $dir x || return 1 ! get_is_making_recovery_progress || return 1 teardown $dir || return 1 } @@ -1024,6 +1076,7 @@ function test_get_num_active_clean() { setup $dir || return 1 run_mon $dir a --osd_pool_default_size=1 || return 1 + run_mgr $dir x || return 1 run_osd $dir 0 || return 1 wait_for_clean || return 1 local num_active_clean=$(get_num_active_clean) @@ -1050,6 +1103,7 @@ function test_get_num_pgs() { setup $dir || return 1 run_mon $dir a --osd_pool_default_size=1 || return 1 + run_mgr $dir x || return 1 run_osd $dir 0 || return 1 wait_for_clean || return 1 local num_pgs=$(get_num_pgs) @@ -1080,6 +1134,7 @@ function test_get_last_scrub_stamp() { setup $dir || return 1 run_mon $dir a --osd_pool_default_size=1 || return 1 + run_mgr $dir x || return 1 run_osd $dir 0 || return 1 wait_for_clean || return 1 stamp=$(get_last_scrub_stamp 1.0) @@ -1106,6 +1161,7 @@ function test_is_clean() { setup $dir || return 1 run_mon $dir a --osd_pool_default_size=1 || return 1 + run_mgr $dir x || return 1 run_osd $dir 0 || return 1 ! is_clean || return 1 wait_for_clean || return 1 @@ -1202,6 +1258,7 @@ function test_wait_for_clean() { setup $dir || return 1 run_mon $dir a --osd_pool_default_size=1 || return 1 + run_mgr $dir x || return 1 ! TIMEOUT=1 wait_for_clean || return 1 run_osd $dir 0 || return 1 wait_for_clean || return 1 @@ -1230,6 +1287,7 @@ function test_repair() { setup $dir || return 1 run_mon $dir a --osd_pool_default_size=1 || return 1 + run_mgr $dir x || return 1 run_osd $dir 0 || return 1 wait_for_clean || return 1 repair 1.0 || return 1 @@ -1268,6 +1326,7 @@ function test_pg_scrub() { setup $dir || return 1 run_mon $dir a --osd_pool_default_size=1 || return 1 + run_mgr $dir x || return 1 run_osd $dir 0 || return 1 wait_for_clean || return 1 pg_scrub 1.0 || return 1 @@ -1358,6 +1417,7 @@ function test_wait_for_scrub() { setup $dir || return 1 run_mon $dir a --osd_pool_default_size=1 || return 1 + run_mgr $dir x || return 1 run_osd $dir 0 || return 1 wait_for_clean || return 1 local pgid=1.0 @@ -1405,6 +1465,7 @@ function test_erasure_code_plugin_exists() { setup $dir || return 1 run_mon $dir a || return 1 + run_mgr $dir x || return 1 erasure_code_plugin_exists jerasure || return 1 ! erasure_code_plugin_exists FAKE || return 1 teardown $dir || return 1 diff --git a/qa/workunits/cephtool/test.sh b/qa/workunits/cephtool/test.sh index a5066fb0d7d..7801702c6ec 100755 --- a/qa/workunits/cephtool/test.sh +++ b/qa/workunits/cephtool/test.sh @@ -536,8 +536,10 @@ function test_auth() function test_auth_profiles() { - ceph auth add client.xx-profile-ro mon 'allow profile read-only' - ceph auth add client.xx-profile-rw mon 'allow profile read-write' + ceph auth add client.xx-profile-ro mon 'allow profile read-only' \ + mgr 'allow profile read-only' + ceph auth add client.xx-profile-rw mon 'allow profile read-write' \ + mgr 'allow profile read-write' ceph auth add client.xx-profile-rd mon 'allow profile role-definer' ceph auth export > client.xx.keyring diff --git a/src/ceph_mgr.cc b/src/ceph_mgr.cc index acdddec194e..c3fcc276c79 100644 --- a/src/ceph_mgr.cc +++ b/src/ceph_mgr.cc @@ -41,10 +41,9 @@ int main(int argc, const char **argv) // For consumption by KeyRing::from_ceph_context in MonClient g_conf->set_val("keyring", "$mgr_data/keyring", false); - MgrStandby mgr; - // Handle --help if ((args.size() == 1 && (std::string(args[0]) == "--help" || std::string(args[0]) == "-h"))) { + MgrStandby mgr; mgr.usage(); return 0; } @@ -53,6 +52,7 @@ int main(int argc, const char **argv) global_init_chdir(g_ceph_context); common_init_finish(g_ceph_context); + MgrStandby mgr; int rc = mgr.init(); if (rc != 0) { std::cerr << "Error in initialization: " << cpp_strerror(rc) << std::endl; diff --git a/src/ceph_mon.cc b/src/ceph_mon.cc index 0ee2b170bfb..6e8fd1d5749 100644 --- a/src/ceph_mon.cc +++ b/src/ceph_mon.cc @@ -421,7 +421,7 @@ int main(int argc, const char **argv) } assert(r == 0); - Monitor mon(g_ceph_context, g_conf->name.get_id(), &store, 0, &monmap); + Monitor mon(g_ceph_context, g_conf->name.get_id(), &store, 0, 0, &monmap); r = mon.mkfs(osdmapbl); if (r < 0) { cerr << argv[0] << ": error creating monfs: " << cpp_strerror(r) << std::endl; @@ -703,6 +703,14 @@ int main(int argc, const char **argv) prefork.exit(1); } + Messenger *mgr_msgr = Messenger::create(g_ceph_context, public_msgr_type, + entity_name_t::MON(rank), "mon-mgrc", + getpid(), 0); + if (!mgr_msgr) { + derr << "unable to create mgr_msgr" << dendl; + prefork.exit(1); + } + cout << "starting " << g_conf->name << " rank " << rank << " at " << ipaddr << " mon_data " << g_conf->mon_data @@ -711,7 +719,7 @@ int main(int argc, const char **argv) // start monitor mon = new Monitor(g_ceph_context, g_conf->name.get_id(), store, - msgr, &monmap); + msgr, mgr_msgr, &monmap); if (force_sync) { derr << "flagging a forced sync ..." << dendl; @@ -736,6 +744,7 @@ int main(int argc, const char **argv) } msgr->start(); + mgr_msgr->start(); mon->init(); @@ -749,6 +758,7 @@ int main(int argc, const char **argv) kill(getpid(), SIGTERM); msgr->wait(); + mgr_msgr->wait(); store->close(); @@ -760,6 +770,7 @@ int main(int argc, const char **argv) delete mon; delete store; delete msgr; + delete mgr_msgr; delete client_throttler; delete daemon_throttler; diff --git a/src/common/CommandTable.h b/src/common/CommandTable.h index 6aab1b8c24b..e952ebe2f27 100644 --- a/src/common/CommandTable.h +++ b/src/common/CommandTable.h @@ -93,6 +93,10 @@ public: { commands.erase(tid); } + + void clear() { + commands.clear(); + } }; #endif diff --git a/src/common/config_opts.h b/src/common/config_opts.h index cadcff155c4..9eb84cc1b9f 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -329,6 +329,7 @@ OPTION(mon_probe_timeout, OPT_DOUBLE, 2.0) OPTION(mon_slurp_timeout, OPT_DOUBLE, 10.0) OPTION(mon_slurp_bytes, OPT_INT, 256*1024) // limit size of slurp messages OPTION(mon_client_bytes, OPT_U64, 100ul << 20) // client msg data allowed in memory (in bytes) +OPTION(mon_mgr_proxy_client_bytes_ratio, OPT_FLOAT, .3) // ratio of mon_client_bytes that can be consumed by proxied mgr commands before we error out to client OPTION(mon_daemon_bytes, OPT_U64, 400ul << 20) // mds, osd message memory cap (in bytes) OPTION(mon_max_log_entries_per_event, OPT_INT, 4096) OPTION(mon_reweight_min_pgs_per_osd, OPT_U64, 10) // min pgs per osd for reweight-by-pg command @@ -1629,13 +1630,26 @@ OPTION(rgw_period_push_interval, OPT_DOUBLE, 2) // seconds to wait before retryi OPTION(rgw_period_push_interval_max, OPT_DOUBLE, 30) // maximum interval after exponential backoff OPTION(rgw_swift_versioning_enabled, OPT_BOOL, false) // whether swift object versioning feature is enabled + OPTION(mgr_module_path, OPT_STR, CEPH_PKGLIBDIR "/mgr") // where to load python modules from OPTION(mgr_modules, OPT_STR, "rest") // Which modules to load OPTION(mgr_data, OPT_STR, "/var/lib/ceph/mgr/$cluster-$id") // where to find keyring etc OPTION(mgr_beacon_period, OPT_INT, 5) // How frequently to send beacon OPTION(mgr_stats_period, OPT_INT, 5) // How frequently to send stats +OPTION(mgr_client_bytes, OPT_U64, 128*1048576) // bytes from clients +OPTION(mgr_client_messages, OPT_U64, 512) // messages from clients +OPTION(mgr_osd_bytes, OPT_U64, 512*1048576) // bytes from osds +OPTION(mgr_osd_messages, OPT_U64, 8192) // messages from osds +OPTION(mgr_mds_bytes, OPT_U64, 128*1048576) // bytes from mdss +OPTION(mgr_mds_messages, OPT_U64, 128) // messages from mdss +OPTION(mgr_mon_bytes, OPT_U64, 128*1048576) // bytes from mons +OPTION(mgr_mon_messages, OPT_U64, 128) // messages from mons + +OPTION(mgr_connect_retry_interval, OPT_DOUBLE, 1.0) + OPTION(mon_mgr_digest_period, OPT_INT, 5) // How frequently to send digests OPTION(mon_mgr_beacon_grace, OPT_INT, 30) // How long to wait to failover +OPTION(mon_mgr_inactive_grace, OPT_INT, 60) // How long before health WARN -> ERR OPTION(rgw_list_bucket_min_readahead, OPT_INT, 1000) // minimum number of entries to read from rados for bucket listing diff --git a/src/librados/RadosClient.cc b/src/librados/RadosClient.cc index 08b960d002a..3e0de3adeb1 100644 --- a/src/librados/RadosClient.cc +++ b/src/librados/RadosClient.cc @@ -825,10 +825,12 @@ int librados::RadosClient::mgr_command(const vector<string>& cmd, Mutex::Locker l(lock); C_SaferCond cond; - mgrclient.start_command(cmd, inbl, outbl, outs, &cond); + int r = mgrclient.start_command(cmd, inbl, outbl, outs, &cond); + if (r < 0) + return r; lock.Unlock(); - int r = cond.wait(); + r = cond.wait(); lock.Lock(); return r; diff --git a/src/messages/MMgrBeacon.h b/src/messages/MMgrBeacon.h index a07ba97f8f6..2962d9c9a1f 100644 --- a/src/messages/MMgrBeacon.h +++ b/src/messages/MMgrBeacon.h @@ -22,7 +22,7 @@ class MMgrBeacon : public PaxosServiceMessage { - static const int HEAD_VERSION = 1; + static const int HEAD_VERSION = 2; static const int COMPAT_VERSION = 1; protected: @@ -30,6 +30,7 @@ protected: entity_addr_t server_addr; bool available; std::string name; + uuid_d fsid; public: MMgrBeacon() @@ -38,10 +39,11 @@ public: { } - MMgrBeacon(uint64_t gid_, const std::string &name_, + MMgrBeacon(const uuid_d& fsid_, uint64_t gid_, const std::string &name_, entity_addr_t server_addr_, bool available_) : PaxosServiceMessage(MSG_MGR_BEACON, 0, HEAD_VERSION, COMPAT_VERSION), - gid(gid_), server_addr(server_addr_), available(available_), name(name_) + gid(gid_), server_addr(server_addr_), available(available_), name(name_), + fsid(fsid_) { } @@ -49,6 +51,7 @@ public: entity_addr_t get_server_addr() const { return server_addr; } bool get_available() const { return available; } const std::string& get_name() const { return name; } + const uuid_d& get_fsid() const { return fsid; } private: ~MMgrBeacon() override {} @@ -58,8 +61,8 @@ public: const char *get_type_name() const override { return "mgrbeacon"; } void print(ostream& out) const override { - out << get_type_name() << " mgr." << name << "(" << gid << ", " - << server_addr << ", " << available << ")"; + out << get_type_name() << " mgr." << name << "(" << fsid << "," + << gid << ", " << server_addr << ", " << available << ")"; } void encode_payload(uint64_t features) override { @@ -68,6 +71,7 @@ public: ::encode(gid, payload); ::encode(available, payload); ::encode(name, payload); + ::encode(fsid, payload); } void decode_payload() override { bufferlist::iterator p = payload.begin(); @@ -76,6 +80,9 @@ public: ::decode(gid, p); ::decode(available, p); ::decode(name, p); + if (header.version >= 2) { + ::decode(fsid, p); + } } }; diff --git a/src/messages/MMgrReport.h b/src/messages/MMgrReport.h index 825cb830858..52090b256b3 100644 --- a/src/messages/MMgrReport.h +++ b/src/messages/MMgrReport.h @@ -55,7 +55,7 @@ WRITE_CLASS_ENCODER(PerfCounterType) class MMgrReport : public Message { - static const int HEAD_VERSION = 1; + static const int HEAD_VERSION = 2; static const int COMPAT_VERSION = 1; public: @@ -65,6 +65,7 @@ public: * counter, it must inline the counter's schema here. */ std::vector<PerfCounterType> declare_types; + std::vector<std::string> undeclare_types; // For all counters present, sorted by idx, output // as many bytes as are needed to represent them @@ -82,18 +83,21 @@ public: ::decode(daemon_name, p); ::decode(declare_types, p); ::decode(packed, p); + if (header.version >= 2) + ::decode(undeclare_types, p); } void encode_payload(uint64_t features) override { ::encode(daemon_name, payload); ::encode(declare_types, payload); ::encode(packed, payload); + ::encode(undeclare_types, payload); } const char *get_type_name() const override { return "mgrreport"; } void print(ostream& out) const override { - out << get_type_name() << "(" << declare_types.size() << " " - << packed.length() << ")"; + out << get_type_name() << "(+" << declare_types.size() << "-" << undeclare_types.size() + << " packed " << packed.length() << ")"; } MMgrReport() diff --git a/src/mgr/DaemonServer.cc b/src/mgr/DaemonServer.cc index 10f70004811..5237f93a5ed 100644 --- a/src/mgr/DaemonServer.cc +++ b/src/mgr/DaemonServer.cc @@ -20,6 +20,7 @@ #include "messages/MCommand.h" #include "messages/MCommandReply.h" #include "messages/MPGStats.h" +#include "messages/MOSDScrub.h" #define dout_context g_ceph_context #define dout_subsys ceph_subsys_mgr @@ -27,13 +28,35 @@ #define dout_prefix *_dout << "mgr.server " << __func__ << " " DaemonServer::DaemonServer(MonClient *monc_, - DaemonStateIndex &daemon_state_, - ClusterState &cluster_state_, - PyModules &py_modules_) - : Dispatcher(g_ceph_context), msgr(nullptr), monc(monc_), + DaemonStateIndex &daemon_state_, + ClusterState &cluster_state_, + PyModules &py_modules_, + LogChannelRef clog_, + LogChannelRef audit_clog_) + : Dispatcher(g_ceph_context), + client_byte_throttler(new Throttle(g_ceph_context, "mgr_client_bytes", + g_conf->mgr_client_bytes)), + client_msg_throttler(new Throttle(g_ceph_context, "mgr_client_messages", + g_conf->mgr_client_messages)), + osd_byte_throttler(new Throttle(g_ceph_context, "mgr_osd_bytes", + g_conf->mgr_osd_bytes)), + osd_msg_throttler(new Throttle(g_ceph_context, "mgr_osd_messsages", + g_conf->mgr_osd_messages)), + mds_byte_throttler(new Throttle(g_ceph_context, "mgr_mds_bytes", + g_conf->mgr_mds_bytes)), + mds_msg_throttler(new Throttle(g_ceph_context, "mgr_mds_messsages", + g_conf->mgr_mds_messages)), + mon_byte_throttler(new Throttle(g_ceph_context, "mgr_mon_bytes", + g_conf->mgr_mon_bytes)), + mon_msg_throttler(new Throttle(g_ceph_context, "mgr_mon_messsages", + g_conf->mgr_mon_messages)), + msgr(nullptr), + monc(monc_), daemon_state(daemon_state_), cluster_state(cluster_state_), py_modules(py_modules_), + clog(clog_), + audit_clog(audit_clog_), auth_registry(g_ceph_context, g_conf->auth_supported.empty() ? g_conf->auth_cluster_required : @@ -48,9 +71,30 @@ DaemonServer::~DaemonServer() { int DaemonServer::init(uint64_t gid, entity_addr_t client_addr) { // Initialize Messenger - std::string public_msgr_type = g_conf->ms_public_type.empty() ? g_conf->get_val<std::string>("ms_type") : g_conf->ms_public_type; + std::string public_msgr_type = g_conf->ms_public_type.empty() ? + g_conf->get_val<std::string>("ms_type") : g_conf->ms_public_type; msgr = Messenger::create(g_ceph_context, public_msgr_type, - entity_name_t::MGR(gid), "server", getpid(), 0); + entity_name_t::MGR(gid), + "mgr", + getpid(), 0); + msgr->set_default_policy(Messenger::Policy::stateless_server(0)); + + // throttle clients + msgr->set_policy_throttlers(entity_name_t::TYPE_CLIENT, + client_byte_throttler.get(), + client_msg_throttler.get()); + + // servers + msgr->set_policy_throttlers(entity_name_t::TYPE_OSD, + osd_byte_throttler.get(), + osd_msg_throttler.get()); + msgr->set_policy_throttlers(entity_name_t::TYPE_MDS, + mds_byte_throttler.get(), + mds_msg_throttler.get()); + msgr->set_policy_throttlers(entity_name_t::TYPE_MON, + mon_byte_throttler.get(), + mon_msg_throttler.get()); + int r = msgr->bind(g_conf->public_addr); if (r < 0) { derr << "unable to bind mgr to " << g_conf->public_addr << dendl; @@ -83,23 +127,47 @@ bool DaemonServer::ms_verify_authorizer(Connection *con, auto handler = auth_registry.get_handler(protocol); if (!handler) { dout(0) << "No AuthAuthorizeHandler found for protocol " << protocol << dendl; - ceph_abort(); is_valid = false; return true; } + MgrSessionRef s(new MgrSession); + s->inst.addr = con->get_peer_addr(); AuthCapsInfo caps_info; - EntityName name; - uint64_t global_id = 0; is_valid = handler->verify_authorizer( cct, monc->rotating_secrets.get(), authorizer_data, - authorizer_reply, name, - global_id, caps_info, + authorizer_reply, s->entity_name, + s->global_id, caps_info, session_key); - // TODO: invent some caps suitable for ceph-mgr + if (is_valid) { + if (caps_info.allow_all) { + dout(10) << " session " << s << " " << s->entity_name + << " allow_all" << dendl; + s->caps.set_allow_all(); + } + if (caps_info.caps.length() > 0) { + bufferlist::iterator p = caps_info.caps.begin(); + string str; + try { + ::decode(str, p); + } + catch (buffer::error& e) { + } + bool success = s->caps.parse(str); + if (success) { + dout(10) << " session " << s << " " << s->entity_name + << " has caps " << s->caps << " '" << str << "'" << dendl; + } else { + dout(10) << " session " << s << " " << s->entity_name + << " failed to parse caps '" << str << "'" << dendl; + is_valid = false; + } + } + con->set_priv(s->get()); + } return true; } @@ -134,7 +202,7 @@ bool DaemonServer::ms_dispatch(Message *m) { Mutex::Locker l(lock); - switch(m->get_type()) { + switch (m->get_type()) { case MSG_PGSTATS: cluster_state.ingest_pgstats(static_cast<MPGStats*>(m)); m->put(); @@ -153,8 +221,10 @@ bool DaemonServer::ms_dispatch(Message *m) void DaemonServer::shutdown() { + dout(10) << __func__ << dendl; msgr->shutdown(); msgr->wait(); + dout(10) << __func__ << " done" << dendl; } @@ -217,21 +287,92 @@ struct MgrCommand { string module; string perm; string availability; + + bool requires_perm(char p) const { + return (perm.find(p) != string::npos); + } + } mgr_commands[] = { #define COMMAND(parsesig, helptext, module, perm, availability) \ {parsesig, helptext, module, perm, availability}, - -COMMAND("foo " \ - "name=bar,type=CephString", \ - "do a thing", "mgr", "rw", "cli") +#include "MgrCommands.h" +#undef COMMAND }; +void DaemonServer::_generate_command_map( + map<string,cmd_vartype>& cmdmap, + map<string,string> ¶m_str_map) +{ + for (map<string,cmd_vartype>::const_iterator p = cmdmap.begin(); + p != cmdmap.end(); ++p) { + if (p->first == "prefix") + continue; + if (p->first == "caps") { + vector<string> cv; + if (cmd_getval(g_ceph_context, cmdmap, "caps", cv) && + cv.size() % 2 == 0) { + for (unsigned i = 0; i < cv.size(); i += 2) { + string k = string("caps_") + cv[i]; + param_str_map[k] = cv[i + 1]; + } + continue; + } + } + param_str_map[p->first] = cmd_vartype_stringify(p->second); + } +} + +const MgrCommand *DaemonServer::_get_mgrcommand( + const string &cmd_prefix, + MgrCommand *cmds, + int cmds_size) +{ + MgrCommand *this_cmd = NULL; + for (MgrCommand *cp = cmds; + cp < &cmds[cmds_size]; cp++) { + if (cp->cmdstring.compare(0, cmd_prefix.size(), cmd_prefix) == 0) { + this_cmd = cp; + break; + } + } + return this_cmd; +} + +bool DaemonServer::_allowed_command( + MgrSession *s, + const string &module, + const string &prefix, + const map<string,cmd_vartype>& cmdmap, + const map<string,string>& param_str_map, + const MgrCommand *this_cmd) { + + if (s->entity_name.is_mon()) { + // mon is all-powerful. even when it is forwarding commands on behalf of + // old clients; we expect the mon is validating commands before proxying! + return true; + } + + bool cmd_r = this_cmd->requires_perm('r'); + bool cmd_w = this_cmd->requires_perm('w'); + bool cmd_x = this_cmd->requires_perm('x'); + + bool capable = s->caps.is_capable( + g_ceph_context, + CEPH_ENTITY_TYPE_MGR, + s->entity_name, + module, prefix, param_str_map, + cmd_r, cmd_w, cmd_x); + + dout(10) << " " << s->entity_name << " " + << (capable ? "" : "not ") << "capable" << dendl; + return capable; +} + bool DaemonServer::handle_command(MCommand *m) { int r = 0; std::stringstream ss; - std::stringstream ds; bufferlist odata; std::string prefix; @@ -239,18 +380,33 @@ bool DaemonServer::handle_command(MCommand *m) cmdmap_t cmdmap; - // TODO enforce some caps - // TODO background the call into python land so that we don't // block a messenger thread on python code. ConnectionRef con = m->get_connection(); + MgrSessionRef session(static_cast<MgrSession*>(con->get_priv())); + if (!session) { + return true; + } + session->put(); // SessionRef takes a ref + if (session->inst.name == entity_name_t()) + session->inst.name = m->get_source(); + + string format; + boost::scoped_ptr<Formatter> f; + const MgrCommand *mgr_cmd; + map<string,string> param_str_map; if (!cmdmap_from_json(m->cmd, &cmdmap, ss)) { r = -EINVAL; goto out; } + { + cmd_getval(g_ceph_context, cmdmap, "format", format, string("plain")); + f.reset(Formatter::create(format)); + } + dout(4) << "decoded " << cmdmap.size() << dendl; cmd_getval(cct, cmdmap, "prefix", prefix); @@ -285,10 +441,96 @@ bool DaemonServer::handle_command(MCommand *m) } #endif f.close_section(); // command_descriptions + goto out; + } - f.flush(ds); + // lookup command + mgr_cmd = _get_mgrcommand(prefix, mgr_commands, + ARRAY_SIZE(mgr_commands)); + _generate_command_map(cmdmap, param_str_map); + if (!mgr_cmd) { + ss << "command not supported"; + r = -EINVAL; goto out; - } else { + } + + // validate user's permissions for requested command + if (!_allowed_command(session.get(), mgr_cmd->module, prefix, cmdmap, + param_str_map, mgr_cmd)) { + dout(1) << __func__ << " access denied" << dendl; + audit_clog->info() << "from='" << session->inst << "' " + << "entity='" << session->entity_name << "' " + << "cmd=" << m->cmd << ": access denied"; + ss << "access denied"; + r = -EACCES; + goto out; + } + + audit_clog->debug() + << "from='" << session->inst << "' " + << "entity='" << session->entity_name << "' " + << "cmd=" << m->cmd << ": dispatch"; + + // ----------- + // PG commands + + if (prefix == "pg scrub" || + prefix == "pg repair" || + prefix == "pg deep-scrub") { + string scrubop = prefix.substr(3, string::npos); + pg_t pgid; + string pgidstr; + cmd_getval(g_ceph_context, cmdmap, "pgid", pgidstr); + if (!pgid.parse(pgidstr.c_str())) { + ss << "invalid pgid '" << pgidstr << "'"; + r = -EINVAL; + goto out; + } + bool pg_exists = false; + cluster_state.with_osdmap([&](const OSDMap& osdmap) { + pg_exists = osdmap.pg_exists(pgid); + }); + if (!pg_exists) { + ss << "pg " << pgid << " dne"; + r = -ENOENT; + goto out; + } + int acting_primary = -1; + entity_inst_t inst; + cluster_state.with_osdmap([&](const OSDMap& osdmap) { + osdmap.pg_to_acting_osds(pgid, nullptr, &acting_primary); + if (acting_primary >= 0) { + inst = osdmap.get_inst(acting_primary); + } + }); + if (acting_primary == -1) { + ss << "pg " << pgid << " has no primary osd"; + r = -EAGAIN; + goto out; + } + vector<pg_t> pgs = { pgid }; + msgr->send_message(new MOSDScrub(monc->get_fsid(), + pgs, + scrubop == "repair", + scrubop == "deep-scrub"), + inst); + ss << "instructing pg " << pgid << " on osd." << acting_primary + << " (" << inst << ") to " << scrubop; + r = 0; + } + + else { + cluster_state.with_pgmap( + [&](const PGMap& pg_map) { + cluster_state.with_osdmap([&](const OSDMap& osdmap) { + r = process_pg_map_command(prefix, cmdmap, pg_map, osdmap, + f.get(), &ss, &odata); + }); + }); + } + + // fall back to registered python handlers + if (r == -EOPNOTSUPP) { // Let's find you a handler! MgrPyModule *handler = nullptr; auto py_commands = py_modules.get_commands(); @@ -312,7 +554,9 @@ bool DaemonServer::handle_command(MCommand *m) // with a ms_dispatch, so that the python part can block if it // wants to. dout(4) << "passing through " << cmdmap.size() << dendl; + stringstream ds; r = handler->handle_command(cmdmap, &ds, &ss); + odata.append(ds); goto out; } @@ -325,7 +569,6 @@ bool DaemonServer::handle_command(MCommand *m) std::string rs; rs = ss.str(); - odata.append(ds); dout(1) << "do_command r=" << r << " " << rs << dendl; if (con) { MCommandReply *reply = new MCommandReply(r, rs); diff --git a/src/mgr/DaemonServer.h b/src/mgr/DaemonServer.h index 91ce05776da..4337d727dd5 100644 --- a/src/mgr/DaemonServer.h +++ b/src/mgr/DaemonServer.h @@ -20,17 +20,20 @@ #include <string> #include "common/Mutex.h" +#include "common/LogClient.h" #include <msg/Messenger.h> #include <mon/MonClient.h> #include "auth/AuthAuthorizeHandler.h" +#include "MgrSession.h" #include "DaemonState.h" class MMgrReport; class MMgrOpen; class MCommand; +struct MgrCommand; /** @@ -40,16 +43,35 @@ class MCommand; class DaemonServer : public Dispatcher { protected: + boost::scoped_ptr<Throttle> client_byte_throttler; + boost::scoped_ptr<Throttle> client_msg_throttler; + boost::scoped_ptr<Throttle> osd_byte_throttler; + boost::scoped_ptr<Throttle> osd_msg_throttler; + boost::scoped_ptr<Throttle> mds_byte_throttler; + boost::scoped_ptr<Throttle> mds_msg_throttler; + boost::scoped_ptr<Throttle> mon_byte_throttler; + boost::scoped_ptr<Throttle> mon_msg_throttler; + Messenger *msgr; MonClient *monc; DaemonStateIndex &daemon_state; ClusterState &cluster_state; PyModules &py_modules; + LogChannelRef clog, audit_clog; AuthAuthorizeHandlerRegistry auth_registry; Mutex lock; + static void _generate_command_map(map<string,cmd_vartype>& cmdmap, + map<string,string> ¶m_str_map); + static const MgrCommand *_get_mgrcommand(const string &cmd_prefix, + MgrCommand *cmds, int cmds_size); + bool _allowed_command( + MgrSession *s, const string &module, const string &prefix, + const map<string,cmd_vartype>& cmdmap, + const map<string,string>& param_str_map, + const MgrCommand *this_cmd); public: int init(uint64_t gid, entity_addr_t client_addr); @@ -58,9 +80,11 @@ public: entity_addr_t get_myaddr() const; DaemonServer(MonClient *monc_, - DaemonStateIndex &daemon_state_, - ClusterState &cluster_state_, - PyModules &py_modules_); + DaemonStateIndex &daemon_state_, + ClusterState &cluster_state_, + PyModules &py_modules_, + LogChannelRef cl, + LogChannelRef auditcl); ~DaemonServer() override; bool ms_dispatch(Message *m) override; diff --git a/src/mgr/DaemonState.cc b/src/mgr/DaemonState.cc index 132f722947b..f83f9749de6 100644 --- a/src/mgr/DaemonState.cc +++ b/src/mgr/DaemonState.cc @@ -110,6 +110,8 @@ void DaemonStateIndex::cull(entity_type_t daemon_type, void DaemonPerfCounters::update(MMgrReport *report) { dout(20) << "loading " << report->declare_types.size() << " new types, " + << report->undeclare_types.size() << " old types, had " + << types.size() << " types, got " << report->packed.length() << " bytes of data" << dendl; // Load any newly declared types @@ -117,6 +119,10 @@ void DaemonPerfCounters::update(MMgrReport *report) types.insert(std::make_pair(t.path, t)); declared_types.insert(t.path); } + // Remove any old types + for (const auto &t : report->undeclare_types) { + declared_types.erase(t); + } const auto now = ceph_clock_now(); diff --git a/src/mgr/Mgr.cc b/src/mgr/Mgr.cc index 6207cd88db2..cdbbb9d0953 100644 --- a/src/mgr/Mgr.cc +++ b/src/mgr/Mgr.cc @@ -37,7 +37,8 @@ #define dout_prefix *_dout << "mgr " << __func__ << " " -Mgr::Mgr(MonClient *monc_, Messenger *clientm_, Objecter *objecter_) : +Mgr::Mgr(MonClient *monc_, Messenger *clientm_, Objecter *objecter_, + LogChannelRef clog_, LogChannelRef audit_clog_) : monc(monc_), objecter(objecter_), client_messenger(clientm_), @@ -47,7 +48,7 @@ Mgr::Mgr(MonClient *monc_, Messenger *clientm_, Objecter *objecter_) : waiting_for_fs_map(NULL), py_modules(daemon_state, cluster_state, *monc, finisher), cluster_state(monc, nullptr), - server(monc, daemon_state, cluster_state, py_modules), + server(monc, daemon_state, cluster_state, py_modules, clog_, audit_clog_), initialized(false), initializing(false) { @@ -422,7 +423,7 @@ void Mgr::handle_osd_map() bool Mgr::ms_dispatch(Message *m) { - derr << *m << dendl; + dout(4) << *m << dendl; Mutex::Locker l(lock); switch (m->get_type()) { @@ -498,7 +499,8 @@ void Mgr::handle_fs_map(MFSMap* m) // FIXME: nothing stopping old daemons being here, they won't have // addr: need to handle case of pre-ceph-mgr daemons that don't have // the fields we expect - if (metadata->metadata.empty()) { + if (metadata->metadata.empty() || + metadata->metadata.count("addr") == 0) { update = true; } else { auto metadata_addr = metadata->metadata.at("addr"); diff --git a/src/mgr/Mgr.h b/src/mgr/Mgr.h index ffe1d1460c9..9c882ec35f9 100644 --- a/src/mgr/Mgr.h +++ b/src/mgr/Mgr.h @@ -69,7 +69,8 @@ protected: bool initializing; public: - Mgr(MonClient *monc_, Messenger *clientm_, Objecter *objecter_); + Mgr(MonClient *monc_, Messenger *clientm_, Objecter *objecter_, + LogChannelRef clog_, LogChannelRef audit_clog_); ~Mgr(); bool is_initialized() const {return initialized;} diff --git a/src/mgr/MgrClient.cc b/src/mgr/MgrClient.cc index d18160ba4e0..e1fa439c2fd 100644 --- a/src/mgr/MgrClient.cc +++ b/src/mgr/MgrClient.cc @@ -31,9 +31,7 @@ MgrClient::MgrClient(CephContext *cct_, Messenger *msgr_) : Dispatcher(cct_), cct(cct_), msgr(msgr_), - lock("mgrc"), - timer(cct_, lock), - report_callback(nullptr) + timer(cct_, lock) { assert(cct != nullptr); } @@ -51,6 +49,10 @@ void MgrClient::shutdown() { Mutex::Locker l(lock); + // forget about in-flight commands if we are prematurely shut down + // (e.g., by control-C) + command_table.clear(); + timer.shutdown(); session.reset(); } @@ -91,42 +93,59 @@ void MgrClient::reconnect() timer.cancel_event(report_callback); report_callback = nullptr; } + } + + if (!map.get_available()) { + ldout(cct, 4) << "No active mgr available yet" << dendl; + return; + } - std::vector<ceph_tid_t> erase_cmds; - auto commands = command_table.get_commands(); - for (const auto &i : commands) { - // FIXME be nicer, retarget command on new mgr? - if (i.second.on_finish != nullptr) { - i.second.on_finish->complete(-ETIMEDOUT); + if (last_connect_attempt != utime_t()) { + utime_t now = ceph_clock_now(); + utime_t when = last_connect_attempt; + when += cct->_conf->mgr_connect_retry_interval; + if (now < when) { + if (!connect_retry_callback) { + connect_retry_callback = new FunctionContext([this](int r){ + connect_retry_callback = nullptr; + reconnect(); + }); + timer.add_event_at(when, connect_retry_callback); } - erase_cmds.push_back(i.first); - } - for (const auto &tid : erase_cmds) { - command_table.erase(tid); + ldout(cct, 4) << "waiting to retry connect until " << when << dendl; + return; } } - if (map.get_available()) { - ldout(cct, 4) << "Starting new session with " << map.get_active_addr() - << dendl; - entity_inst_t inst; - inst.addr = map.get_active_addr(); - inst.name = entity_name_t::MGR(map.get_active_gid()); - - session.reset(new MgrSessionState()); - session->con = msgr->get_connection(inst); - - // Don't send an open if we're just a client (i.e. doing - // command-sending, not stats etc) - if (g_conf && !g_conf->name.is_client()) { - auto open = new MMgrOpen(); - open->daemon_name = g_conf->name.get_id(); - session->con->send_message(open); - } + if (connect_retry_callback) { + timer.cancel_event(connect_retry_callback); + connect_retry_callback = nullptr; + } - signal_cond_list(waiting_for_session); - } else { - ldout(cct, 4) << "No active mgr available yet" << dendl; + ldout(cct, 4) << "Starting new session with " << map.get_active_addr() + << dendl; + entity_inst_t inst; + inst.addr = map.get_active_addr(); + inst.name = entity_name_t::MGR(map.get_active_gid()); + last_connect_attempt = ceph_clock_now(); + + session.reset(new MgrSessionState()); + session->con = msgr->get_connection(inst); + + // Don't send an open if we're just a client (i.e. doing + // command-sending, not stats etc) + if (g_conf && !g_conf->name.is_client()) { + auto open = new MMgrOpen(); + open->daemon_name = g_conf->name.get_id(); + session->con->send_message(open); + } + + // resend any pending commands + for (const auto &p : command_table.get_commands()) { + MCommand *m = p.second.get_message({}); + assert(session); + assert(session->con); + session->con->send_message(m); } } @@ -180,45 +199,45 @@ void MgrClient::send_report() pcc->with_counters([this, report]( const PerfCountersCollection::CounterMap &by_path) { - bool const declared_all = (session->declared.size() == by_path.size()); - - if (!declared_all) { - for (const auto &i : by_path) { - auto path = i.first; - auto data = *(i.second); - - if (session->declared.count(path) == 0) { - PerfCounterType type; - type.path = path; - if (data.description) { - type.description = data.description; - } - if (data.nick) { - type.nick = data.nick; - } - type.type = data.type; - report->declare_types.push_back(std::move(type)); - session->declared.insert(path); - } + ENCODE_START(1, 1, report->packed); + for (auto p = session->declared.begin(); p != session->declared.end(); ) { + if (by_path.count(*p) == 0) { + report->undeclare_types.push_back(*p); + ldout(cct,20) << __func__ << " undeclare " << *p << dendl; + p = session->declared.erase(p); + } else { + ++p; } } + for (const auto &i : by_path) { + auto& path = i.first; + auto& data = *(i.second); + + if (session->declared.count(path) == 0) { + ldout(cct,20) << __func__ << " declare " << path << dendl; + PerfCounterType type; + type.path = path; + if (data.description) { + type.description = data.description; + } + if (data.nick) { + type.nick = data.nick; + } + type.type = data.type; + report->declare_types.push_back(std::move(type)); + session->declared.insert(path); + } - ldout(cct, 20) << by_path.size() << " counters, of which " - << report->declare_types.size() << " new" << dendl; - - ENCODE_START(1, 1, report->packed); - for (const auto &path : session->declared) { - auto data = by_path.at(path); - ::encode(static_cast<uint64_t>(data->u64.read()), - report->packed); - if (data->type & PERFCOUNTER_LONGRUNAVG) { - ::encode(static_cast<uint64_t>(data->avgcount.read()), - report->packed); - ::encode(static_cast<uint64_t>(data->avgcount2.read()), - report->packed); + ::encode(static_cast<uint64_t>(data.u64.read()), report->packed); + if (data.type & PERFCOUNTER_LONGRUNAVG) { + ::encode(static_cast<uint64_t>(data.avgcount.read()), report->packed); + ::encode(static_cast<uint64_t>(data.avgcount2.read()), report->packed); } } ENCODE_FINISH(report->packed); + + ldout(cct, 20) << by_path.size() << " counters, of which " + << report->declare_types.size() << " new" << dendl; }); ldout(cct, 20) << "encoded " << report->packed.length() << " bytes" << dendl; @@ -262,22 +281,6 @@ bool MgrClient::handle_mgr_configure(MMgrConfigure *m) return true; } -void MgrClient::wait_on_list(list<Cond*>& ls) -{ - assert(lock.is_locked_by_me()); - - Cond cond; - ls.push_back(&cond); - cond.Wait(lock); - ls.remove(&cond); -} - -void MgrClient::signal_cond_list(list<Cond*>& ls) -{ - for (list<Cond*>::iterator it = ls.begin(); it != ls.end(); ++it) - (*it)->Signal(); -} - int MgrClient::start_command(const vector<string>& cmd, const bufferlist& inbl, bufferlist *outbl, string *outs, Context *onfinish) @@ -286,13 +289,11 @@ int MgrClient::start_command(const vector<string>& cmd, const bufferlist& inbl, ldout(cct, 20) << "cmd: " << cmd << dendl; - if (!session) { - lderr(cct) << "no session, waiting" << dendl; - wait_on_list(waiting_for_session); + if (map.epoch == 0) { + ldout(cct,20) << " no MgrMap, assuming EACCES" << dendl; + return -EACCES; } - assert(map.epoch > 0); - auto &op = command_table.start_command(); op.cmd = cmd; op.inbl = inbl; @@ -300,12 +301,11 @@ int MgrClient::start_command(const vector<string>& cmd, const bufferlist& inbl, op.outs = outs; op.on_finish = onfinish; - // Leaving fsid argument null because it isn't used. - MCommand *m = op.get_message({}); - assert(session); - assert(session->con); - session->con->send_message(m); - + if (session && session->con) { + // Leaving fsid argument null because it isn't used. + MCommand *m = op.get_message({}); + session->con->send_message(m); + } return 0; } diff --git a/src/mgr/MgrClient.h b/src/mgr/MgrClient.h index 8f771d891b3..adc35fe406f 100644 --- a/src/mgr/MgrClient.h +++ b/src/mgr/MgrClient.h @@ -56,18 +56,17 @@ protected: unique_ptr<MgrSessionState> session; - Mutex lock; + Mutex lock = {"MgrClient::lock"}; - uint32_t stats_period; - SafeTimer timer; + uint32_t stats_period = 0; + SafeTimer timer; CommandTable<MgrCommand> command_table; - void wait_on_list(list<Cond*>& ls); - void signal_cond_list(list<Cond*>& ls); + utime_t last_connect_attempt; - list<Cond*> waiting_for_session; - Context *report_callback; + Context *report_callback = nullptr; + Context *connect_retry_callback = nullptr; // If provided, use this to compose an MPGStats to send with // our reports (hook for use by OSD) diff --git a/src/mgr/MgrCommands.h b/src/mgr/MgrCommands.h new file mode 100644 index 00000000000..6a407f4e669 --- /dev/null +++ b/src/mgr/MgrCommands.h @@ -0,0 +1,56 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +/* no guard; may be included multiple times */ + +// see MonCommands.h + +COMMAND("pg stat", "show placement group status.", + "pg", "r", "cli,rest") +COMMAND("pg getmap", "get binary pg map to -o/stdout", "pg", "r", "cli,rest") + +COMMAND("pg dump " \ + "name=dumpcontents,type=CephChoices,strings=all|summary|sum|delta|pools|osds|pgs|pgs_brief,n=N,req=false", \ + "show human-readable versions of pg map (only 'all' valid with plain)", "pg", "r", "cli,rest") +COMMAND("pg dump_json " \ + "name=dumpcontents,type=CephChoices,strings=all|summary|sum|pools|osds|pgs,n=N,req=false", \ + "show human-readable version of pg map in json only",\ + "pg", "r", "cli,rest") +COMMAND("pg dump_pools_json", "show pg pools info in json only",\ + "pg", "r", "cli,rest") + +COMMAND("pg ls-by-pool " \ + "name=poolstr,type=CephString " \ + "name=states,type=CephChoices,strings=active|clean|down|scrubbing|degraded|inconsistent|peering|repair|recovering|backfill_wait|incomplete|stale|remapped|deep_scrub|backfill|backfill_toofull|recovery_wait|undersized|activating|peered,n=N,req=false ", \ + "list pg with pool = [poolname]", "pg", "r", "cli,rest") +COMMAND("pg ls-by-primary " \ + "name=osd,type=CephOsdName " \ + "name=pool,type=CephInt,req=false " \ + "name=states,type=CephChoices,strings=active|clean|down|scrubbing|degraded|inconsistent|peering|repair|recovering|backfill_wait|incomplete|stale|remapped|deep_scrub|backfill|backfill_toofull|recovery_wait|undersized|activating|peered,n=N,req=false ", \ + "list pg with primary = [osd]", "pg", "r", "cli,rest") +COMMAND("pg ls-by-osd " \ + "name=osd,type=CephOsdName " \ + "name=pool,type=CephInt,req=false " \ + "name=states,type=CephChoices,strings=active|clean|down|scrubbing|degraded|inconsistent|peering|repair|recovering|backfill_wait|incomplete|stale|remapped|deep_scrub|backfill|backfill_toofull|recovery_wait|undersized|activating|peered,n=N,req=false ", \ + "list pg on osd [osd]", "pg", "r", "cli,rest") +COMMAND("pg ls " \ + "name=pool,type=CephInt,req=false " \ + "name=states,type=CephChoices,strings=active|clean|down|scrubbing|degraded|inconsistent|peering|repair|recovering|backfill_wait|incomplete|stale|remapped|deep_scrub|backfill|backfill_toofull|recovery_wait|undersized|activating|peered,n=N,req=false ", \ + "list pg with specific pool, osd, state", "pg", "r", "cli,rest") +COMMAND("pg map name=pgid,type=CephPgid", "show mapping of pg to osds", \ + "pg", "r", "cli,rest") +COMMAND("pg dump_stuck " \ + "name=stuckops,type=CephChoices,strings=inactive|unclean|stale|undersized|degraded,n=N,req=false " \ + "name=threshold,type=CephInt,req=false", + "show information about stuck pgs",\ + "pg", "r", "cli,rest") +COMMAND("pg debug " \ + "name=debugop,type=CephChoices,strings=unfound_objects_exist|degraded_pgs_exist", \ + "show debug info about pgs", "pg", "r", "cli,rest") + +COMMAND("pg scrub name=pgid,type=CephPgid", "start scrub on <pgid>", \ + "pg", "rw", "cli,rest") +COMMAND("pg deep-scrub name=pgid,type=CephPgid", "start deep-scrub on <pgid>", \ + "pg", "rw", "cli,rest") +COMMAND("pg repair name=pgid,type=CephPgid", "start repair on <pgid>", \ + "pg", "rw", "cli,rest") diff --git a/src/mgr/MgrSession.h b/src/mgr/MgrSession.h new file mode 100644 index 00000000000..328871d2ef9 --- /dev/null +++ b/src/mgr/MgrSession.h @@ -0,0 +1,31 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_MGR_MGRSESSION_H +#define CEPH_MGR_MGRSESSION_H + +#include "common/RefCountedObj.h" +#include "common/entity_name.h" +#include "msg/msg_types.h" +#include "mon/MonCap.h" + + +/** + * Session state associated with the Connection. + */ +struct MgrSession : public RefCountedObject { + uint64_t global_id = 0; + EntityName entity_name; + entity_inst_t inst; + + // mon caps are suitably generic for mgr + MonCap caps; + + MgrSession() : RefCountedObject(0) {} + ~MgrSession() override {} +}; + +typedef boost::intrusive_ptr<MgrSession> MgrSessionRef; + + +#endif diff --git a/src/mgr/MgrStandby.cc b/src/mgr/MgrStandby.cc index 8e460e1ac98..843ecd0a3a0 100644 --- a/src/mgr/MgrStandby.cc +++ b/src/mgr/MgrStandby.cc @@ -36,12 +36,15 @@ MgrStandby::MgrStandby() : Dispatcher(g_ceph_context), monc(new MonClient(g_ceph_context)), + client_messenger(Messenger::create_client_messenger(g_ceph_context, "mgr")), + objecter(new Objecter(g_ceph_context, client_messenger, monc, NULL, 0, 0)), + log_client(g_ceph_context, client_messenger, &monc->monmap, LogClient::NO_FLAGS), + clog(log_client.create_channel(CLOG_CHANNEL_CLUSTER)), + audit_clog(log_client.create_channel(CLOG_CHANNEL_AUDIT)), lock("MgrStandby::lock"), timer(g_ceph_context, lock), active_mgr(nullptr) { - client_messenger = Messenger::create_client_messenger(g_ceph_context, "mgr"); - objecter = new Objecter(g_ceph_context, client_messenger, monc, NULL, 0, 0); } @@ -52,6 +55,41 @@ MgrStandby::~MgrStandby() delete client_messenger; } +const char** MgrStandby::get_tracked_conf_keys() const +{ + static const char* KEYS[] = { + // clog & admin clog + "clog_to_monitors", + "clog_to_syslog", + "clog_to_syslog_facility", + "clog_to_syslog_level", + "osd_objectstore_fuse", + "clog_to_graylog", + "clog_to_graylog_host", + "clog_to_graylog_port", + "host", + "fsid", + NULL + }; + return KEYS; +} + +void MgrStandby::handle_conf_change( + const struct md_config_t *conf, + const std::set <std::string> &changed) +{ + if (changed.count("clog_to_monitors") || + changed.count("clog_to_syslog") || + changed.count("clog_to_syslog_level") || + changed.count("clog_to_syslog_facility") || + changed.count("clog_to_graylog") || + changed.count("clog_to_graylog_host") || + changed.count("clog_to_graylog_port") || + changed.count("host") || + changed.count("fsid")) { + _update_log_config(); + } +} int MgrStandby::init() { @@ -92,6 +130,9 @@ int MgrStandby::init() client_t whoami = monc->get_global_id(); client_messenger->set_myname(entity_name_t::CLIENT(whoami.v)); + monc->set_log_client(&log_client); + _update_log_config(); + objecter->set_client_incarnation(0); objecter->init(); client_messenger->add_dispatcher_head(objecter); @@ -112,7 +153,8 @@ void MgrStandby::send_beacon() bool available = active_mgr != nullptr && active_mgr->is_initialized(); auto addr = available ? active_mgr->get_server_addr() : entity_addr_t(); - MMgrBeacon *m = new MMgrBeacon(monc->get_global_id(), + MMgrBeacon *m = new MMgrBeacon(monc->get_fsid(), + monc->get_global_id(), g_conf->name.get_id(), addr, available); @@ -150,6 +192,33 @@ void MgrStandby::shutdown() client_messenger->shutdown(); } +void MgrStandby::_update_log_config() +{ + map<string,string> log_to_monitors; + map<string,string> log_to_syslog; + map<string,string> log_channel; + map<string,string> log_prio; + map<string,string> log_to_graylog; + map<string,string> log_to_graylog_host; + map<string,string> log_to_graylog_port; + uuid_d fsid; + string host; + + if (parse_log_client_options(cct, log_to_monitors, log_to_syslog, + log_channel, log_prio, log_to_graylog, + log_to_graylog_host, log_to_graylog_port, + fsid, host) == 0) { + clog->update_config(log_to_monitors, log_to_syslog, + log_channel, log_prio, log_to_graylog, + log_to_graylog_host, log_to_graylog_port, + fsid, host); + audit_clog->update_config(log_to_monitors, log_to_syslog, + log_channel, log_prio, log_to_graylog, + log_to_graylog_host, log_to_graylog_port, + fsid, host); + } +} + void MgrStandby::handle_mgr_map(MMgrMap* mmap) { auto map = mmap->get_map(); @@ -160,7 +229,7 @@ void MgrStandby::handle_mgr_map(MMgrMap* mmap) if (active_in_map) { if (!active_mgr) { dout(1) << "Activating!" << dendl; - active_mgr.reset(new Mgr(monc, client_messenger, objecter)); + active_mgr.reset(new Mgr(monc, client_messenger, objecter, clog, audit_clog)); active_mgr->background_init(); dout(1) << "I am now active" << dendl; } else { diff --git a/src/mgr/MgrStandby.h b/src/mgr/MgrStandby.h index 6fa524b87d9..89ede8cee2c 100644 --- a/src/mgr/MgrStandby.h +++ b/src/mgr/MgrStandby.h @@ -18,6 +18,7 @@ #include "auth/Auth.h" #include "common/Finisher.h" #include "common/Timer.h" +#include "common/LogClient.h" #include "DaemonServer.h" #include "PyModules.h" @@ -28,11 +29,21 @@ class MMgrMap; class Mgr; -class MgrStandby : public Dispatcher { +class MgrStandby : public Dispatcher, + public md_config_obs_t { +public: + // config observer bits + const char** get_tracked_conf_keys() const override; + void handle_conf_change(const struct md_config_t *conf, + const std::set <std::string> &changed) override; + protected: MonClient *monc; - Objecter *objecter; Messenger *client_messenger; + Objecter *objecter; + + LogClient log_client; + LogChannelRef clog, audit_clog; Mutex lock; SafeTimer timer; @@ -42,6 +53,7 @@ protected: std::string state_str(); void handle_mgr_map(MMgrMap *m); + void _update_log_config(); public: MgrStandby(); diff --git a/src/mon/AuthMonitor.cc b/src/mon/AuthMonitor.cc index 7ff20371b06..48f334a0064 100644 --- a/src/mon/AuthMonitor.cc +++ b/src/mon/AuthMonitor.cc @@ -111,7 +111,7 @@ void AuthMonitor::create_initial() inc.max_global_id = max_global_id; pending_auth.push_back(inc); - format_version = 1; + format_version = 2; } void AuthMonitor::update_from_paxos(bool *need_bootstrap) @@ -1033,64 +1033,139 @@ bool AuthMonitor::prepare_global_id(MonOpRequestRef op) void AuthMonitor::upgrade_format() { - unsigned int current = 1; + unsigned int current = 2; + if (!mon->get_quorum_mon_features().contains_all( + ceph::features::mon::FEATURE_LUMINOUS)) { + current = 1; + } if (format_version >= current) { dout(20) << __func__ << " format " << format_version << " is current" << dendl; return; } - dout(1) << __func__ << " upgrading from format " << format_version << " to " << current << dendl; bool changed = false; - map<EntityName, EntityAuth>::iterator p; - for (p = mon->key_server.secrets_begin(); - p != mon->key_server.secrets_end(); - ++p) { - // grab mon caps, if any - string mon_caps; - if (p->second.caps.count("mon") == 0) - continue; - try { - bufferlist::iterator it = p->second.caps["mon"].begin(); - ::decode(mon_caps, it); - } - catch (buffer::error) { - dout(10) << __func__ << " unable to parse mon cap for " - << p->first << dendl; - continue; - } + if (format_version == 0) { + dout(1) << __func__ << " upgrading from format 0 to 1" << dendl; + map<EntityName, EntityAuth>::iterator p; + for (p = mon->key_server.secrets_begin(); + p != mon->key_server.secrets_end(); + ++p) { + // grab mon caps, if any + string mon_caps; + if (p->second.caps.count("mon") == 0) + continue; + try { + bufferlist::iterator it = p->second.caps["mon"].begin(); + ::decode(mon_caps, it); + } + catch (buffer::error) { + dout(10) << __func__ << " unable to parse mon cap for " + << p->first << dendl; + continue; + } - string n = p->first.to_str(); - string new_caps; + string n = p->first.to_str(); + string new_caps; - // set daemon profiles - if ((p->first.is_osd() || p->first.is_mds()) && - mon_caps == "allow rwx") { - new_caps = string("allow profile ") + string(p->first.get_type_name()); - } + // set daemon profiles + if ((p->first.is_osd() || p->first.is_mds()) && + mon_caps == "allow rwx") { + new_caps = string("allow profile ") + string(p->first.get_type_name()); + } - // update bootstrap keys - if (n == "client.bootstrap-osd") { - new_caps = "allow profile bootstrap-osd"; - } - if (n == "client.bootstrap-mds") { - new_caps = "allow profile bootstrap-mds"; + // update bootstrap keys + if (n == "client.bootstrap-osd") { + new_caps = "allow profile bootstrap-osd"; + } + if (n == "client.bootstrap-mds") { + new_caps = "allow profile bootstrap-mds"; + } + + if (new_caps.length() > 0) { + dout(5) << __func__ << " updating " << p->first << " mon cap from " + << mon_caps << " to " << new_caps << dendl; + + bufferlist bl; + ::encode(new_caps, bl); + + KeyServerData::Incremental auth_inc; + auth_inc.name = p->first; + auth_inc.auth = p->second; + auth_inc.auth.caps["mon"] = bl; + auth_inc.op = KeyServerData::AUTH_INC_ADD; + push_cephx_inc(auth_inc); + changed = true; + } } + } + + if (format_version == 1) { + dout(1) << __func__ << " upgrading from format 1 to 2" << dendl; + map<EntityName, EntityAuth>::iterator p; + for (p = mon->key_server.secrets_begin(); + p != mon->key_server.secrets_end(); + ++p) { + string n = p->first.to_str(); + + string newcap; + if (n == "client.admin") { + // admin gets it all + newcap = "allow *"; + } else if (n.find("osd.") == 0 || + n.find("mds.") == 0 || + n.find("mon.") == 0) { + // daemons follow their profile + string type = n.substr(0, 3); + newcap = "allow profile " + type; + } else if (p->second.caps.count("mon")) { + // if there are any mon caps, give them 'r' mgr caps + newcap = "allow r"; + } + + if (newcap.length() > 0) { + dout(5) << " giving " << n << " mgr '" << newcap << "'" << dendl; + bufferlist bl; + ::encode(newcap, bl); - if (new_caps.length() > 0) { - dout(5) << __func__ << " updating " << p->first << " mon cap from " - << mon_caps << " to " << new_caps << dendl; + KeyServerData::Incremental auth_inc; + auth_inc.name = p->first; + auth_inc.auth = p->second; + auth_inc.auth.caps["mgr"] = bl; + auth_inc.op = KeyServerData::AUTH_INC_ADD; + push_cephx_inc(auth_inc); + } - bufferlist bl; - ::encode(new_caps, bl); + if (n.find("mgr.") == 0 && + p->second.caps.count("mon")) { + // the kraken ceph-mgr@.service set the mon cap to 'allow *'. + auto blp = p->second.caps["mon"].begin(); + string oldcaps; + ::decode(oldcaps, blp); + if (oldcaps == "allow *") { + dout(5) << " fixing " << n << " mon cap to 'allow profile mgr'" + << dendl; + bufferlist bl; + ::encode("allow profile mgr", bl); + KeyServerData::Incremental auth_inc; + auth_inc.name = p->first; + auth_inc.auth = p->second; + auth_inc.auth.caps["mon"] = bl; + auth_inc.op = KeyServerData::AUTH_INC_ADD; + push_cephx_inc(auth_inc); + } + } + } + // add bootstrap key + { KeyServerData::Incremental auth_inc; - auth_inc.name = p->first; - auth_inc.auth = p->second; - auth_inc.auth.caps["mon"] = bl; + bool r = auth_inc.name.from_str("client.bootstrap-mgr"); + assert(r); + ::encode("allow profile bootstrap-mgr", auth_inc.auth.caps["mon"]); auth_inc.op = KeyServerData::AUTH_INC_ADD; push_cephx_inc(auth_inc); - changed = true; } + changed = true; } if (changed) { diff --git a/src/mon/ConfigKeyService.cc b/src/mon/ConfigKeyService.cc index df180b7a636..8d083199331 100644 --- a/src/mon/ConfigKeyService.cc +++ b/src/mon/ConfigKeyService.cc @@ -91,7 +91,8 @@ bool ConfigKeyService::service_dispatch(MonOpRequestRef op) dout(10) << __func__ << " " << *m << dendl; if (!in_quorum()) { - dout(1) << __func__ << " not in quorum -- ignore message" << dendl; + dout(1) << __func__ << " not in quorum -- waiting" << dendl; + paxos->wait_for_readable(op, new Monitor::C_RetryMessage(mon, op)); return false; } diff --git a/src/mon/MgrMap.h b/src/mon/MgrMap.h index d9c8e4eb8ce..1fd0500131c 100644 --- a/src/mon/MgrMap.h +++ b/src/mon/MgrMap.h @@ -98,46 +98,57 @@ public: DECODE_FINISH(p); } + void dump(Formatter *f) const { + f->dump_int("epoch", epoch); + f->dump_int("active_gid", get_active_gid()); + f->dump_string("active_name", get_active_name()); + f->dump_stream("active_addr") << active_addr; + f->dump_bool("available", available); + f->open_array_section("standbys"); + for (const auto &i : standbys) { + f->open_object_section("standby"); + f->dump_int("gid", i.second.gid); + f->dump_string("name", i.second.name); + f->close_section(); + } + f->close_section(); + } + + static void generate_test_instances(list<MgrMap*> &l) { + l.push_back(new MgrMap); + } + void print_summary(Formatter *f, std::ostream *ss) const { // One or the other, not both assert((ss != nullptr) != (f != nullptr)); - if (f) { - f->dump_int("active_gid", get_active_gid()); - f->dump_string("active_name", get_active_name()); + dump(f); } else { if (get_active_gid() != 0) { - *ss << "active: " << get_active_name() << " "; + *ss << "active: " << get_active_name() << " "; } else { - *ss << "no daemons active "; - } - } - - - if (f) { - f->open_array_section("standbys"); - for (const auto &i : standbys) { - f->open_object_section("standby"); - f->dump_int("gid", i.second.gid); - f->dump_string("name", i.second.name); - f->close_section(); + *ss << "no daemons active "; } - f->close_section(); - } else { if (standbys.size()) { - *ss << "standbys: "; - bool first = true; - for (const auto &i : standbys) { - if (!first) { - *ss << ", "; - } - *ss << i.second.name; - first = false; - } + *ss << "standbys: "; + bool first = true; + for (const auto &i : standbys) { + if (!first) { + *ss << ", "; + } + *ss << i.second.name; + first = false; + } } } } + + friend ostream& operator<<(ostream& out, const MgrMap& m) { + ostringstream ss; + m.print_summary(nullptr, &ss); + return out << ss.str(); + } }; WRITE_CLASS_ENCODER_FEATURES(MgrMap) diff --git a/src/mon/MgrMonitor.cc b/src/mon/MgrMonitor.cc index d9cee80b463..cdca72a110d 100644 --- a/src/mon/MgrMonitor.cc +++ b/src/mon/MgrMonitor.cc @@ -19,6 +19,7 @@ #include "PGMonitor.h" #include "include/stringify.h" #include "mgr/MgrContext.h" +#include "OSDMonitor.h" #include "MgrMonitor.h" @@ -33,23 +34,30 @@ void MgrMonitor::create_initial() void MgrMonitor::update_from_paxos(bool *need_bootstrap) { version_t version = get_last_committed(); - if (version == map.epoch) { - return; - } + if (version != map.epoch) { + dout(4) << "loading version " << version << dendl; - dout(4) << "loading version " << version << dendl; + bufferlist bl; + int err = get_version(version, bl); + assert(err == 0); - bufferlist bl; - int err = get_version(version, bl); - assert(err == 0); + bufferlist::iterator p = bl.begin(); + map.decode(p); - bufferlist::iterator p = bl.begin(); - map.decode(p); + dout(4) << "active server: " << map.active_addr + << "(" << map.active_gid << ")" << dendl; - dout(4) << "active server: " << map.active_addr - << "(" << map.active_gid << ")" << dendl; + if (map.available) { + first_seen_inactive = utime_t(); + } else { + first_seen_inactive = ceph_clock_now(); + } - check_subs(); + check_subs(); + } + + // feed our pet MgrClient + mon->mgr_client.ms_dispatch(new MMgrMap(map)); } void MgrMonitor::create_pending() @@ -60,12 +68,31 @@ void MgrMonitor::create_pending() void MgrMonitor::encode_pending(MonitorDBStore::TransactionRef t) { + dout(10) << __func__ << " " << pending_map << dendl; bufferlist bl; pending_map.encode(bl, 0); put_version(t, pending_map.epoch, bl); put_last_committed(t, pending_map.epoch); } +bool MgrMonitor::check_caps(MonOpRequestRef op, const uuid_d& fsid) +{ + // check permissions + MonSession *session = op->get_session(); + if (!session) + return false; + if (!session->is_capable("mgr", MON_CAP_X)) { + dout(1) << __func__ << " insufficient caps " << session->caps << dendl; + return false; + } + if (fsid != mon->monmap->fsid) { + dout(1) << __func__ << " op fsid " << fsid + << " != " << mon->monmap->fsid << dendl; + return false; + } + return true; +} + bool MgrMonitor::preprocess_query(MonOpRequestRef op) { PaxosServiceMessage *m = static_cast<PaxosServiceMessage*>(op->get_req()); @@ -122,6 +149,10 @@ bool MgrMonitor::preprocess_beacon(MonOpRequestRef op) MMgrBeacon *m = static_cast<MMgrBeacon*>(op->get_req()); dout(4) << "beacon from " << m->get_gid() << dendl; + if (!check_caps(op, m->get_fsid())) { + return true; + } + last_beacon[m->get_gid()] = ceph_clock_now(); if (pending_map.active_gid == m->get_gid() @@ -171,7 +202,8 @@ bool MgrMonitor::prepare_beacon(MonOpRequestRef op) if (pending_map.active_gid == m->get_gid()) { // A beacon from the currently active daemon if (pending_map.active_addr != m->get_server_addr()) { - dout(4) << "learned address " << m->get_server_addr() << dendl; + dout(4) << "learned address " << m->get_server_addr() + << " (was " << pending_map.active_addr << ")" << dendl; pending_map.active_addr = m->get_server_addr(); updated = true; } @@ -186,10 +218,13 @@ bool MgrMonitor::prepare_beacon(MonOpRequestRef op) if (pending_map.standbys.count(m->get_gid())) { drop_standby(m->get_gid()); } + dout(4) << "selecting new active " << m->get_gid() + << " " << m->get_name() + << " (was " << pending_map.active_gid << " " + << pending_map.active_name << ")" << dendl; pending_map.active_gid = m->get_gid(); pending_map.active_name = m->get_name(); - dout(4) << "selecting new active in epoch " << pending_map.epoch << dendl; updated = true; } else { if (pending_map.standbys.count(m->get_gid()) > 0) { @@ -274,8 +309,47 @@ void MgrMonitor::send_digests() mon->timer.add_event_after(g_conf->mon_mgr_digest_period, digest_callback); } +void MgrMonitor::on_active() +{ + if (mon->is_leader()) + mon->clog->info() << "mgrmap e" << map.epoch << ": " << map; +} + +void MgrMonitor::get_health( + list<pair<health_status_t,string> >& summary, + list<pair<health_status_t,string> > *detail, + CephContext *cct) const +{ + // start mgr warnings as soon as the mons and osds are all upgraded, + // but before the require_luminous osdmap flag is set. this way the + // user gets some warning before the osd flag is set and mgr is + // actually *required*. + if (!mon->monmap->get_required_features().contains_all( + ceph::features::mon::FEATURE_LUMINOUS) || + !HAVE_FEATURE(mon->osdmon()->osdmap.get_up_osd_features(), + SERVER_LUMINOUS)) { + return; + } + + if (!map.available) { + auto level = HEALTH_WARN; + // do not escalate to ERR if they are still upgrading to jewel. + if (mon->osdmon()->osdmap.test_flag(CEPH_OSDMAP_REQUIRE_LUMINOUS)) { + utime_t now = ceph_clock_now(); + if (first_seen_inactive != utime_t() && + now - first_seen_inactive > g_conf->mon_mgr_inactive_grace) { + level = HEALTH_ERR; + } + } + summary.push_back(make_pair(level, "no active mgr")); + } +} + void MgrMonitor::tick() { + if (!is_active() || !mon->is_leader()) + return; + const utime_t now = ceph_clock_now(); utime_t cutoff = now; cutoff -= g_conf->mon_mgr_beacon_grace; diff --git a/src/mon/MgrMonitor.h b/src/mon/MgrMonitor.h index bfddba5943a..a582d95b1f6 100644 --- a/src/mon/MgrMonitor.h +++ b/src/mon/MgrMonitor.h @@ -22,6 +22,8 @@ class MgrMonitor : public PaxosService MgrMap map; MgrMap pending_map; + utime_t first_seen_inactive; + std::map<uint64_t, utime_t> last_beacon; /** @@ -36,6 +38,8 @@ class MgrMonitor : public PaxosService Context *digest_callback; + bool check_caps(MonOpRequestRef op, const uuid_d& fsid); + public: MgrMonitor(Monitor *mn, Paxos *p, const string& service_name) : PaxosService(mn, p, service_name), digest_callback(nullptr) @@ -68,6 +72,10 @@ public: void check_subs(); void send_digests(); + void on_active() override; + void get_health(list<pair<health_status_t,string> >& summary, + list<pair<health_status_t,string> > *detail, + CephContext *cct) const override; void tick() override; void print_summary(Formatter *f, std::ostream *ss) const; diff --git a/src/mon/MonCap.cc b/src/mon/MonCap.cc index b4bd1c5098e..7f6fdab4fd2 100644 --- a/src/mon/MonCap.cc +++ b/src/mon/MonCap.cc @@ -113,12 +113,42 @@ BOOST_FUSION_ADAPT_STRUCT(StringConstraint, // </magic> -void MonCapGrant::expand_profile(EntityName name) const +void MonCapGrant::expand_profile(int daemon_type, const EntityName& name) const { // only generate this list once if (!profile_grants.empty()) return; + if (profile == "read-only") { + // grants READ-ONLY caps monitor-wide + // 'auth' requires MON_CAP_X even for RO, which we do not grant here. + profile_grants.push_back(mon_rwxa_t(MON_CAP_R)); + return; + } + + if (profile == "read-write") { + // grants READ-WRITE caps monitor-wide + // 'auth' requires MON_CAP_X for all operations, which we do not grant. + profile_grants.push_back(mon_rwxa_t(MON_CAP_R | MON_CAP_W)); + return; + } + + switch (daemon_type) { + case CEPH_ENTITY_TYPE_MON: + expand_profile_mon(name); + return; + case CEPH_ENTITY_TYPE_MGR: + expand_profile_mgr(name); + return; + } +} + +void MonCapGrant::expand_profile_mgr(const EntityName& name) const +{ +} + +void MonCapGrant::expand_profile_mon(const EntityName& name) const +{ if (profile == "mon") { profile_grants.push_back(MonCapGrant("mon", MON_CAP_ALL)); profile_grants.push_back(MonCapGrant("log", MON_CAP_ALL)); @@ -137,7 +167,25 @@ void MonCapGrant::expand_profile(EntityName name) const profile_grants.push_back(MonCapGrant("osd pool rmsnap")); profile_grants.push_back(MonCapGrant("log", MON_CAP_W)); } - if (profile == "osd" || profile == "mds" || profile == "mon") { + if (profile == "mgr") { + profile_grants.push_back(MonCapGrant("mgr", MON_CAP_ALL)); + profile_grants.push_back(MonCapGrant("log", MON_CAP_W)); + profile_grants.push_back(MonCapGrant("mon", MON_CAP_R)); + profile_grants.push_back(MonCapGrant("mds", MON_CAP_R)); + profile_grants.push_back(MonCapGrant("osd", MON_CAP_R)); + profile_grants.push_back(MonCapGrant("config-key", MON_CAP_R)); + string prefix = string("daemon-private/mgr/"); + profile_grants.push_back(MonCapGrant("config-key get", "key", + StringConstraint("", prefix))); + profile_grants.push_back(MonCapGrant("config-key put", "key", + StringConstraint("", prefix))); + profile_grants.push_back(MonCapGrant("config-key exists", "key", + StringConstraint("", prefix))); + profile_grants.push_back(MonCapGrant("config-key delete", "key", + StringConstraint("", prefix))); + } + if (profile == "osd" || profile == "mds" || profile == "mon" || + profile == "mgr") { string prefix = string("daemon-private/") + stringify(name) + string("/"); profile_grants.push_back(MonCapGrant("config-key get", "key", StringConstraint("", prefix))); profile_grants.push_back(MonCapGrant("config-key put", "key", StringConstraint("", prefix))); @@ -170,6 +218,14 @@ void MonCapGrant::expand_profile(EntityName name) const profile_grants.back().command_args["caps_osd"] = StringConstraint("allow rwx", ""); profile_grants.back().command_args["caps_mds"] = StringConstraint("allow", ""); } + if (profile == "bootstrap-mgr") { + profile_grants.push_back(MonCapGrant("mon", MON_CAP_R)); // read monmap + profile_grants.push_back(MonCapGrant("osd", MON_CAP_R)); // read osdmap + profile_grants.push_back(MonCapGrant("mon getmap")); + profile_grants.push_back(MonCapGrant("auth get-or-create")); // FIXME: this can expose other mgr keys + profile_grants.back().command_args["entity"] = StringConstraint("", "mgr."); + profile_grants.back().command_args["caps_mon"] = StringConstraint("allow profile mgr", ""); + } if (profile == "bootstrap-rgw") { profile_grants.push_back(MonCapGrant("mon", MON_CAP_R)); // read monmap profile_grants.push_back(MonCapGrant("osd", MON_CAP_R)); // read osdmap @@ -191,18 +247,6 @@ void MonCapGrant::expand_profile(EntityName name) const profile_grants.push_back(MonCapGrant("pg", MON_CAP_R)); } - if (profile == "read-only") { - // grants READ-ONLY caps monitor-wide - // 'auth' requires MON_CAP_X even for RO, which we do not grant here. - profile_grants.push_back(mon_rwxa_t(MON_CAP_R)); - } - - if (profile == "read-write") { - // grants READ-WRITE caps monitor-wide - // 'auth' requires MON_CAP_X for all operations, which we do not grant. - profile_grants.push_back(mon_rwxa_t(MON_CAP_R | MON_CAP_W)); - } - if (profile == "role-definer") { // grants ALL caps to the auth subsystem, read-only on the // monitor subsystem and nothing else. @@ -212,16 +256,17 @@ void MonCapGrant::expand_profile(EntityName name) const } mon_rwxa_t MonCapGrant::get_allowed(CephContext *cct, + int daemon_type, EntityName name, const std::string& s, const std::string& c, const map<string,string>& c_args) const { if (profile.length()) { - expand_profile(name); + expand_profile(daemon_type, name); mon_rwxa_t a; for (list<MonCapGrant>::const_iterator p = profile_grants.begin(); p != profile_grants.end(); ++p) - a = a | p->get_allowed(cct, name, s, c, c_args); + a = a | p->get_allowed(cct, daemon_type, name, s, c, c_args); return a; } if (service.length()) { @@ -278,6 +323,7 @@ void MonCap::set_allow_all() } bool MonCap::is_capable(CephContext *cct, + int daemon_type, EntityName name, const string& service, const string& command, const map<string,string>& command_args, @@ -303,7 +349,8 @@ bool MonCap::is_capable(CephContext *cct, } // check enumerated caps - allow = allow | p->get_allowed(cct, name, service, command, command_args); + allow = allow | p->get_allowed(cct, daemon_type, name, service, command, + command_args); if ((!op_may_read || (allow & MON_CAP_R)) && (!op_may_write || (allow & MON_CAP_W)) && (!op_may_exec || (allow & MON_CAP_X))) { diff --git a/src/mon/MonCap.h b/src/mon/MonCap.h index 3bdd5479838..25684d30762 100644 --- a/src/mon/MonCap.h +++ b/src/mon/MonCap.h @@ -78,7 +78,9 @@ struct MonCapGrant { // needed by expand_profile() (via is_match()) and cached here. mutable list<MonCapGrant> profile_grants; - void expand_profile(EntityName name) const; + void expand_profile(int daemon_type, const EntityName& name) const; + void expand_profile_mon(const EntityName& name) const; + void expand_profile_mgr(const EntityName& name) const; MonCapGrant() : allow(0) {} // cppcheck-suppress noExplicitConstructor @@ -101,6 +103,7 @@ struct MonCapGrant { * @return bits we allow */ mon_rwxa_t get_allowed(CephContext *cct, + int daemon_type, ///< CEPH_ENTITY_TYPE_* EntityName name, const std::string& service, const std::string& command, @@ -138,6 +141,7 @@ struct MonCap { * This method actually checks a description of a particular operation against * what the capability has specified. * + * @param daemon_type CEPH_ENTITY_TYPE_* for the service (MON or MGR) * @param service service name * @param command command id * @param command_args @@ -147,6 +151,7 @@ struct MonCap { * @return true if the operation is allowed, false otherwise */ bool is_capable(CephContext *cct, + int daemon_type, EntityName name, const string& service, const string& command, const map<string,string>& command_args, diff --git a/src/mon/MonCommands.h b/src/mon/MonCommands.h index af44384c900..6e0e70ddecf 100644 --- a/src/mon/MonCommands.h +++ b/src/mon/MonCommands.h @@ -111,6 +111,7 @@ * NOFORWARD - command may not be forwarded * OBSOLETE - command is considered obsolete * DEPRECATED - command is considered deprecated + * MGR - command goes to ceph-mgr (for luminous+) * * A command should always be first considered DEPRECATED before being * considered OBSOLETE, giving due consideration to users and conforming @@ -118,60 +119,16 @@ */ /* - * pg commands PgMonitor.cc + * pg commands PGMonitor.cc */ -COMMAND("pg stat", "show placement group status.", "pg", "r", "cli,rest") -COMMAND("pg getmap", "get binary pg map to -o/stdout", "pg", "r", "cli,rest") -COMMAND("pg dump " \ - "name=dumpcontents,type=CephChoices,strings=all|summary|sum|delta|pools|osds|pgs|pgs_brief,n=N,req=false", \ - "show human-readable versions of pg map (only 'all' valid with plain)", "pg", "r", "cli,rest") -COMMAND("pg dump_json " \ - "name=dumpcontents,type=CephChoices,strings=all|summary|sum|pools|osds|pgs,n=N,req=false", \ - "show human-readable version of pg map in json only",\ - "pg", "r", "cli,rest") -COMMAND("pg dump_pools_json", "show pg pools info in json only",\ - "pg", "r", "cli,rest") -COMMAND("pg dump_stuck " \ - "name=stuckops,type=CephChoices,strings=inactive|unclean|stale|undersized|degraded,n=N,req=false " \ - "name=threshold,type=CephInt,req=false", - "show information about stuck pgs",\ - "pg", "r", "cli,rest") -COMMAND("pg ls-by-pool " \ - "name=poolstr,type=CephString " \ - "name=states,type=CephChoices,strings=active|clean|down|scrubbing|degraded|inconsistent|peering|repair|recovering|backfill_wait|incomplete|stale|remapped|deep_scrub|backfill|backfill_toofull|recovery_wait|undersized|activating|peered,n=N,req=false ", \ - "list pg with pool = [poolname]", "pg", "r", "cli,rest") -COMMAND("pg ls-by-primary " \ - "name=osd,type=CephOsdName " \ - "name=pool,type=CephInt,req=false " \ - "name=states,type=CephChoices,strings=active|clean|down|scrubbing|degraded|inconsistent|peering|repair|recovering|backfill_wait|incomplete|stale|remapped|deep_scrub|backfill|backfill_toofull|recovery_wait|undersized|activating|peered,n=N,req=false ", \ - "list pg with primary = [osd]", "pg", "r", "cli,rest") -COMMAND("pg ls-by-osd " \ - "name=osd,type=CephOsdName " \ - "name=pool,type=CephInt,req=false " \ - "name=states,type=CephChoices,strings=active|clean|down|scrubbing|degraded|inconsistent|peering|repair|recovering|backfill_wait|incomplete|stale|remapped|deep_scrub|backfill|backfill_toofull|recovery_wait|undersized|activating|peered,n=N,req=false ", \ - "list pg on osd [osd]", "pg", "r", "cli,rest") -COMMAND("pg ls " \ - "name=pool,type=CephInt,req=false " \ - "name=states,type=CephChoices,strings=active|clean|down|scrubbing|degraded|inconsistent|peering|repair|recovering|backfill_wait|incomplete|stale|remapped|deep_scrub|backfill|backfill_toofull|recovery_wait|undersized|activating|peered,n=N,req=false ", \ - "list pg with specific pool, osd, state", "pg", "r", "cli,rest") -COMMAND("pg map name=pgid,type=CephPgid", "show mapping of pg to osds", \ - "pg", "r", "cli,rest") -COMMAND("pg scrub name=pgid,type=CephPgid", "start scrub on <pgid>", \ - "pg", "rw", "cli,rest") -COMMAND("pg deep-scrub name=pgid,type=CephPgid", "start deep-scrub on <pgid>", \ - "pg", "rw", "cli,rest") -COMMAND("pg repair name=pgid,type=CephPgid", "start repair on <pgid>", \ - "pg", "rw", "cli,rest") -COMMAND("pg debug " \ - "name=debugop,type=CephChoices,strings=unfound_objects_exist|degraded_pgs_exist", \ - "show debug info about pgs", "pg", "r", "cli,rest") +// note: this should be replaced shortly! COMMAND("pg force_create_pg name=pgid,type=CephPgid", \ "force creation of pg <pgid>", "pg", "rw", "cli,rest") COMMAND("pg set_full_ratio name=ratio,type=CephFloat,range=0.0|1.0", \ "set ratio at which pgs are considered full", "pg", "rw", "cli,rest") COMMAND("pg set_nearfull_ratio name=ratio,type=CephFloat,range=0.0|1.0", \ - "set ratio at which pgs are considered nearly full", \ + "set ratio at which pgs are considered nearly full", \ "pg", "rw", "cli,rest") /* diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc index 24ac15d4bcf..9c4ae3ffe64 100644 --- a/src/mon/Monitor.cc +++ b/src/mon/Monitor.cc @@ -84,6 +84,8 @@ #include "include/compat.h" #include "perfglue/heap_profiler.h" +#include "auth/none/AuthNoneClientHandler.h" + #define dout_subsys ceph_subsys_mon #undef dout_prefix #define dout_prefix _prefix(_dout, this) @@ -106,6 +108,19 @@ MonCommand mon_commands[] = { #define COMMAND_WITH_FLAG(parsesig, helptext, modulename, req_perms, avail, flags) \ {parsesig, helptext, modulename, req_perms, avail, flags}, #include <mon/MonCommands.h> +#undef COMMAND +#undef COMMAND_WITH_FLAG + + // FIXME: slurp up the Mgr commands too + +#define COMMAND(parsesig, helptext, modulename, req_perms, avail) \ + {parsesig, helptext, modulename, req_perms, avail, FLAG(MGR)}, +#define COMMAND_WITH_FLAG(parsesig, helptext, modulename, req_perms, avail, flags) \ + {parsesig, helptext, modulename, req_perms, avail, flags | FLAG(MGR)}, +#include <mgr/MgrCommands.h> +#undef COMMAND +#undef COMMAND_WITH_FLAG + }; @@ -133,7 +148,7 @@ long parse_pos_long(const char *s, ostream *pss) } Monitor::Monitor(CephContext* cct_, string nm, MonitorDBStore *s, - Messenger *m, MonMap *map) : + Messenger *m, Messenger *mgr_m, MonMap *map) : Dispatcher(cct_), name(nm), rank(-1), @@ -141,6 +156,7 @@ Monitor::Monitor(CephContext* cct_, string nm, MonitorDBStore *s, con_self(m ? m->get_loopback_connection() : NULL), lock("Monitor::lock"), timer(cct_, lock), + finisher(cct_, "mon_finisher", "fin"), cpu_tp(cct, "Monitor::cpu_tp", "cpu_tp", g_conf->mon_cpu_threads), has_ever_joined(false), logger(NULL), cluster_logger(NULL), cluster_logger_registered(false), @@ -155,6 +171,8 @@ Monitor::Monitor(CephContext* cct_, string nm, MonitorDBStore *s, cct->_conf->auth_service_required : cct->_conf->auth_supported ), leader_supported_mon_commands(NULL), leader_supported_mon_commands_size(0), + mgr_messenger(mgr_m), + mgr_client(cct_, mgr_m), store(s), state(STATE_PROBING), @@ -819,6 +837,8 @@ int Monitor::init() dout(2) << "init" << dendl; Mutex::Locker l(lock); + finisher.start(); + // start ticker timer.init(); new_tick(); @@ -828,6 +848,10 @@ int Monitor::init() // i'm ready! messenger->add_dispatcher_tail(this); + mgr_client.init(); + mgr_messenger->add_dispatcher_tail(&mgr_client); + mgr_messenger->add_dispatcher_tail(this); // for auth ms_* calls + bootstrap(); // encode command sets @@ -911,6 +935,7 @@ void Monitor::shutdown() dout(1) << "shutdown" << dendl; lock.Lock(); + wait_for_paxos_write(); state = STATE_SHUTDOWN; @@ -938,6 +963,13 @@ void Monitor::shutdown() elector.shutdown(); + mgr_client.shutdown(); + + lock.Unlock(); + finisher.wait_for_empty(); + finisher.stop(); + lock.Lock(); + // clean up paxos->shutdown(); for (vector<PaxosService*>::iterator p = paxos_service.begin(); p != paxos_service.end(); ++p) @@ -971,6 +1003,7 @@ void Monitor::shutdown() lock.Unlock(); messenger->shutdown(); // last thing! ceph_mon.cc will delete mon. + mgr_messenger->shutdown(); } void Monitor::wait_for_paxos_write() @@ -2650,9 +2683,12 @@ bool Monitor::_allowed_command(MonSession *s, string &module, string &prefix, bool cmd_w = this_cmd->requires_perm('w'); bool cmd_x = this_cmd->requires_perm('x'); - bool capable = s->caps.is_capable(g_ceph_context, s->entity_name, - module, prefix, param_str_map, - cmd_r, cmd_w, cmd_x); + bool capable = s->caps.is_capable( + g_ceph_context, + CEPH_ENTITY_TYPE_MON, + s->entity_name, + module, prefix, param_str_map, + cmd_r, cmd_w, cmd_x); dout(10) << __func__ << " " << (capable ? "" : "not ") << "capable" << dendl; return capable; @@ -2661,18 +2697,23 @@ bool Monitor::_allowed_command(MonSession *s, string &module, string &prefix, void Monitor::format_command_descriptions(const MonCommand *commands, unsigned commands_size, Formatter *f, - bufferlist *rdata) + bufferlist *rdata, + bool hide_mgr_flag) { int cmdnum = 0; f->open_object_section("command_descriptions"); for (const MonCommand *cp = commands; cp < &commands[commands_size]; cp++) { + unsigned flags = cp->flags; + if (hide_mgr_flag) { + flags &= ~MonCommand::FLAG_MGR; + } ostringstream secname; secname << "cmd" << setfill('0') << std::setw(3) << cmdnum; dump_cmddesc_to_json(f, secname.str(), cp->cmdstring, cp->helpstring, cp->module, - cp->req_perms, cp->availability, cp->flags); + cp->req_perms, cp->availability, flags); cmdnum++; } f->close_section(); // command_descriptions @@ -2705,6 +2746,21 @@ bool Monitor::is_keyring_required() auth_cluster_required == "cephx"; } +struct C_MgrProxyCommand : public Context { + Monitor *mon; + MonOpRequestRef op; + uint64_t size; + bufferlist outbl; + string outs; + C_MgrProxyCommand(Monitor *mon, MonOpRequestRef op, uint64_t s) + : mon(mon), op(op), size(s) { } + void finish(int r) { + Mutex::Locker l(mon->lock); + mon->mgr_proxy_bytes -= size; + mon->reply_command(op, r, outs, outbl, 0); + } +}; + void Monitor::handle_command(MonOpRequestRef op) { assert(op->is_type_command()); @@ -2751,7 +2807,7 @@ void Monitor::handle_command(MonOpRequestRef op) // check return value. If no prefix parameter provided, // return value will be false, then return error info. - if(!cmd_getval(g_ceph_context, cmdmap, "prefix", prefix)) { + if (!cmd_getval(g_ceph_context, cmdmap, "prefix", prefix)) { reply_command(op, -EINVAL, "command prefix not found", 0); return; } @@ -2765,8 +2821,12 @@ void Monitor::handle_command(MonOpRequestRef op) if (prefix == "get_command_descriptions") { bufferlist rdata; Formatter *f = Formatter::create("json"); + // hide mgr commands until luminous upgrade is complete + bool hide_mgr_flag = + !osdmon()->osdmap.test_flag(CEPH_OSDMAP_REQUIRE_LUMINOUS); format_command_descriptions(leader_supported_mon_commands, - leader_supported_mon_commands_size, f, &rdata); + leader_supported_mon_commands_size, f, &rdata, + hide_mgr_flag); delete f; reply_command(op, 0, "", rdata, 0); return; @@ -2877,6 +2937,30 @@ void Monitor::handle_command(MonOpRequestRef op) << "entity='" << session->entity_name << "' " << "cmd=" << m->cmd << ": dispatch"; + if (mon_cmd->is_mgr() && + osdmon()->osdmap.test_flag(CEPH_OSDMAP_REQUIRE_LUMINOUS)) { + const auto& hdr = m->get_header(); + uint64_t size = hdr.front_len + hdr.middle_len + hdr.data_len; + uint64_t max = + g_conf->mon_client_bytes * g_conf->mon_mgr_proxy_client_bytes_ratio; + if (mgr_proxy_bytes + size > max) { + dout(10) << __func__ << " current mgr proxy bytes " << mgr_proxy_bytes + << " + " << size << " > max " << max << dendl; + reply_command(op, -EAGAIN, "hit limit on proxied mgr commands", rdata, 0); + return; + } + mgr_proxy_bytes += size; + dout(10) << __func__ << " proxying mgr command (+" << size + << " -> " << mgr_proxy_bytes << ")" << dendl; + C_MgrProxyCommand *fin = new C_MgrProxyCommand(this, op, size); + mgr_client.start_command(m->cmd, + m->get_data(), + &fin->outbl, + &fin->outs, + new C_OnFinisher(fin, &finisher)); + return; + } + if (module == "mds" || module == "fs") { mdsmon()->dispatch(op); return; @@ -5230,54 +5314,78 @@ void Monitor::extract_save_mon_key(KeyRing& keyring) } } -bool Monitor::ms_get_authorizer(int service_id, AuthAuthorizer **authorizer, bool force_new) +bool Monitor::ms_get_authorizer(int service_id, AuthAuthorizer **authorizer, + bool force_new) { - dout(10) << "ms_get_authorizer for " << ceph_entity_type_name(service_id) << dendl; + dout(10) << "ms_get_authorizer for " << ceph_entity_type_name(service_id) + << dendl; if (is_shutdown()) return false; - // we only connect to other monitors; every else connects to us. - if (service_id != CEPH_ENTITY_TYPE_MON) + // we only connect to other monitors and mgr; every else connects to us. + if (service_id != CEPH_ENTITY_TYPE_MON && + service_id != CEPH_ENTITY_TYPE_MGR) return false; - if (!auth_cluster_required.is_supported_auth(CEPH_AUTH_CEPHX)) - return false; + if (!auth_cluster_required.is_supported_auth(CEPH_AUTH_CEPHX)) { + // auth_none + dout(20) << __func__ << " building auth_none authorizer" << dendl; + AuthNoneClientHandler handler(g_ceph_context, nullptr); + handler.set_global_id(0); + *authorizer = handler.build_authorizer(service_id); + return true; + } CephXServiceTicketInfo auth_ticket_info; CephXSessionAuthInfo info; int ret; + EntityName name; name.set_type(CEPH_ENTITY_TYPE_MON); - auth_ticket_info.ticket.name = name; auth_ticket_info.ticket.global_id = 0; - CryptoKey secret; - if (!keyring.get_secret(name, secret) && - !key_server.get_secret(name, secret)) { - dout(0) << " couldn't get secret for mon service from keyring or keyserver" << dendl; - stringstream ss, ds; - int err = key_server.list_secrets(ds); - if (err < 0) - ss << "no installed auth entries!"; - else - ss << "installed auth entries:"; - dout(0) << ss.str() << "\n" << ds.str() << dendl; - return false; - } + if (service_id == CEPH_ENTITY_TYPE_MON) { + // mon to mon authentication uses the private monitor shared key and not the + // rotating key + CryptoKey secret; + if (!keyring.get_secret(name, secret) && + !key_server.get_secret(name, secret)) { + dout(0) << " couldn't get secret for mon service from keyring or keyserver" + << dendl; + stringstream ss, ds; + int err = key_server.list_secrets(ds); + if (err < 0) + ss << "no installed auth entries!"; + else + ss << "installed auth entries:"; + dout(0) << ss.str() << "\n" << ds.str() << dendl; + return false; + } - /* mon to mon authentication uses the private monitor shared key and not the - rotating key */ - ret = key_server.build_session_auth_info(service_id, auth_ticket_info, info, secret, (uint64_t)-1); - if (ret < 0) { - dout(0) << "ms_get_authorizer failed to build session auth_info for use with mon ret " << ret << dendl; - return false; + ret = key_server.build_session_auth_info(service_id, auth_ticket_info, info, + secret, (uint64_t)-1); + if (ret < 0) { + dout(0) << __func__ << " failed to build mon session_auth_info " + << cpp_strerror(ret) << dendl; + return false; + } + } else if (service_id == CEPH_ENTITY_TYPE_MGR) { + // mgr + ret = key_server.build_session_auth_info(service_id, auth_ticket_info, info); + if (ret < 0) { + derr << __func__ << " failed to build mgr service session_auth_info " + << cpp_strerror(ret) << dendl; + return false; + } + } else { + ceph_abort(); // see check at top of fn } CephXTicketBlob blob; if (!cephx_build_service_ticket_blob(cct, info, blob)) { - dout(0) << "ms_get_authorizer failed to build service ticket use with mon" << dendl; + dout(0) << "ms_get_authorizer failed to build service ticket" << dendl; return false; } bufferlist ticket_data; @@ -5295,7 +5403,8 @@ bool Monitor::ms_get_authorizer(int service_id, AuthAuthorizer **authorizer, boo } bool Monitor::ms_verify_authorizer(Connection *con, int peer_type, - int protocol, bufferlist& authorizer_data, bufferlist& authorizer_reply, + int protocol, bufferlist& authorizer_data, + bufferlist& authorizer_reply, bool& isvalid, CryptoKey& session_key) { dout(10) << "ms_verify_authorizer " << con->get_peer_addr() diff --git a/src/mon/Monitor.h b/src/mon/Monitor.h index fa230c55749..285997e18aa 100644 --- a/src/mon/Monitor.h +++ b/src/mon/Monitor.h @@ -23,6 +23,9 @@ #ifndef CEPH_MONITOR_H #define CEPH_MONITOR_H +#include <errno.h> +#include <cmath> + #include "include/types.h" #include "msg/Messenger.h" @@ -40,8 +43,7 @@ #include "messages/MMonCommand.h" #include "mon/MonitorDBStore.h" #include "include/memory.h" -#include <errno.h> -#include <cmath> +#include "mgr/MgrClient.h" #include "mon/MonOpRequest.h" #include "common/WorkQueue.h" @@ -122,6 +124,7 @@ public: ConnectionRef con_self; Mutex lock; SafeTimer timer; + Finisher finisher; ThreadPool cpu_tp; ///< threadpool for CPU intensive work /// true if we have ever joined a quorum. if false, we are either a @@ -154,6 +157,10 @@ public: const MonCommand *leader_supported_mon_commands; int leader_supported_mon_commands_size; + Messenger *mgr_messenger; + MgrClient mgr_client; + uint64_t mgr_proxy_bytes = 0; // in-flight proxied mgr command message bytes + private: void new_tick(); friend class C_Mon_Tick; @@ -648,6 +655,7 @@ public: friend class MonmapMonitor; friend class PGMonitor; friend class LogMonitor; + friend class ConfigKeyService; QuorumService *health_monitor; QuorumService *config_key_service; @@ -905,7 +913,7 @@ public: public: Monitor(CephContext *cct_, string nm, MonitorDBStore *s, - Messenger *m, MonMap *map); + Messenger *m, Messenger *mgr_m, MonMap *map); ~Monitor() override; static int check_features(MonitorDBStore *store); @@ -955,7 +963,8 @@ public: static void format_command_descriptions(const MonCommand *commands, unsigned commands_size, Formatter *f, - bufferlist *rdata); + bufferlist *rdata, + bool hide_mgr_flag=false); void get_locally_supported_monitor_commands(const MonCommand **cmds, int *count); /// the Monitor owns this pointer once you pass it in void set_leader_supported_commands(const MonCommand *cmds, int size); @@ -987,7 +996,8 @@ struct MonCommand { static const uint64_t FLAG_NOFORWARD = 1 << 0; static const uint64_t FLAG_OBSOLETE = 1 << 1; static const uint64_t FLAG_DEPRECATED = 1 << 2; - + static const uint64_t FLAG_MGR = 1 << 3; + bool has_flag(uint64_t flag) const { return (flags & flag) != 0; } void set_flag(uint64_t flag) { flags |= flag; } void unset_flag(uint64_t flag) { flags &= ~flag; } @@ -1029,6 +1039,10 @@ struct MonCommand { return has_flag(MonCommand::FLAG_DEPRECATED); } + bool is_mgr() const { + return has_flag(MonCommand::FLAG_MGR); + } + static void encode_array(const MonCommand *cmds, int size, bufferlist &bl) { ENCODE_START(2, 1, bl); uint16_t s = size; diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc index fd737fef60f..543338bdf3d 100644 --- a/src/mon/OSDMonitor.cc +++ b/src/mon/OSDMonitor.cc @@ -2830,7 +2830,10 @@ bool OSDMonitor::preprocess_remove_snaps(MonOpRequestRef op) MonSession *session = m->get_session(); if (!session) goto ignore; - if (!session->caps.is_capable(g_ceph_context, session->entity_name, + if (!session->caps.is_capable( + g_ceph_context, + CEPH_ENTITY_TYPE_MON, + session->entity_name, "osd", "osd pool rmsnap", {}, true, true, false)) { dout(0) << "got preprocess_remove_snaps from entity with insufficient caps " << session->caps << dendl; diff --git a/src/mon/PGMap.cc b/src/mon/PGMap.cc index ae19174e4f5..590b2f545ea 100644 --- a/src/mon/PGMap.cc +++ b/src/mon/PGMap.cc @@ -1112,6 +1112,44 @@ void PGMap::dump_stuck_plain(ostream& ss, int types, utime_t cutoff) const dump_pg_stats_plain(ss, stuck_pg_stats, true); } +int PGMap::dump_stuck_pg_stats( + stringstream &ds, + Formatter *f, + int threshold, + vector<string>& args) const +{ + int stuck_types = 0; + + for (vector<string>::iterator i = args.begin(); i != args.end(); ++i) { + if (*i == "inactive") + stuck_types |= PGMap::STUCK_INACTIVE; + else if (*i == "unclean") + stuck_types |= PGMap::STUCK_UNCLEAN; + else if (*i == "undersized") + stuck_types |= PGMap::STUCK_UNDERSIZED; + else if (*i == "degraded") + stuck_types |= PGMap::STUCK_DEGRADED; + else if (*i == "stale") + stuck_types |= PGMap::STUCK_STALE; + else { + ds << "Unknown type: " << *i << std::endl; + return -EINVAL; + } + } + + utime_t now(ceph_clock_now()); + utime_t cutoff = now - utime_t(threshold, 0); + + if (!f) { + dump_stuck_plain(ds, stuck_types, cutoff); + } else { + dump_stuck(f, stuck_types, cutoff); + f->flush(ds); + } + + return 0; +} + void PGMap::dump_osd_perf_stats(Formatter *f) const { f->open_array_section("osd_perf_infos"); @@ -1742,7 +1780,7 @@ void PGMap::generate_test_instances(list<PGMap*>& o) } void PGMap::get_filtered_pg_stats(uint32_t state, int64_t poolid, int64_t osdid, - bool primary, set<pg_t>& pgs) + bool primary, set<pg_t>& pgs) const { for (ceph::unordered_map<pg_t, pg_stat_t>::const_iterator i = pg_stat.begin(); i != pg_stat.end(); @@ -1757,11 +1795,11 @@ void PGMap::get_filtered_pg_stats(uint32_t state, int64_t poolid, int64_t osdid, } } -void PGMap::dump_filtered_pg_stats(Formatter *f, set<pg_t>& pgs) +void PGMap::dump_filtered_pg_stats(Formatter *f, set<pg_t>& pgs) const { f->open_array_section("pg_stats"); for (set<pg_t>::iterator i = pgs.begin(); i != pgs.end(); ++i) { - pg_stat_t& st = pg_stat[*i]; + const pg_stat_t& st = pg_stat.at(*i); f->open_object_section("pg_stat"); f->dump_stream("pgid") << *i; st.dump(f); @@ -1770,7 +1808,7 @@ void PGMap::dump_filtered_pg_stats(Formatter *f, set<pg_t>& pgs) f->close_section(); } -void PGMap::dump_filtered_pg_stats(ostream& ss, set<pg_t>& pgs) +void PGMap::dump_filtered_pg_stats(ostream& ss, set<pg_t>& pgs) const { TextTable tab; @@ -1797,7 +1835,7 @@ void PGMap::dump_filtered_pg_stats(ostream& ss, set<pg_t>& pgs) tab.define_column("DEEP_SCRUB_STAMP", TextTable::LEFT, TextTable::RIGHT); for (set<pg_t>::iterator i = pgs.begin(); i != pgs.end(); ++i) { - pg_stat_t& st = pg_stat[*i]; + const pg_stat_t& st = pg_stat.at(*i); ostringstream reported; reported << st.reported_epoch << ":" << st.reported_seq; @@ -2079,6 +2117,285 @@ void PGMap::dump_object_stat_sum(TextTable &tbl, Formatter *f, } +int process_pg_map_command( + const string& orig_prefix, + const map<string,cmd_vartype>& orig_cmdmap, + const PGMap& pg_map, + const OSDMap& osdmap, + Formatter *f, + stringstream *ss, + bufferlist *odata) +{ + string prefix = orig_prefix; + map<string,cmd_vartype> cmdmap = orig_cmdmap; + + // perhaps these would be better in the parsing, but it's weird + bool primary = false; + if (prefix == "pg dump_json") { + vector<string> v; + v.push_back(string("all")); + cmd_putval(g_ceph_context, cmdmap, "format", string("json")); + cmd_putval(g_ceph_context, cmdmap, "dumpcontents", v); + prefix = "pg dump"; + } else if (prefix == "pg dump_pools_json") { + vector<string> v; + v.push_back(string("pools")); + cmd_putval(g_ceph_context, cmdmap, "format", string("json")); + cmd_putval(g_ceph_context, cmdmap, "dumpcontents", v); + prefix = "pg dump"; + } else if (prefix == "pg ls-by-primary") { + primary = true; + prefix = "pg ls"; + } else if (prefix == "pg ls-by-osd") { + prefix = "pg ls"; + } else if (prefix == "pg ls-by-pool") { + prefix = "pg ls"; + string poolstr; + cmd_getval(g_ceph_context, cmdmap, "poolstr", poolstr); + int64_t pool = osdmap.lookup_pg_pool_name(poolstr.c_str()); + if (pool < 0) { + *ss << "pool " << poolstr << " does not exist"; + return -ENOENT; + } + cmd_putval(g_ceph_context, cmdmap, "pool", pool); + } + + int r = 0; + stringstream ds; + if (prefix == "pg stat") { + if (f) { + f->open_object_section("pg_summary"); + pg_map.print_oneline_summary(f, NULL); + f->close_section(); + f->flush(ds); + } else { + ds << pg_map; + } + odata->append(ds); + return 0; + } + + if (prefix == "pg getmap") { + pg_map.encode(*odata); + *ss << "got pgmap version " << pg_map.version; + return 0; + } + + if (prefix == "pg dump") { + string val; + vector<string> dumpcontents; + set<string> what; + if (cmd_getval(g_ceph_context, cmdmap, "dumpcontents", dumpcontents)) { + copy(dumpcontents.begin(), dumpcontents.end(), + inserter(what, what.end())); + } + if (what.empty()) + what.insert("all"); + if (f) { + if (what.count("all")) { + f->open_object_section("pg_map"); + pg_map.dump(f); + f->close_section(); + } else if (what.count("summary") || what.count("sum")) { + f->open_object_section("pg_map"); + pg_map.dump_basic(f); + f->close_section(); + } else { + if (what.count("pools")) { + pg_map.dump_pool_stats(f); + } + if (what.count("osds")) { + pg_map.dump_osd_stats(f); + } + if (what.count("pgs")) { + pg_map.dump_pg_stats(f, false); + } + if (what.count("pgs_brief")) { + pg_map.dump_pg_stats(f, true); + } + if (what.count("delta")) { + f->open_object_section("delta"); + pg_map.dump_delta(f); + f->close_section(); + } + } + f->flush(*odata); + } else { + if (what.count("all")) { + pg_map.dump(ds); + } else if (what.count("summary") || what.count("sum")) { + pg_map.dump_basic(ds); + pg_map.dump_pg_sum_stats(ds, true); + pg_map.dump_osd_sum_stats(ds); + } else { + if (what.count("pgs_brief")) { + pg_map.dump_pg_stats(ds, true); + } + bool header = true; + if (what.count("pgs")) { + pg_map.dump_pg_stats(ds, false); + header = false; + } + if (what.count("pools")) { + pg_map.dump_pool_stats(ds, header); + } + if (what.count("osds")) { + pg_map.dump_osd_stats(ds); + } + } + odata->append(ds); + } + *ss << "dumped " << what; + return 0; + } + + if (prefix == "pg ls") { + int64_t osd = -1; + int64_t pool = -1; + vector<string>states; + set<pg_t> pgs; + cmd_getval(g_ceph_context, cmdmap, "pool", pool); + cmd_getval(g_ceph_context, cmdmap, "osd", osd); + cmd_getval(g_ceph_context, cmdmap, "states", states); + if (pool >= 0 && !osdmap.have_pg_pool(pool)) { + *ss << "pool " << pool << " does not exist"; + return -ENOENT; + } + if (osd >= 0 && !osdmap.is_up(osd)) { + *ss << "osd " << osd << " is not up"; + return -EAGAIN; + } + if (states.empty()) + states.push_back("all"); + + uint32_t state = 0; + + while (!states.empty()) { + string state_str = states.back(); + + if (state_str == "all") { + state = -1; + break; + } else { + int filter = pg_string_state(state_str); + assert(filter != -1); + state |= filter; + } + + states.pop_back(); + } + + pg_map.get_filtered_pg_stats(state, pool, osd, primary, pgs); + + if (f && !pgs.empty()) { + pg_map.dump_filtered_pg_stats(f, pgs); + f->flush(*odata); + } else if (!pgs.empty()) { + pg_map.dump_filtered_pg_stats(ds, pgs); + odata->append(ds); + } + return 0; + } + + if (prefix == "pg dump_stuck") { + vector<string> stuckop_vec; + cmd_getval(g_ceph_context, cmdmap, "stuckops", stuckop_vec); + if (stuckop_vec.empty()) + stuckop_vec.push_back("unclean"); + int64_t threshold; + cmd_getval(g_ceph_context, cmdmap, "threshold", threshold, + int64_t(g_conf->mon_pg_stuck_threshold)); + + r = pg_map.dump_stuck_pg_stats(ds, f, (int)threshold, stuckop_vec); + odata->append(ds); + if (r < 0) + *ss << "failed"; + else + *ss << "ok"; + return 0; + } + + if (prefix == "pg map") { + pg_t pgid; + string pgidstr; + cmd_getval(g_ceph_context, cmdmap, "pgid", pgidstr); + if (!pgid.parse(pgidstr.c_str())) { + *ss << "invalid pgid '" << pgidstr << "'"; + return -EINVAL; + } + vector<int> up, acting; + if (!osdmap.have_pg_pool(pgid.pool())) { + *ss << "pg '" << pgidstr << "' does not exist"; + return -ENOENT; + } + pg_t mpgid = osdmap.raw_pg_to_pg(pgid); + osdmap.pg_to_up_acting_osds(pgid, up, acting); + if (f) { + f->open_object_section("pg_map"); + f->dump_unsigned("epoch", osdmap.get_epoch()); + f->dump_stream("raw_pgid") << pgid; + f->dump_stream("pgid") << mpgid; + + f->open_array_section("up"); + for (vector<int>::iterator it = up.begin(); it != up.end(); ++it) + f->dump_int("up_osd", *it); + f->close_section(); + + f->open_array_section("acting"); + for (vector<int>::iterator it = acting.begin(); it != acting.end(); ++it) + f->dump_int("acting_osd", *it); + f->close_section(); + + f->close_section(); + f->flush(*odata); + } else { + ds << "osdmap e" << osdmap.get_epoch() + << " pg " << pgid << " (" << mpgid << ")" + << " -> up " << up << " acting " << acting; + odata->append(ds); + } + return 0; + } + + if (prefix == "pg debug") { + string debugop; + cmd_getval(g_ceph_context, cmdmap, "debugop", debugop, + string("unfound_objects_exist")); + if (debugop == "unfound_objects_exist") { + bool unfound_objects_exist = false; + for (const auto& p : pg_map.pg_stat) { + if (p.second.stats.sum.num_objects_unfound > 0) { + unfound_objects_exist = true; + break; + } + } + if (unfound_objects_exist) + ds << "TRUE"; + else + ds << "FALSE"; + odata->append(ds); + return 0; + } + if (debugop == "degraded_pgs_exist") { + bool degraded_pgs_exist = false; + for (const auto& p : pg_map.pg_stat) { + if (p.second.stats.sum.num_objects_degraded > 0) { + degraded_pgs_exist = true; + break; + } + } + if (degraded_pgs_exist) + ds << "TRUE"; + else + ds << "FALSE"; + odata->append(ds); + return 0; + } + } + + return -EOPNOTSUPP; +} + void PGMapUpdater::check_osd_map(const OSDMap::Incremental &osd_inc, std::set<int> *need_check_down_pg_osds, std::map<int,utime_t> *last_osd_report, @@ -2175,7 +2492,7 @@ void PGMapUpdater::register_pg( stats.last_deep_scrub_stamp = ps.last_deep_scrub_stamp; stats.last_clean_scrub_stamp = ps.last_clean_scrub_stamp; } else { - utime_t now = ceph_clock_now(); + utime_t now = osd_map.get_modified(); stats.last_fresh = now; stats.last_active = now; stats.last_change = now; diff --git a/src/mon/PGMap.h b/src/mon/PGMap.h index 14802e7ea73..80b41b93110 100644 --- a/src/mon/PGMap.h +++ b/src/mon/PGMap.h @@ -289,7 +289,7 @@ public: void dump_pool_stats(Formatter *f) const; void dump_osd_stats(Formatter *f) const; void dump_delta(Formatter *f) const; - void dump_filtered_pg_stats(Formatter *f, set<pg_t>& pgs); + void dump_filtered_pg_stats(Formatter *f, set<pg_t>& pgs) const; void dump_pg_stats_plain(ostream& ss, const ceph::unordered_map<pg_t, pg_stat_t>& pg_stats, @@ -299,7 +299,10 @@ public: bool get_stuck_counts(const utime_t cutoff, map<string, int>& note) const; void dump_stuck(Formatter *f, int types, utime_t cutoff) const; void dump_stuck_plain(ostream& ss, int types, utime_t cutoff) const; - + int dump_stuck_pg_stats(stringstream &ds, + Formatter *f, + int threshold, + vector<string>& args) const; void dump(ostream& ss) const; void dump_basic(ostream& ss) const; void dump_pg_stats(ostream& ss, bool brief) const; @@ -307,7 +310,7 @@ public: void dump_pool_stats(ostream& ss, bool header) const; void dump_osd_stats(ostream& ss) const; void dump_osd_sum_stats(ostream& ss) const; - void dump_filtered_pg_stats(ostream& ss, set<pg_t>& pgs); + void dump_filtered_pg_stats(ostream& ss, set<pg_t>& pgs) const; void dump_osd_perf_stats(Formatter *f) const; void print_osd_perf_stats(std::ostream *ss) const; @@ -316,7 +319,7 @@ public: void print_osd_blocked_by_stats(std::ostream *ss) const; void get_filtered_pg_stats(uint32_t state, int64_t poolid, int64_t osdid, - bool primary, set<pg_t>& pgs); + bool primary, set<pg_t>& pgs) const; void recovery_summary(Formatter *f, list<string> *psl, const pool_stat_t& delta_sum) const; void overall_recovery_summary(Formatter *f, list<string> *psl) const; @@ -386,6 +389,15 @@ inline ostream& operator<<(ostream& out, const PGMap& m) { return out; } +int process_pg_map_command( + const string& prefix, + const map<string,cmd_vartype>& cmdmap, + const PGMap& pg_map, + const OSDMap& osdmap, + Formatter *f, + stringstream *ss, + bufferlist *odata); + class PGMapUpdater { public: diff --git a/src/mon/PGMonitor.cc b/src/mon/PGMonitor.cc index 617ab1a1878..6669ffcd4b3 100644 --- a/src/mon/PGMonitor.cc +++ b/src/mon/PGMonitor.cc @@ -952,7 +952,6 @@ bool PGMonitor::preprocess_command(MonOpRequestRef op) int r = -1; bufferlist rdata; stringstream ss, ds; - bool primary = false; if (m->fsid != mon->monmap->fsid) { dout(0) << __func__ << " drop message on fsid " << m->fsid << " != " @@ -977,228 +976,13 @@ bool PGMonitor::preprocess_command(MonOpRequestRef op) return true; } - // perhaps these would be better in the parsing, but it's weird - if (prefix == "pg dump_json") { - vector<string> v; - v.push_back(string("all")); - cmd_putval(g_ceph_context, cmdmap, "format", string("json")); - cmd_putval(g_ceph_context, cmdmap, "dumpcontents", v); - prefix = "pg dump"; - } else if (prefix == "pg dump_pools_json") { - vector<string> v; - v.push_back(string("pools")); - cmd_putval(g_ceph_context, cmdmap, "format", string("json")); - cmd_putval(g_ceph_context, cmdmap, "dumpcontents", v); - prefix = "pg dump"; - } else if (prefix == "pg ls-by-primary") { - primary = true; - prefix = "pg ls"; - } else if (prefix == "pg ls-by-osd") { - prefix = "pg ls"; - } else if (prefix == "pg ls-by-pool") { - prefix = "pg ls"; - string poolstr; - cmd_getval(g_ceph_context, cmdmap, "poolstr", poolstr); - int64_t pool = mon->osdmon()->osdmap.lookup_pg_pool_name(poolstr.c_str()); - if (pool < 0) { - r = -ENOENT; - ss << "pool " << poolstr << " does not exist"; - string rs = ss.str(); - mon->reply_command(op, r, rs, get_last_committed()); - return true; - } - cmd_putval(g_ceph_context, cmdmap, "pool", pool); - } - - string format; cmd_getval(g_ceph_context, cmdmap, "format", format, string("plain")); boost::scoped_ptr<Formatter> f(Formatter::create(format)); - if (prefix == "pg stat") { - if (f) { - f->open_object_section("pg_summary"); - pg_map.print_oneline_summary(f.get(), NULL); - f->close_section(); - f->flush(ds); - } else { - ds << pg_map; - } - r = 0; - } else if (prefix == "pg getmap") { - pg_map.encode(rdata); - ss << "got pgmap version " << pg_map.version; - r = 0; - } else if (prefix == "pg dump") { - string val; - vector<string> dumpcontents; - set<string> what; - if (cmd_getval(g_ceph_context, cmdmap, "dumpcontents", dumpcontents)) { - copy(dumpcontents.begin(), dumpcontents.end(), - inserter(what, what.end())); - } - if (what.empty()) - what.insert("all"); - if (f) { - if (what.count("all")) { - f->open_object_section("pg_map"); - pg_map.dump(f.get()); - f->close_section(); - } else if (what.count("summary") || what.count("sum")) { - f->open_object_section("pg_map"); - pg_map.dump_basic(f.get()); - f->close_section(); - } else { - if (what.count("pools")) { - pg_map.dump_pool_stats(f.get()); - } - if (what.count("osds")) { - pg_map.dump_osd_stats(f.get()); - } - if (what.count("pgs")) { - pg_map.dump_pg_stats(f.get(), false); - } - if (what.count("pgs_brief")) { - pg_map.dump_pg_stats(f.get(), true); - } - if (what.count("delta")) { - f->open_object_section("delta"); - pg_map.dump_delta(f.get()); - f->close_section(); - } - } - f->flush(ds); - } else { - if (what.count("all")) { - pg_map.dump(ds); - } else if (what.count("summary") || what.count("sum")) { - pg_map.dump_basic(ds); - pg_map.dump_pg_sum_stats(ds, true); - pg_map.dump_osd_sum_stats(ds); - } else { - if (what.count("pgs_brief")) { - pg_map.dump_pg_stats(ds, true); - } - bool header = true; - if (what.count("pgs")) { - pg_map.dump_pg_stats(ds, false); - header = false; - } - if (what.count("pools")) { - pg_map.dump_pool_stats(ds, header); - } - if (what.count("osds")) { - pg_map.dump_osd_stats(ds); - } - } - } - ss << "dumped " << what << " in format " << format; - r = 0; - } else if (prefix == "pg ls") { - int64_t osd = -1; - int64_t pool = -1; - vector<string>states; - set<pg_t> pgs; - cmd_getval(g_ceph_context, cmdmap, "pool", pool); - cmd_getval(g_ceph_context, cmdmap, "osd", osd); - cmd_getval(g_ceph_context, cmdmap, "states", states); - if (pool >= 0 && !mon->osdmon()->osdmap.have_pg_pool(pool)) { - r = -ENOENT; - ss << "pool " << pool << " does not exist"; - goto reply; - } - if (osd >= 0 && !mon->osdmon()->osdmap.is_up(osd)) { - ss << "osd " << osd << " is not up"; - r = -EAGAIN; - goto reply; - } - if (states.empty()) - states.push_back("all"); - - uint32_t state = 0; - - while (!states.empty()) { - string state_str = states.back(); - - if (state_str == "all") { - state = -1; - break; - } else { - int filter = pg_string_state(state_str); - assert(filter != -1); - state |= filter; - } - - states.pop_back(); - } - - pg_map.get_filtered_pg_stats(state, pool, osd, primary, pgs); - - if (f && !pgs.empty()) { - pg_map.dump_filtered_pg_stats(f.get(), pgs); - f->flush(ds); - } else if (!pgs.empty()) { - pg_map.dump_filtered_pg_stats(ds, pgs); - } - r = 0; - } else if (prefix == "pg dump_stuck") { - vector<string> stuckop_vec; - cmd_getval(g_ceph_context, cmdmap, "stuckops", stuckop_vec); - if (stuckop_vec.empty()) - stuckop_vec.push_back("unclean"); - int64_t threshold; - cmd_getval(g_ceph_context, cmdmap, "threshold", threshold, - int64_t(g_conf->mon_pg_stuck_threshold)); - - r = dump_stuck_pg_stats(ds, f.get(), (int)threshold, stuckop_vec); - if (r < 0) - ss << "failed"; - else - ss << "ok"; - } else if (prefix == "pg map") { - pg_t pgid; - string pgidstr; - cmd_getval(g_ceph_context, cmdmap, "pgid", pgidstr); - if (!pgid.parse(pgidstr.c_str())) { - ss << "invalid pgid '" << pgidstr << "'"; - r = -EINVAL; - goto reply; - } - vector<int> up, acting; - if (!mon->osdmon()->osdmap.have_pg_pool(pgid.pool())) { - r = -ENOENT; - ss << "pg '" << pgidstr << "' does not exist"; - goto reply; - } - pg_t mpgid = mon->osdmon()->osdmap.raw_pg_to_pg(pgid); - mon->osdmon()->osdmap.pg_to_up_acting_osds(pgid, up, acting); - if (f) { - f->open_object_section("pg_map"); - f->dump_unsigned("epoch", mon->osdmon()->osdmap.get_epoch()); - f->dump_stream("raw_pgid") << pgid; - f->dump_stream("pgid") << mpgid; - - f->open_array_section("up"); - for (vector<int>::iterator it = up.begin(); it != up.end(); ++it) - f->dump_int("up_osd", *it); - f->close_section(); - - f->open_array_section("acting"); - for (vector<int>::iterator it = acting.begin(); it != acting.end(); ++it) - f->dump_int("acting_osd", *it); - f->close_section(); - - f->close_section(); - f->flush(ds); - } else { - ds << "osdmap e" << mon->osdmon()->osdmap.get_epoch() - << " pg " << pgid << " (" << mpgid << ")" - << " -> up " << up << " acting " << acting; - } - r = 0; - } else if (prefix == "pg scrub" || - prefix == "pg repair" || - prefix == "pg deep-scrub") { + if (prefix == "pg scrub" || + prefix == "pg repair" || + prefix == "pg deep-scrub") { string scrubop = prefix.substr(3, string::npos); pg_t pgid; string pgidstr; @@ -1232,43 +1016,12 @@ bool PGMonitor::preprocess_command(MonOpRequestRef op) mon->osdmon()->osdmap.get_inst(osd)); ss << "instructing pg " << pgid << " on osd." << osd << " to " << scrubop; r = 0; - } else if (prefix == "pg debug") { - string debugop; - cmd_getval(g_ceph_context, cmdmap, "debugop", debugop, string("unfound_objects_exist")); - if (debugop == "unfound_objects_exist") { - bool unfound_objects_exist = false; - ceph::unordered_map<pg_t,pg_stat_t>::const_iterator end = pg_map.pg_stat.end(); - for (ceph::unordered_map<pg_t,pg_stat_t>::const_iterator s = pg_map.pg_stat.begin(); - s != end; ++s) { - if (s->second.stats.sum.num_objects_unfound > 0) { - unfound_objects_exist = true; - break; - } - } - if (unfound_objects_exist) - ds << "TRUE"; - else - ds << "FALSE"; - r = 0; - } else if (debugop == "degraded_pgs_exist") { - bool degraded_pgs_exist = false; - ceph::unordered_map<pg_t,pg_stat_t>::const_iterator end = pg_map.pg_stat.end(); - for (ceph::unordered_map<pg_t,pg_stat_t>::const_iterator s = pg_map.pg_stat.begin(); - s != end; ++s) { - if (s->second.stats.sum.num_objects_degraded > 0) { - degraded_pgs_exist = true; - break; - } - } - if (degraded_pgs_exist) - ds << "TRUE"; - else - ds << "FALSE"; - r = 0; - } + } else { + r = process_pg_map_command(prefix, cmdmap, pg_map, mon->osdmon()->osdmap, + f.get(), &ss, &rdata); } - if (r == -1) + if (r == -EOPNOTSUPP) return false; reply: @@ -1880,43 +1633,6 @@ void PGMonitor::check_full_osd_health(list<pair<health_status_t,string> >& summa } } -int PGMonitor::dump_stuck_pg_stats(stringstream &ds, - Formatter *f, - int threshold, - vector<string>& args) const -{ - int stuck_types = 0; - - for (vector<string>::iterator i = args.begin(); i != args.end(); ++i) { - if (*i == "inactive") - stuck_types |= PGMap::STUCK_INACTIVE; - else if (*i == "unclean") - stuck_types |= PGMap::STUCK_UNCLEAN; - else if (*i == "undersized") - stuck_types |= PGMap::STUCK_UNDERSIZED; - else if (*i == "degraded") - stuck_types |= PGMap::STUCK_DEGRADED; - else if (*i == "stale") - stuck_types |= PGMap::STUCK_STALE; - else { - ds << "Unknown type: " << *i << std::endl; - return -EINVAL; - } - } - - utime_t now(ceph_clock_now()); - utime_t cutoff = now - utime_t(threshold, 0); - - if (!f) { - pg_map.dump_stuck_plain(ds, stuck_types, cutoff); - } else { - pg_map.dump_stuck(f, stuck_types, cutoff); - f->flush(ds); - } - - return 0; -} - void PGMonitor::check_subs() { dout(10) << __func__ << dendl; diff --git a/src/mon/PGMonitor.h b/src/mon/PGMonitor.h index dd53b6fe2f5..5f7650b0357 100644 --- a/src/mon/PGMonitor.h +++ b/src/mon/PGMonitor.h @@ -89,15 +89,6 @@ private: epoch_t send_pg_creates(int osd, Connection *con, epoch_t next); - /** - * Dump stats from pgs stuck in specified states. - * - * @return 0 on success, negative error code on failure - */ - int dump_stuck_pg_stats(stringstream &ds, Formatter *f, - int threshold, - vector<string>& args) const; - public: PGMonitor(Monitor *mn, Paxos *p, const string& service_name) : PaxosService(mn, p, service_name), diff --git a/src/mon/Session.h b/src/mon/Session.h index 0f253fce3b5..75051bf6312 100644 --- a/src/mon/Session.h +++ b/src/mon/Session.h @@ -78,10 +78,12 @@ struct MonSession : public RefCountedObject { bool is_capable(string service, int mask) { map<string,string> args; - return caps.is_capable(g_ceph_context, - entity_name, - service, "", args, - mask & MON_CAP_R, mask & MON_CAP_W, mask & MON_CAP_X); + return caps.is_capable( + g_ceph_context, + CEPH_ENTITY_TYPE_MON, + entity_name, + service, "", args, + mask & MON_CAP_R, mask & MON_CAP_W, mask & MON_CAP_X); } }; diff --git a/src/msg/Connection.h b/src/msg/Connection.h index 76910b65b7b..94e934c55f1 100644 --- a/src/msg/Connection.h +++ b/src/msg/Connection.h @@ -155,6 +155,7 @@ public: void set_peer_type(int t) { peer_type = t; } bool peer_is_mon() const { return peer_type == CEPH_ENTITY_TYPE_MON; } + bool peer_is_mgr() const { return peer_type == CEPH_ENTITY_TYPE_MGR; } bool peer_is_mds() const { return peer_type == CEPH_ENTITY_TYPE_MDS; } bool peer_is_osd() const { return peer_type == CEPH_ENTITY_TYPE_OSD; } bool peer_is_client() const { return peer_type == CEPH_ENTITY_TYPE_CLIENT; } diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index b9e2828c0ad..4e3afa6e174 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -6514,7 +6514,7 @@ void OSD::handle_pg_scrub(MOSDScrub *m, PG *pg) void OSD::handle_scrub(MOSDScrub *m) { dout(10) << "handle_scrub " << *m << dendl; - if (!require_mon_peer(m)) { + if (!require_mon_or_mgr_peer(m)) { m->put(); return; } @@ -7559,6 +7559,18 @@ bool OSD::require_mon_peer(const Message *m) return true; } +bool OSD::require_mon_or_mgr_peer(const Message *m) +{ + if (!m->get_connection()->peer_is_mon() && + !m->get_connection()->peer_is_mgr()) { + dout(0) << "require_mon_or_mgr_peer received from non-mon, non-mgr " + << m->get_connection()->get_peer_addr() + << " " << *m << dendl; + return false; + } + return true; +} + bool OSD::require_osd_peer(const Message *m) { if (!m->get_connection()->peer_is_osd()) { diff --git a/src/osd/OSD.h b/src/osd/OSD.h index 9e95a01ddce..7efcd376cb4 100644 --- a/src/osd/OSD.h +++ b/src/osd/OSD.h @@ -2182,6 +2182,7 @@ protected: OSDMapRef map); bool require_mon_peer(const Message *m); + bool require_mon_or_mgr_peer(const Message *m); bool require_osd_peer(const Message *m); /*** * Verifies that we were alive in the given epoch, and that diff --git a/src/pybind/ceph_argparse.py b/src/pybind/ceph_argparse.py index b60e71fb1b5..9328b6c47d8 100644 --- a/src/pybind/ceph_argparse.py +++ b/src/pybind/ceph_argparse.py @@ -23,6 +23,9 @@ import threading import uuid +FLAG_MGR = 8 # command is intended for mgr + + try: basestring except NameError: @@ -884,9 +887,9 @@ def store_arg(desc, d): d[desc.name] = desc.instance.val -def validate(args, signature, partial=False): +def validate(args, signature, flags=0, partial=False): """ - validate(args, signature, partial=False) + validate(args, signature, flags=0, partial=False) args is a list of either words or k,v pairs representing a possible command input following format of signature. Runs a validation; no @@ -982,6 +985,9 @@ def validate(args, signature, partial=False): print(save_exception[0], 'not valid: ', save_exception[1], file=sys.stderr) raise ArgumentError("unused arguments: " + str(myargs)) + if flags & FLAG_MGR: + d['target'] = ('mgr','') + # Finally, success return d @@ -1037,7 +1043,7 @@ def validate_command(sigdict, args, verbose=False): for cmd in cmdsig.values(): sig = cmd['sig'] try: - valid_dict = validate(args, sig) + valid_dict = validate(args, sig, flags=cmd.get('flags', 0)) found = cmd break except ArgumentPrefix: @@ -1078,7 +1084,7 @@ def find_cmd_target(childargs): should be sent to a monitor or an osd. We do this before even asking for the 'real' set of command signatures, so we can ask the right daemon. - Returns ('osd', osdid), ('pg', pgid), or ('mon', '') + Returns ('osd', osdid), ('pg', pgid), ('mgr', '') or ('mon', '') """ sig = parse_funcsig(['tell', {'name': 'target', 'type': 'CephName'}]) try: @@ -1307,6 +1313,8 @@ def json_command(cluster, target=('mon', ''), prefix=None, argdict=None, cmddict.update({'prefix': prefix}) if argdict: cmddict.update(argdict) + if 'target' in argdict: + target = argdict.get('target') # grab prefix for error messages prefix = cmddict['prefix'] diff --git a/src/test/common/get_command_descriptions.cc b/src/test/common/get_command_descriptions.cc index 1b4da3f8959..236122ae798 100644 --- a/src/test/common/get_command_descriptions.cc +++ b/src/test/common/get_command_descriptions.cc @@ -63,7 +63,19 @@ static void all() #define COMMAND_WITH_FLAG(parsesig, helptext, modulename, req_perms, avail, flags) \ {parsesig, helptext, modulename, req_perms, avail, flags}, #include <mon/MonCommands.h> - }; +#undef COMMAND +#undef COMMAND_WITH_FLAG + + // FIXME: slurp up the Mgr commands too + +#define COMMAND(parsesig, helptext, modulename, req_perms, avail) \ + {parsesig, helptext, modulename, req_perms, avail, FLAG(MGR)}, +#define COMMAND_WITH_FLAG(parsesig, helptext, modulename, req_perms, avail, flags) \ + {parsesig, helptext, modulename, req_perms, avail, flags | FLAG(MGR)}, +#include <mgr/MgrCommands.h> + #undef COMMAND +#undef COMMAND_WITH_FLAG + }; json_print(mon_commands, ARRAY_SIZE(mon_commands)); } diff --git a/src/test/encoding/types.h b/src/test/encoding/types.h index 6be33fff761..874b8250cbb 100644 --- a/src/test/encoding/types.h +++ b/src/test/encoding/types.h @@ -154,6 +154,9 @@ TYPE_FEATUREFUL(MonMap) #include "mon/MonCap.h" TYPE(MonCap) +#include "mon/MgrMap.h" +TYPE_FEATUREFUL(MgrMap) + #include "mon/mon_types.h" TYPE(LevelDBStoreStats) diff --git a/src/test/erasure-code/test-erasure-code-plugins.sh b/src/test/erasure-code/test-erasure-code-plugins.sh index 48c23195f31..ffba85b7f80 100755 --- a/src/test/erasure-code/test-erasure-code-plugins.sh +++ b/src/test/erasure-code/test-erasure-code-plugins.sh @@ -43,6 +43,7 @@ function TEST_preload_warning() { for plugin in ${legacy_jerasure_plugins[*]} ${legacy_shec_plugins[*]}; do setup $dir || return 1 run_mon $dir a --osd_erasure_code_plugins="${plugin}" || return 1 + run_mgr $dir x || return 1 CEPH_ARGS='' ceph --admin-daemon $dir/ceph-mon.a.asok log flush || return 1 run_osd $dir 0 --osd_erasure_code_plugins="${plugin}" || return 1 CEPH_ARGS='' ceph --admin-daemon $dir/ceph-osd.0.asok log flush || return 1 @@ -59,6 +60,7 @@ function TEST_preload_no_warning() { for plugin in ${plugins[*]}; do setup $dir || return 1 run_mon $dir a --osd_erasure_code_plugins="${plugin}" || return 1 + run_mgr $dir x || return 1 CEPH_ARGS='' ceph --admin-daemon $dir/ceph-mon.a.asok log flush || return 1 run_osd $dir 0 --osd_erasure_code_plugins="${plugin}" || return 1 CEPH_ARGS='' ceph --admin-daemon $dir/ceph-osd.0.asok log flush || return 1 @@ -76,6 +78,7 @@ function TEST_preload_no_warning_default() { setup $dir || return 1 run_mon $dir a || return 1 CEPH_ARGS='' ceph --admin-daemon $dir/ceph-mon.a.asok log flush || return 1 + run_mgr $dir x || return 1 run_osd $dir 0 || return 1 CEPH_ARGS='' ceph --admin-daemon $dir/ceph-osd.0.asok log flush || return 1 ! grep "WARNING: osd_erasure_code_plugins" $dir/mon.a.log || return 1 @@ -90,6 +93,7 @@ function TEST_ec_profile_warning() { setup $dir || return 1 run_mon $dir a || return 1 + run_mgr $dir x || return 1 for id in $(seq 0 2) ; do run_osd $dir $id || return 1 done diff --git a/src/test/erasure-code/test-erasure-code.sh b/src/test/erasure-code/test-erasure-code.sh index cec40eca815..b4417f664f6 100755 --- a/src/test/erasure-code/test-erasure-code.sh +++ b/src/test/erasure-code/test-erasure-code.sh @@ -30,6 +30,7 @@ function run() { setup $dir || return 1 run_mon $dir a || return 1 + run_mgr $dir x || return 1 # check that erasure code plugins are preloaded CEPH_ARGS='' ceph --admin-daemon $dir/ceph-mon.a.asok log flush || return 1 grep 'load: jerasure.*lrc' $dir/mon.a.log || return 1 diff --git a/src/test/erasure-code/test-erasure-eio.sh b/src/test/erasure-code/test-erasure-eio.sh index b78b420d3ec..710b20343e9 100755 --- a/src/test/erasure-code/test-erasure-eio.sh +++ b/src/test/erasure-code/test-erasure-eio.sh @@ -32,6 +32,7 @@ function run() { for func in $funcs ; do setup $dir || return 1 run_mon $dir a || return 1 + run_mgr $dir x || return 1 # check that erasure code plugins are preloaded CEPH_ARGS='' ceph --admin-daemon $dir/ceph-mon.a.asok log flush || return 1 grep 'load: jerasure.*lrc' $dir/mon.a.log || return 1 diff --git a/src/test/mon/moncap.cc b/src/test/mon/moncap.cc index f9a0b5c06e7..8b55719fe16 100644 --- a/src/test/mon/moncap.cc +++ b/src/test/mon/moncap.cc @@ -177,7 +177,7 @@ TEST(MonCap, AllowAll) { ASSERT_TRUE(cap.parse("allow *", NULL)); ASSERT_TRUE(cap.is_allow_all()); - ASSERT_TRUE(cap.is_capable(NULL, EntityName(), + ASSERT_TRUE(cap.is_capable(NULL, CEPH_ENTITY_TYPE_MON, EntityName(), "foo", "asdf", map<string,string>(), true, true, true)); MonCap cap2; @@ -195,29 +195,46 @@ TEST(MonCap, ProfileOSD) { name.from_str("osd.123"); map<string,string> ca; - ASSERT_TRUE(cap.is_capable(NULL, name, "osd", "", ca, true, false, false)); - ASSERT_TRUE(cap.is_capable(NULL, name, "osd", "", ca, true, true, false)); - ASSERT_TRUE(cap.is_capable(NULL, name, "osd", "", ca, true, true, true)); - ASSERT_TRUE(cap.is_capable(NULL, name, "osd", "", ca, true, true, true)); - ASSERT_TRUE(cap.is_capable(NULL, name, "mon", "", ca, true, false,false)); - - ASSERT_FALSE(cap.is_capable(NULL, name, "mds", "", ca, true, true, true)); - ASSERT_FALSE(cap.is_capable(NULL, name, "mon", "", ca, true, true, true)); + ASSERT_TRUE(cap.is_capable(NULL, CEPH_ENTITY_TYPE_MON, + name, "osd", "", ca, true, false, false)); + ASSERT_TRUE(cap.is_capable(NULL, CEPH_ENTITY_TYPE_MON, + name, "osd", "", ca, true, true, false)); + ASSERT_TRUE(cap.is_capable(NULL, CEPH_ENTITY_TYPE_MON, + name, "osd", "", ca, true, true, true)); + ASSERT_TRUE(cap.is_capable(NULL, CEPH_ENTITY_TYPE_MON, + name, "osd", "", ca, true, true, true)); + ASSERT_TRUE(cap.is_capable(NULL, CEPH_ENTITY_TYPE_MON, + name, "mon", "", ca, true, false,false)); + + ASSERT_FALSE(cap.is_capable(NULL, CEPH_ENTITY_TYPE_MON, + name, "mds", "", ca, true, true, true)); + ASSERT_FALSE(cap.is_capable(NULL, CEPH_ENTITY_TYPE_MON, + name, "mon", "", ca, true, true, true)); ca.clear(); - ASSERT_FALSE(cap.is_capable(NULL, name, "", "config-key get", ca, true, true, true)); + ASSERT_FALSE(cap.is_capable(NULL, CEPH_ENTITY_TYPE_MON, + name, "", "config-key get", ca, true, true, true)); ca["key"] = "daemon-private/osd.123"; - ASSERT_FALSE(cap.is_capable(NULL, name, "", "config-key get", ca, true, true, true)); + ASSERT_FALSE(cap.is_capable(NULL, CEPH_ENTITY_TYPE_MON, + name, "", "config-key get", ca, true, true, true)); ca["key"] = "daemon-private/osd.12/asdf"; - ASSERT_FALSE(cap.is_capable(NULL, name, "", "config-key get", ca, true, true, true)); + ASSERT_FALSE(cap.is_capable(NULL, CEPH_ENTITY_TYPE_MON, + name, "", "config-key get", ca, true, true, true)); ca["key"] = "daemon-private/osd.123/"; - ASSERT_TRUE(cap.is_capable(NULL, name, "", "config-key get", ca, true, true, true)); - ASSERT_TRUE(cap.is_capable(NULL, name, "", "config-key get", ca, true, true, true)); - ASSERT_TRUE(cap.is_capable(NULL, name, "", "config-key get", ca, true, true, true)); + ASSERT_TRUE(cap.is_capable(NULL, CEPH_ENTITY_TYPE_MON, + name, "", "config-key get", ca, true, true, true)); + ASSERT_TRUE(cap.is_capable(NULL, CEPH_ENTITY_TYPE_MON, + name, "", "config-key get", ca, true, true, true)); + ASSERT_TRUE(cap.is_capable(NULL, CEPH_ENTITY_TYPE_MON, + name, "", "config-key get", ca, true, true, true)); ca["key"] = "daemon-private/osd.123/foo"; - ASSERT_TRUE(cap.is_capable(NULL, name, "", "config-key get", ca, true, true, true)); - ASSERT_TRUE(cap.is_capable(NULL, name, "", "config-key put", ca, true, true, true)); - ASSERT_TRUE(cap.is_capable(NULL, name, "", "config-key exists", ca, true, true, true)); - ASSERT_TRUE(cap.is_capable(NULL, name, "", "config-key delete", ca, true, true, true)); + ASSERT_TRUE(cap.is_capable(NULL, CEPH_ENTITY_TYPE_MON, + name, "", "config-key get", ca, true, true, true)); + ASSERT_TRUE(cap.is_capable(NULL, CEPH_ENTITY_TYPE_MON, + name, "", "config-key put", ca, true, true, true)); + ASSERT_TRUE(cap.is_capable(NULL, CEPH_ENTITY_TYPE_MON, + name, "", "config-key exists", ca, true, true, true)); + ASSERT_TRUE(cap.is_capable(NULL, CEPH_ENTITY_TYPE_MON, + name, "", "config-key delete", ca, true, true, true)); } diff --git a/src/test/osd/osd-bench.sh b/src/test/osd/osd-bench.sh index f14a35152f7..c44eb28e56a 100755 --- a/src/test/osd/osd-bench.sh +++ b/src/test/osd/osd-bench.sh @@ -40,6 +40,7 @@ function TEST_bench() { local dir=$1 run_mon $dir a || return 1 + run_mgr $dir x || return 1 run_osd $dir 0 || return 1 local osd_bench_small_size_max_iops=$(CEPH_ARGS='' ceph-conf \ diff --git a/src/test/osd/osd-config.sh b/src/test/osd/osd-config.sh index a27862c2235..2a924042a13 100755 --- a/src/test/osd/osd-config.sh +++ b/src/test/osd/osd-config.sh @@ -40,6 +40,7 @@ function TEST_config_init() { local dir=$1 run_mon $dir a || return 1 + run_mgr $dir x || return 1 local advance=1000 local stale=1000 local cache=500 @@ -57,6 +58,7 @@ function TEST_config_track() { local dir=$1 run_mon $dir a || return 1 + run_mgr $dir x || return 1 run_osd $dir 0 || return 1 local osd_map_cache_size=$(CEPH_ARGS='' ceph-conf \ diff --git a/src/test/osd/osd-copy-from.sh b/src/test/osd/osd-copy-from.sh index 3f94743ddaf..ce7c04dfe4c 100755 --- a/src/test/osd/osd-copy-from.sh +++ b/src/test/osd/osd-copy-from.sh @@ -41,6 +41,7 @@ function TEST_copy_from() { local dir=$1 run_mon $dir a || return 1 + run_mgr $dir x || return 1 run_osd $dir 0 || return 1 run_osd $dir 1 || return 1 diff --git a/src/test/osd/osd-fast-mark-down.sh b/src/test/osd/osd-fast-mark-down.sh index e1bde834210..5c8f420fecb 100755 --- a/src/test/osd/osd-fast-mark-down.sh +++ b/src/test/osd/osd-fast-mark-down.sh @@ -53,6 +53,7 @@ function test_fast_kill() { # create cluster with 3 osds setup $dir || return 1 run_mon $dir a --osd_pool_default_size=3 || return 1 + run_mgr $dir x || return 1 for oi in {0..2}; do run_osd $dir $oi || return 1 pids[$oi]=$(cat $dir/osd.$oi.pid) diff --git a/src/test/osd/osd-markdown.sh b/src/test/osd/osd-markdown.sh index f655021f3f4..aa67d9e7706 100755 --- a/src/test/osd/osd-markdown.sh +++ b/src/test/osd/osd-markdown.sh @@ -56,6 +56,7 @@ function TEST_markdown_exceed_maxdown_count() { local dir=$1 run_mon $dir a || return 1 + run_mgr $dir x || return 1 run_osd $dir 0 || return 1 run_osd $dir 1 || return 1 run_osd $dir 2 || return 1 @@ -75,6 +76,7 @@ function TEST_markdown_boot() { local dir=$1 run_mon $dir a || return 1 + run_mgr $dir x || return 1 run_osd $dir 0 || return 1 run_osd $dir 1 || return 1 run_osd $dir 2 || return 1 @@ -96,6 +98,7 @@ function TEST_markdown_boot_exceed_time() { local dir=$1 run_mon $dir a || return 1 + run_mgr $dir x || return 1 run_osd $dir 0 || return 1 run_osd $dir 1 || return 1 run_osd $dir 2 || return 1 diff --git a/src/test/osd/osd-reactivate.sh b/src/test/osd/osd-reactivate.sh index 311bda457bc..5088baf4eb4 100755 --- a/src/test/osd/osd-reactivate.sh +++ b/src/test/osd/osd-reactivate.sh @@ -37,6 +37,7 @@ function TEST_reactivate() { local dir=$1 run_mon $dir a || return 1 + run_mgr $dir x || return 1 run_osd $dir 0 || return 1 kill_daemons $dir TERM osd || return 1 diff --git a/src/test/osd/osd-reuse-id.sh b/src/test/osd/osd-reuse-id.sh index 74f3fe38699..fb0afb13886 100755 --- a/src/test/osd/osd-reuse-id.sh +++ b/src/test/osd/osd-reuse-id.sh @@ -37,6 +37,7 @@ function TEST_reuse_id() { setup $dir || return 1 run_mon $dir a --osd_pool_default_size=1 || return 1 + run_mgr $dir x || return 1 run_osd $dir 0 || return 1 run_osd $dir 1 || return 1 wait_for_clean || return 1 diff --git a/src/test/osd/osd-scrub-repair.sh b/src/test/osd/osd-scrub-repair.sh index a912933c0c8..866383eb111 100755 --- a/src/test/osd/osd-scrub-repair.sh +++ b/src/test/osd/osd-scrub-repair.sh @@ -75,6 +75,7 @@ function TEST_corrupt_and_repair_replicated() { setup $dir || return 1 run_mon $dir a --osd_pool_default_size=2 || return 1 + run_mgr $dir x || return 1 run_osd $dir 0 || return 1 run_osd $dir 1 || return 1 wait_for_clean || return 1 @@ -182,6 +183,7 @@ function TEST_auto_repair_erasure_coded() { # Launch a cluster with 5 seconds scrub interval setup $dir || return 1 run_mon $dir a || return 1 + run_mgr $dir x || return 1 local ceph_osd_args="--osd-scrub-auto-repair=true \ --osd-deep-scrub-interval=5 \ --osd-scrub-max-interval=5 \ @@ -227,6 +229,7 @@ function TEST_corrupt_and_repair_jerasure() { setup $dir || return 1 run_mon $dir a || return 1 + run_mgr $dir x || return 1 for id in $(seq 0 3) ; do run_osd $dir $id || return 1 done @@ -247,6 +250,7 @@ function TEST_corrupt_and_repair_lrc() { setup $dir || return 1 run_mon $dir a || return 1 + run_mgr $dir x || return 1 for id in $(seq 0 9) ; do run_osd $dir $id || return 1 done @@ -269,6 +273,7 @@ function TEST_unfound_erasure_coded() { setup $dir || return 1 run_mon $dir a || return 1 + run_mgr $dir x || return 1 run_osd $dir 0 || return 1 run_osd $dir 1 || return 1 run_osd $dir 2 || return 1 @@ -308,6 +313,11 @@ function TEST_unfound_erasure_coded() { # # 3) check pg state # + # it may take a bit to appear due to mon/mgr asynchrony + for f in `seq 1 60`; do + ceph -s | grep "1/1 unfound" && break + sleep 1 + done ceph -s|grep "4 osds: 4 up, 4 in" || return 1 ceph -s|grep "1/1 unfound" || return 1 @@ -324,6 +334,7 @@ function TEST_list_missing_erasure_coded() { setup $dir || return 1 run_mon $dir a || return 1 + run_mgr $dir x || return 1 for id in $(seq 0 2) ; do run_osd $dir $id || return 1 done @@ -393,6 +404,7 @@ function TEST_corrupt_scrub_replicated() { setup $dir || return 1 run_mon $dir a --osd_pool_default_size=2 || return 1 + run_mgr $dir x || return 1 run_osd $dir 0 || return 1 run_osd $dir 1 || return 1 wait_for_clean || return 1 @@ -1509,6 +1521,7 @@ function TEST_corrupt_scrub_erasure() { setup $dir || return 1 run_mon $dir a || return 1 + run_mgr $dir x || return 1 for id in $(seq 0 2) ; do run_osd $dir $id || return 1 done @@ -2163,6 +2176,7 @@ function TEST_periodic_scrub_replicated() { setup $dir || return 1 run_mon $dir a --osd_pool_default_size=2 || return 1 + run_mgr $dir x || return 1 local ceph_osd_args="--osd-scrub-interval-randomize-ratio=0 --osd-deep-scrub-randomize-ratio=0" run_osd $dir 0 $ceph_osd_args || return 1 run_osd $dir 1 $ceph_osd_args || return 1 diff --git a/src/test/osd/osd-scrub-snaps.sh b/src/test/osd/osd-scrub-snaps.sh index 558ce6e2d61..5410e820c4a 100755 --- a/src/test/osd/osd-scrub-snaps.sh +++ b/src/test/osd/osd-scrub-snaps.sh @@ -40,6 +40,7 @@ function TEST_scrub_snaps() { setup $dir || return 1 run_mon $dir a --osd_pool_default_size=1 || return 1 + run_mgr $dir x || return 1 run_osd $dir 0 || return 1 wait_for_clean || return 1 diff --git a/src/tools/ceph_authtool.cc b/src/tools/ceph_authtool.cc index db6b63d1418..5172391832e 100644 --- a/src/tools/ceph_authtool.cc +++ b/src/tools/ceph_authtool.cc @@ -254,7 +254,7 @@ int main(int argc, const char **argv) } complain_about_parse_errors(g_ceph_context, &parse_errors); map<string, bufferlist> caps; - const char *key_names[] = { "mon", "osd", "mds", NULL }; + const char *key_names[] = { "mon", "osd", "mds", "mgr", NULL }; for (int i=0; key_names[i]; i++) { std::string val; if (cf.read("global", key_names[i], val) == 0) { diff --git a/src/vstart.sh b/src/vstart.sh index 06a804f16b0..50126580225 100755 --- a/src/vstart.sh +++ b/src/vstart.sh @@ -516,6 +516,7 @@ start_mon() { --cap mon 'allow *' \ --cap osd 'allow *' \ --cap mds 'allow *' \ + --cap mgr 'allow *' \ "$keyring_fn" # build a fresh fs monmap, mon fs @@ -574,7 +575,7 @@ EOF local key_fn=$CEPH_DEV_DIR/osd$osd/keyring echo adding osd$osd key to auth repository - ceph_adm -i "$key_fn" auth add osd.$osd osd "allow *" mon "allow profile osd" mgr "allow" + ceph_adm -i "$key_fn" auth add osd.$osd osd "allow *" mon "allow profile osd" mgr "allow profile osd" fi echo start osd$osd run 'osd' $SUDO $CEPH_BIN/ceph-osd -i $osd $ARGS $COSD_ARGS @@ -591,7 +592,7 @@ start_mgr() { mkdir -p $CEPH_DEV_DIR/mgr.$name key_fn=$CEPH_DEV_DIR/mgr.$name/keyring $SUDO $CEPH_BIN/ceph-authtool --create-keyring --gen-key --name=mgr.$name $key_fn - ceph_adm -i $key_fn auth add mgr.$name mon 'allow *' + ceph_adm -i $key_fn auth add mgr.$name mon 'allow profile mgr' fi wconf <<EOF @@ -600,7 +601,7 @@ start_mgr() { EOF echo "Starting mgr.${name}" - run 'mgr' $CEPH_BIN/ceph-mgr -i $name + run 'mgr' $CEPH_BIN/ceph-mgr -i $name $ARGS done } @@ -651,7 +652,7 @@ EOF prun $SUDO "$CEPH_BIN/ceph-authtool" --create-keyring --gen-key --name="mds.${name}s" \ "$CEPH_DEV_DIR/mds.${name}s/keyring" ceph_adm -i "$CEPH_DEV_DIR/mds.${name}s/keyring" auth add "mds.${name}s" \ - mon 'allow *' osd 'allow *' mds 'allow' mgr 'allow' + mon 'allow profile mds' osd 'allow *' mds 'allow' mgr 'allow profile mds' fi fi @@ -680,6 +681,7 @@ else debug mon = 20 debug paxos = 20 debug auth = 20 + debug mgrc = 20 debug ms = 1' COSDDEBUG=' debug ms = 1 diff --git a/systemd/ceph-mgr@.service b/systemd/ceph-mgr@.service index 4de0b8c2d9f..b6e9fb687f2 100644 --- a/systemd/ceph-mgr@.service +++ b/systemd/ceph-mgr@.service @@ -9,6 +9,7 @@ LimitNOFILE=1048576 LimitNPROC=1048576 EnvironmentFile=-/etc/sysconfig/ceph Environment=CLUSTER=ceph + # This ExecStartPre business is a hack to inject a key for the mgr daemon, # using whatever key already exists on the mon on this node to gain sufficient # permissions to create the mgr key. Failure is ignored at every step (the @@ -19,7 +20,8 @@ Environment=CLUSTER=ceph ExecStartPre=-/bin/sh -c "exec mkdir -p /var/lib/ceph/mgr/${CLUSTER}-%i" ExecStartPre=-/bin/sh -c "[ -f /var/lib/ceph/mgr/${CLUSTER}-%i/keyring ] || /usr/bin/ceph-authtool --create-keyring --gen-key --name=mgr.%i /var/lib/ceph/mgr/${CLUSTER}-%i/keyring" ExecStartPre=-/bin/sh -c "exec chown -R ceph.ceph /var/lib/ceph/mgr/${CLUSTER}-%i" -ExecStartPre=-/usr/bin/ceph -i /var/lib/ceph/mgr/${CLUSTER}-%i/keyring auth add mgr.%i mon 'allow *' --keyring=/var/lib/ceph/mon/${CLUSTER}-%i/keyring --name=mon. +ExecStartPre=-/usr/bin/ceph -i /var/lib/ceph/mgr/${CLUSTER}-%i/keyring auth add mgr.%i mon 'allow profile mgr' --keyring=/var/lib/ceph/mon/${CLUSTER}-%i/keyring --name=mon. + ExecStart=/usr/bin/ceph-mgr -f --cluster ${CLUSTER} --id %i --setuser ceph --setgroup ceph ExecReload=/bin/kill -HUP $MAINPID Restart=on-failure |