Merge pull request #48226 from sseshasa/wip-fix-recovery-backfill-limits-for-mclock

osd: Reduce backfill/recovery default limits for mClock and other optimizations

Reviewed-by: Vikhyat Umrao <vikhyat@redhat.com>
Reviewed-by: Neha Ojha <nojha@redhat.com>

8 files changed, 86 insertions, 27 deletions

diff --git a/qa/config/rados.yaml b/qa/config/rados.yaml
index 2f5779b0e9c..1785089fb4b 100644
--- a/qa/config/rados.yaml
+++ b/qa/config/rados.yaml
@@ -7,5 +7,6 @@ overrides:
         osd debug verify missing on start: true
         osd debug verify cached snaps: true
         bluestore zero block detection: true
+        osd mclock override recovery settings: true
       mon:
         mon scrub interval: 300
diff --git a/qa/standalone/ceph-helpers.sh b/qa/standalone/ceph-helpers.sh
index 0901815ee9b..41e92edce44 100755
--- a/qa/standalone/ceph-helpers.sh
+++ b/qa/standalone/ceph-helpers.sh
@@ -754,24 +754,14 @@ function test_run_osd() {
     echo "$backfills" | grep --quiet 'osd_max_backfills' || return 1
 
     run_osd $dir 1 --osd-max-backfills 20 || return 1
-    local scheduler=$(get_op_scheduler 1)
     local backfills=$(CEPH_ARGS='' ceph --format=json daemon $(get_asok_path osd.1) \
         config get osd_max_backfills)
-    if [ "$scheduler" = "mclock_scheduler" ]; then
-      test "$backfills" = '{"osd_max_backfills":"1000"}' || return 1
-    else
-      test "$backfills" = '{"osd_max_backfills":"20"}' || return 1
-    fi
+    test "$backfills" = '{"osd_max_backfills":"20"}' || return 1
 
     CEPH_ARGS="$CEPH_ARGS --osd-max-backfills 30" run_osd $dir 2 || return 1
-    local scheduler=$(get_op_scheduler 2)
     local backfills=$(CEPH_ARGS='' ceph --format=json daemon $(get_asok_path osd.2) \
         config get osd_max_backfills)
-    if [ "$scheduler" = "mclock_scheduler" ]; then
-      test "$backfills" = '{"osd_max_backfills":"1000"}' || return 1
-    else
-      test "$backfills" = '{"osd_max_backfills":"30"}' || return 1
-    fi
+    test "$backfills" = '{"osd_max_backfills":"30"}' || return 1
 
     teardown $dir || return 1
 }
@@ -906,14 +896,9 @@ function test_activate_osd() {
     kill_daemons $dir TERM osd || return 1
 
     activate_osd $dir 0 --osd-max-backfills 20 || return 1
-    local scheduler=$(get_op_scheduler 0)
     local backfills=$(CEPH_ARGS='' ceph --format=json daemon $(get_asok_path osd.0) \
         config get osd_max_backfills)
-    if [ "$scheduler" = "mclock_scheduler" ]; then
-      test "$backfills" = '{"osd_max_backfills":"1000"}' || return 1
-    else
-      test "$backfills" = '{"osd_max_backfills":"20"}' || return 1
-    fi
+    test "$backfills" = '{"osd_max_backfills":"20"}' || return 1
 
     teardown $dir || return 1
 }
@@ -936,14 +921,9 @@ function test_activate_osd_after_mark_down() {
     wait_for_osd down 0 || return 1
 
     activate_osd $dir 0 --osd-max-backfills 20 || return 1
-    local scheduler=$(get_op_scheduler 0)
     local backfills=$(CEPH_ARGS='' ceph --format=json daemon $(get_asok_path osd.0) \
         config get osd_max_backfills)
-    if [ "$scheduler" = "mclock_scheduler" ]; then
-      test "$backfills" = '{"osd_max_backfills":"1000"}' || return 1
-    else
-      test "$backfills" = '{"osd_max_backfills":"20"}' || return 1
-    fi
+    test "$backfills" = '{"osd_max_backfills":"20"}' || return 1
 
     teardown $dir || return 1
 }
diff --git a/qa/standalone/erasure-code/test-erasure-eio.sh b/qa/standalone/erasure-code/test-erasure-eio.sh
index e08c68377f6..85c2763307f 100755
--- a/qa/standalone/erasure-code/test-erasure-eio.sh
+++ b/qa/standalone/erasure-code/test-erasure-eio.sh
@@ -27,6 +27,7 @@ function run() {
     CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none "
     CEPH_ARGS+="--mon-host=$CEPH_MON "
     CEPH_ARGS+="--osd-mclock-profile=high_recovery_ops "
+    CEPH_ARGS+="--osd_mclock_override_recovery_settings=true "
 
     local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')}
     for func in $funcs ; do
diff --git a/qa/standalone/misc/test-mclock-profile-switch.sh b/qa/standalone/misc/mclock-config.sh
index 1bbd6ee81c6..702c5096756 100644..100755
--- a/qa/standalone/misc/test-mclock-profile-switch.sh
+++ b/qa/standalone/misc/mclock-config.sh
@@ -25,7 +25,7 @@ function run() {
     export CEPH_ARGS
     CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none "
     CEPH_ARGS+="--mon-host=$CEPH_MON "
-    CEPH_ARGS+="--debug-bluestore 20 "
+    CEPH_ARGS+="--debug-mclock 20 "
 
     local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')}
     for func in $funcs ; do
@@ -190,9 +190,77 @@ function TEST_profile_custom_to_builtin() {
     teardown $dir || return 1
 }
 
-main test-mclock-profile-switch "$@"
+function TEST_recovery_limit_adjustment_mclock() {
+    local dir=$1
+
+    setup $dir || return 1
+    run_mon $dir a || return 1
+    run_mgr $dir x || return 1
+
+    run_osd $dir 0 --osd_op_queue=mclock_scheduler || return 1
+    local recoveries=$(CEPH_ARGS='' ceph --format=json daemon $(get_asok_path osd.0) \
+        config get osd_recovery_max_active)
+    # Get default value
+    echo "$recoveries" | grep --quiet 'osd_recovery_max_active' || return 1
+
+    # Change the recovery limit without setting
+    # osd_mclock_override_recovery_settings option. Verify that the recovery
+    # limit is retained at its default value.
+    ceph config set osd.0 osd_recovery_max_active 10 || return 1
+    sleep 2 # Allow time for change to take effect
+    local max_recoveries=$(CEPH_ARGS='' ceph --format=json daemon $(get_asok_path osd.0) \
+        config get osd_recovery_max_active)
+    test "$max_recoveries" = "$recoveries" || return 1
+
+    # Change recovery limit after setting osd_mclock_override_recovery_settings.
+    # Verify that the recovery limit is modified.
+    ceph config set osd.0 osd_mclock_override_recovery_settings true || return 1
+    ceph config set osd.0 osd_recovery_max_active 10 || return 1
+    sleep 2 # Allow time for change to take effect
+    max_recoveries=$(CEPH_ARGS='' ceph --format=json daemon $(get_asok_path osd.0) \
+        config get osd_recovery_max_active)
+    test "$max_recoveries" = '{"osd_recovery_max_active":"10"}' || return 1
+
+    teardown $dir || return 1
+}
+
+function TEST_backfill_limit_adjustment_mclock() {
+    local dir=$1
+
+    setup $dir || return 1
+    run_mon $dir a || return 1
+    run_mgr $dir x || return 1
+
+    run_osd $dir 0 --osd_op_queue=mclock_scheduler || return 1
+    local backfills=$(CEPH_ARGS='' ceph --format=json daemon $(get_asok_path osd.0) \
+        config get osd_max_backfills)
+    # Get default value
+    echo "$backfills" | grep --quiet 'osd_max_backfills' || return 1
+
+    # Change the backfill limit without setting
+    # osd_mclock_override_recovery_settings option. Verify that the backfill
+    # limit is retained at its default value.
+    ceph config set osd.0 osd_max_backfills 20 || return 1
+    sleep 2 # Allow time for change to take effect
+    local max_backfills=$(CEPH_ARGS='' ceph --format=json daemon $(get_asok_path osd.0) \
+        config get osd_max_backfills)
+    test "$max_backfills" = "$backfills" || return 1
+
+    # Change backfills limit after setting osd_mclock_override_recovery_settings.
+    # Verify that the backfills limit is modified.
+    ceph config set osd.0 osd_mclock_override_recovery_settings true || return 1
+    ceph config set osd.0 osd_max_backfills 20 || return 1
+    sleep 2 # Allow time for change to take effect
+    max_backfills=$(CEPH_ARGS='' ceph --format=json daemon $(get_asok_path osd.0) \
+        config get osd_max_backfills)
+    test "$max_backfills" =  '{"osd_max_backfills":"20"}' || return 1
+
+    teardown $dir || return 1
+}
+
+main mclock-config "$@"
 
 # Local Variables:
 # compile-command: "cd build ; make -j4 && \
-#   ../qa/run-standalone.sh test-mclock-profile-switch.sh"
+#   ../qa/run-standalone.sh mclock-config.sh"
 # End:
diff --git a/qa/standalone/osd-backfill/osd-backfill-recovery-log.sh b/qa/standalone/osd-backfill/osd-backfill-recovery-log.sh
index 6f20d90b5b5..f9a14493215 100755
--- a/qa/standalone/osd-backfill/osd-backfill-recovery-log.sh
+++ b/qa/standalone/osd-backfill/osd-backfill-recovery-log.sh
@@ -26,6 +26,7 @@ function run() {
     export CEPH_ARGS
     CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none "
     CEPH_ARGS+="--mon-host=$CEPH_MON --osd_max_backfills=1 --debug_reserver=20 "
+    CEPH_ARGS+="--osd_mclock_override_recovery_settings=true "
 
     local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')}
     for func in $funcs ; do
diff --git a/qa/standalone/osd-backfill/osd-backfill-space.sh b/qa/standalone/osd-backfill/osd-backfill-space.sh
index 8bc452d6cbb..671bcdee138 100755
--- a/qa/standalone/osd-backfill/osd-backfill-space.sh
+++ b/qa/standalone/osd-backfill/osd-backfill-space.sh
@@ -29,6 +29,7 @@ function run() {
     CEPH_ARGS+="--fake_statfs_for_testing=3686400 "
     CEPH_ARGS+="--osd_max_backfills=10 "
     CEPH_ARGS+="--osd_mclock_profile=high_recovery_ops "
+    CEPH_ARGS+="--osd_mclock_override_recovery_settings=true "
     export objects=600
     export poolprefix=test
 
diff --git a/qa/standalone/osd/osd-recovery-space.sh b/qa/standalone/osd/osd-recovery-space.sh
index 07ed09b4380..3bafc5138bb 100755
--- a/qa/standalone/osd/osd-recovery-space.sh
+++ b/qa/standalone/osd/osd-recovery-space.sh
@@ -26,6 +26,7 @@ function run() {
     CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none "
     CEPH_ARGS+="--mon-host=$CEPH_MON "
     CEPH_ARGS+="--osd_max_backfills=10 "
+    CEPH_ARGS+="--osd_mclock_override_recovery_settings=true "
     export objects=600
     export poolprefix=test
 
diff --git a/qa/suites/rados/dashboard/tasks/dashboard.yaml b/qa/suites/rados/dashboard/tasks/dashboard.yaml
index fd2c47cba5e..c781061de9c 100644
--- a/qa/suites/rados/dashboard/tasks/dashboard.yaml
+++ b/qa/suites/rados/dashboard/tasks/dashboard.yaml
@@ -2,6 +2,12 @@ roles:
 - [mgr.x, mon.a, mon.c, mds.a, mds.c, osd.0, client.0]
 - [mgr.y, mgr.z, mon.b, mds.b, osd.1, osd.2, osd.3, client.1]
 
+overrides:
+  ceph:
+    conf:
+      osd:
+        osd mclock override recovery settings: true
+
 tasks:
   - install:
   - ceph: