mgr/dashboard: Add RGW per user/bucket panels in grafana

Fixes: https://tracker.ceph.com/issues/64359

Signed-off-by: Aashish Sharma <aasharma@redhat.com>

5 files changed, 7153 insertions, 1 deletions

diff --git a/monitoring/ceph-mixin/dashboards.libsonnet b/monitoring/ceph-mixin/dashboards.libsonnet
index 4c67f21c039..8cb0bcceaf7 100644
--- a/monitoring/ceph-mixin/dashboards.libsonnet
+++ b/monitoring/ceph-mixin/dashboards.libsonnet
@@ -7,5 +7,6 @@
     (import 'dashboards/rbd.libsonnet') +
     (import 'dashboards/rgw.libsonnet') +
     (import 'dashboards/ceph-cluster.libsonnet') +
+    (import 'dashboards/rgw-s3-analytics.libsonnet') +
     { _config:: $._config },
 }
diff --git a/monitoring/ceph-mixin/dashboards/rgw-s3-analytics.libsonnet b/monitoring/ceph-mixin/dashboards/rgw-s3-analytics.libsonnet
new file mode 100644
index 00000000000..c470dbe5f19
--- /dev/null
+++ b/monitoring/ceph-mixin/dashboards/rgw-s3-analytics.libsonnet
@@ -0,0 +1,2446 @@
+local g = import 'grafonnet/grafana.libsonnet';
+
+(import 'utils.libsonnet') {
+  'rgw-s3-analytics.json':
+    $.dashboardSchema(
+      'RGW S3 Analytics',
+      '',
+      'BnxelG7Sz',
+      'now-1h',
+      '30s',
+      22,
+      $._config.dashboardTags,
+      ''
+    )
+    .addAnnotation(
+      $.addAnnotationSchema(
+        1,
+        '-- Grafana --',
+        true,
+        true,
+        'rgba(0, 211, 255, 1)',
+        'Annotations & Alerts',
+        'dashboard'
+      )
+    )
+
+    .addTemplate(
+      g.template.datasource('datasource', 'prometheus', 'default', label='Data Source')
+    )
+
+    .addTemplate(
+      $.addTemplateSchema('rgw_servers',
+                          '$datasource',
+                          'label_values(ceph_rgw_metadata{}, ceph_daemon)',
+                          2,
+                          true,
+                          0,
+                          null,
+                          '')
+    )
+
+    .addTemplate(
+      g.template.adhoc('Filters', '$datasource', 'filters', 0)
+    )
+
+
+    .addPanels([
+      $.addRowSchema(false, true, 'Overview') + { gridPos: { x: 0, y: 0, w: 24, h: 1 } },
+      $.addStatPanel(
+        title='Total PUTs',
+        datasource='${datasource}',
+        gridPosition={ x: 0, y: 1, w: 6, h: 3 },
+        graphMode='none',
+        colorMode='none',
+        unit='decbytes',
+        orientation='auto',
+        justifyMode='auto',
+        thresholdsMode='absolute',
+        pluginVersion='9.4.7',
+      ).addThresholds([
+        { color: 'green', value: null },
+        { color: 'red', value: 80 },
+      ])
+      .addTargets([
+        $.addTargetSchema(
+          expr='sum(ceph_rgw_op_put_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource='${datasource}',
+          legendFormat='__auto',
+          range=true
+        ),
+      ]),
+
+      $.addStatPanel(
+        title='Total GETs',
+        datasource='${datasource}',
+        gridPosition={ x: 6, y: 1, w: 6, h: 3 },
+        graphMode='none',
+        colorMode='none',
+        unit='decbytes',
+        orientation='auto',
+        justifyMode='auto',
+        thresholdsMode='absolute',
+        pluginVersion='9.4.7',
+      ).addThresholds([
+        { color: 'green', value: null },
+        { color: 'red', value: 80 },
+      ])
+      .addTargets([
+        $.addTargetSchema(
+          expr='sum\n(ceph_rgw_op_get_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource='${datasource}',
+          legendFormat='__auto',
+          range=true
+        ),
+      ]),
+
+      $.addStatPanel(
+        title='Total Objects',
+        datasource='${datasource}',
+        gridPosition={ x: 12, y: 1, w: 6, h: 3 },
+        graphMode='none',
+        colorMode='none',
+        unit='none',
+        orientation='auto',
+        justifyMode='auto',
+        thresholdsMode='absolute',
+        pluginVersion='9.4.7',
+      ).addThresholds([
+        { color: 'green', value: null },
+        { color: 'red', value: 80 },
+      ])
+      .addTargets([
+        $.addTargetSchema(
+          expr='sum(ceph_rgw_op_put_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource='${datasource}',
+          legendFormat='__auto',
+          range=true
+        ),
+      ]),
+
+      $.addStatPanel(
+        title='Average Object Size',
+        datasource='${datasource}',
+        gridPosition={ x: 18, y: 1, w: 6, h: 3 },
+        graphMode='none',
+        colorMode='none',
+        unit='decbytes',
+        orientation='auto',
+        justifyMode='auto',
+        thresholdsMode='absolute',
+        pluginVersion='9.4.7',
+      ).addThresholds([
+        { color: 'green', value: null },
+        { color: 'red', value: 80 },
+      ])
+      .addTargets([
+        $.addTargetSchema(
+          expr='sum\n((sum by(instance_id)(ceph_rgw_op_put_obj_bytes) > 0) / (sum by(instance_id)(ceph_rgw_op_put_obj_ops) > 0) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource='${datasource}',
+          legendFormat='__auto',
+          range=true
+        ),
+      ]),
+
+      $.addBarGaugePanel(
+        title='Total Operations',
+        datasource='${datasource}',
+        gridPosition={ x: 0, y: 4, w: 8, h: 8 },
+        unit='none',
+        thresholds={ color: 'green', value: null }
+      )
+      .addTargets([
+        $.addTargetSchema(
+          expr='sum(ceph_rgw_op_list_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource='${datasource}',
+          legendFormat='List Objects',
+          range=false,
+          instant=true
+        ),
+        $.addTargetSchema(
+          expr='sum(ceph_rgw_op_list_buckets_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource='${datasource}',
+          legendFormat='List Buckets',
+          range=true
+        ),
+        $.addTargetSchema(
+          expr='sum(ceph_rgw_op_put_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource='${datasource}',
+          legendFormat='Put Objects',
+          range=false,
+          instant=true
+        ),
+        $.addTargetSchema(
+          expr='sum(ceph_rgw_op_per_bucket_get_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource='${datasource}',
+          legendFormat='Get Objects',
+          range=false,
+          instant=true
+        ),
+        $.addTargetSchema(
+          expr='sum(ceph_rgw_op_del_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource='${datasource}',
+          legendFormat='Delete Objects',
+          range=false,
+          instant=true
+        ),
+        $.addTargetSchema(
+          expr='sum(ceph_rgw_op_del_bucket_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource='${datasource}',
+          legendFormat='Delete Buckets',
+          range=false,
+          instant=true
+        ),
+        $.addTargetSchema(
+          expr='sum(ceph_rgw_op_copy_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource='${datasource}',
+          legendFormat='Copy Objects',
+          range=true
+        ),
+      ]) + { fieldConfig: { defaults: { color: { mode: 'thresholds' }, thresholds: { mode: 'absolute', steps: [{ color: 'green', value: null }] } } } }
+      + { options: { orientation: 'horizontal', reduceOptions: { calcs: ['lastNotNull'] }, displayMode: 'gradient' } },
+
+
+      $.addBarGaugePanel(
+        title='Total Size',
+        datasource='${datasource}',
+        gridPosition={ x: 8, y: 4, w: 8, h: 8 },
+        unit='none',
+        thresholds={ color: 'green', value: null }
+      )
+      .addTargets([
+        $.addTargetSchema(
+          expr='sum(ceph_rgw_op_put_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource='${datasource}',
+          legendFormat='Put Objects',
+          range=false,
+          instant=true
+        ),
+        $.addTargetSchema(
+          expr='sum(ceph_rgw_op_per_bucket_get_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource='${datasource}',
+          legendFormat='Get Objects',
+          range=false,
+          instant=true
+        ),
+        $.addTargetSchema(
+          expr='sum(ceph_rgw_op_del_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource='${datasource}',
+          legendFormat='Delete Objects',
+          range=false,
+          instant=true
+        ),
+        $.addTargetSchema(
+          expr='sum(ceph_rgw_op_copy_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource='${datasource}',
+          legendFormat='Copy Objects',
+          range=true
+        ),
+      ]) + { fieldConfig: { defaults: { color: { mode: 'thresholds' }, thresholds: { mode: 'absolute', steps: [{ color: 'green', value: null }] } }, overrides: [{ matcher: { id: 'byType', unit: 'number' }, properties: [{ id: 'unit', value: 'decbytes' }] }] } }
+      + { options: { orientation: 'horizontal', reduceOptions: { calcs: ['lastNotNull'] }, displayMode: 'gradient' } },
+
+      $.addBarGaugePanel(
+        title='Total Latencies',
+        datasource='${datasource}',
+        gridPosition={ x: 16, y: 4, w: 8, h: 8 },
+        unit='none',
+        thresholds={ color: 'green', value: null }
+      )
+      .addTargets([
+        $.addTargetSchema(
+          expr='sum(ceph_rgw_op_list_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource='${datasource}',
+          legendFormat='List Object',
+          range=false,
+          instant=true
+        ),
+        $.addTargetSchema(
+          expr='sum(ceph_rgw_op_list_buckets_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource='${datasource}',
+          legendFormat='List Bucket',
+          range=true
+        ),
+        $.addTargetSchema(
+          expr='sum(ceph_rgw_op_put_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource='${datasource}',
+          legendFormat='Put Object',
+          range=false,
+          instant=true
+        ),
+        $.addTargetSchema(
+          expr='sum(ceph_rgw_op_get_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource='${datasource}',
+          legendFormat='Get Object',
+          range=false,
+          instant=true
+        ),
+        $.addTargetSchema(
+          expr='sum(ceph_rgw_op_del_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource='${datasource}',
+          legendFormat='Delete Object',
+          range=false,
+          instant=true
+        ),
+        $.addTargetSchema(
+          expr='sum(ceph_rgw_op_del_bucket_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource='${datasource}',
+          legendFormat='Delete Bucket',
+          range=false,
+          instant=true
+        ),
+        $.addTargetSchema(
+          expr='sum(ceph_rgw_op_copy_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource='${datasource}',
+          legendFormat='Copy Object',
+          range=true
+        ),
+      ]) + { fieldConfig: { defaults: { color: { mode: 'thresholds' }, thresholds: { mode: 'absolute', steps: [{ color: 'green', value: null }] } }, overrides: [{ matcher: { id: 'byType', unit: 'number' }, properties: [{ id: 'unit', value: 'ms' }] }] } }
+      + { options: { orientation: 'horizontal', reduceOptions: { calcs: ['lastNotNull'] }, displayMode: 'gradient' } },
+
+
+      $.addTableExtended(
+        datasource='${datasource}',
+        title='Summary Per Bucket by Bandwidth',
+        gridPosition={ h: 8, w: 12, x: 0, y: 12 },
+        options={
+          footer: {
+            fields: '',
+            reducer: ['sum'],
+            countRows: false,
+            enablePagination: false,
+            show: false,
+          },
+          frameIndex: 1,
+          showHeader: true,
+          sortBy: [
+            {
+              desc: true,
+              displayName: 'PUTs',
+            },
+          ],
+        },
+        custom={ align: 'auto', cellOptions: { type: 'color-text' }, filterable: false, inspect: false },
+        thresholds={
+          mode: 'absolute',
+          steps: [
+            { color: 'green', value: null },
+          ],
+        },
+        overrides=[{
+          matcher: { id: 'byType', options: 'number' },
+          properties: [
+            { id: 'unit', value: 'decbytes' },
+          ],
+        }],
+        pluginVersion='9.4.7'
+      )
+      .addTransformations([
+        {
+          id: 'merge',
+          options: {},
+        },
+        {
+          id: 'groupBy',
+          options: {
+            fields: {
+              Bucket: {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              'Value #A': {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              'Value #B': {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              'Value #D': {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              'Value #F': {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              bucket: {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              ceph_daemon: {
+                aggregations: [],
+                operation: 'groupby',
+              },
+            },
+          },
+        },
+        {
+          id: 'organize',
+          options: {
+            excludeByName: {
+              'Time 1': true,
+              'Time 2': true,
+              'Time 3': true,
+              'Time 4': true,
+              'Time 5': true,
+              'Time 6': true,
+              'Time 7': true,
+              '__name__ 1': true,
+              '__name__ 2': true,
+              '__name__ 3': true,
+              '__name__ 4': true,
+              '__name__ 5': true,
+              '__name__ 6': true,
+              '__name__ 7': true,
+              'ceph_daemon 1': false,
+              'ceph_daemon 2': true,
+              'ceph_daemon 3': true,
+              'ceph_daemon 4': true,
+              'instance 1': true,
+              'instance 2': true,
+              'instance 3': true,
+              'instance 4': true,
+              'instance 5': true,
+              'instance 6': true,
+              'instance 7': true,
+              'instance_id 1': true,
+              'instance_id 2': true,
+              'instance_id 3': true,
+              'instance_id 4': true,
+              'instance_id 5': true,
+              'instance_id 6': true,
+              'instance_id 7': true,
+              'job 1': true,
+              'job 2': true,
+              'job 3': true,
+              'job 4': true,
+              'job 5': true,
+              'job 6': true,
+              'job 7': true,
+            },
+            indexByName: {
+              'Value #A': 2,
+              'Value #B': 3,
+              'Value #D': 4,
+              'Value #F': 5,
+              bucket: 1,
+              ceph_daemon: 0,
+            },
+            renameByName: {
+              Bucket: '',
+              'Value #A': 'PUTs',
+              'Value #B': 'GETs',
+              'Value #C': 'List',
+              'Value #D': 'Delete',
+              'Value #E': 'Copy',
+              'Value #F': 'Copy',
+              'Value #G': '',
+              bucket: 'Bucket',
+              ceph_daemon: 'Daemon',
+              'ceph_daemon 1': 'Daemon',
+            },
+          },
+        },
+      ]).addTargets([
+        $.addTargetSchema(
+          expr='sum by (bucket, ceph_daemon) (ceph_rgw_op_per_bucket_put_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource={ type: 'prometheus', uid: '${datasource}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          interval='',
+          legendFormat='Upload Objects',
+          range=false,
+        ),
+        $.addTargetSchema(
+          expr='sum by (bucket, ceph_daemon) (ceph_rgw_op_per_bucket_get_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource={ type: 'prometheus', uid: '${datasource}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          interval='',
+          legendFormat='Get Objects',
+          range=false,
+        ),
+        $.addTargetSchema(
+          expr='sum by (bucket, ceph_daemon) (ceph_rgw_op_per_bucket_del_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource={ type: 'prometheus', uid: '${datasource}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          interval='',
+          legendFormat='Delete Objects',
+          range=false,
+        ),
+        $.addTargetSchema(
+          expr='sum by (bucket, ceph_daemon) (ceph_rgw_op_per_bucket_copy_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+          datasource={ type: 'prometheus', uid: '${datasource}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          interval='',
+          legendFormat='Copy Objects',
+          range=false,
+        ),
+      ]),
+
+
+      $.addTableExtended(
+        datasource='${datasource}',
+        title='Latency(ms) Per Bucket',
+        gridPosition={ h: 8, w: 12, x: 12, y: 12 },
+        options={
+          footer: {
+            fields: '',
+            reducer: ['sum'],
+            countRows: false,
+            enablePagination: false,
+            show: false,
+          },
+          frameIndex: 1,
+          showHeader: true,
+          sortBy: [
+            {
+              desc: true,
+              displayName: 'PUTs',
+            },
+          ],
+        },
+        custom={ align: 'auto', cellOptions: { type: 'auto' }, filterable: false, inspect: false },
+        thresholds={
+          mode: 'absolute',
+          steps: [
+            { color: 'green', value: null },
+          ],
+        },
+        overrides=[{
+          matcher: { id: 'byType', options: 'number' },
+          properties: [
+            { id: 'unit', value: 'ms' },
+          ],
+        }],
+        pluginVersion='9.4.7'
+      )
+      .addTransformations([
+        {
+          id: 'merge',
+          options: {},
+        },
+        {
+          id: 'joinByField',
+          options: {
+            byField: 'Bucket',
+            mode: 'outer',
+          },
+        },
+        {
+          id: 'groupBy',
+          options: {
+            fields: {
+              Bucket: {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              'Value #A': {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              'Value #B': {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              'Value #C': {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              'Value #D': {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              'Value #F': {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              bucket: {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              ceph_daemon: {
+                aggregations: [],
+                operation: 'groupby',
+              },
+            },
+          },
+        },
+        {
+          id: 'organize',
+          options: {
+            excludeByName: {
+              'Time 1': true,
+              'Time 2': true,
+              'Time 3': true,
+              'Time 4': true,
+              'Time 5': true,
+              'Time 6': true,
+              'Time 7': true,
+              '__name__ 1': true,
+              '__name__ 2': true,
+              '__name__ 3': true,
+              '__name__ 4': true,
+              '__name__ 5': true,
+              '__name__ 6': true,
+              '__name__ 7': true,
+              'ceph_daemon 1': true,
+              'ceph_daemon 2': true,
+              'ceph_daemon 3': true,
+              'ceph_daemon 4': true,
+              'ceph_daemon 5': true,
+              'instance 1': true,
+              'instance 2': true,
+              'instance 3': true,
+              'instance 4': true,
+              'instance 5': true,
+              'instance 6': true,
+              'instance 7': true,
+              'instance_id 1': true,
+              'instance_id 2': true,
+              'instance_id 3': true,
+              'instance_id 4': true,
+              'instance_id 5': true,
+              'instance_id 6': true,
+              'instance_id 7': true,
+              'job 1': true,
+              'job 2': true,
+              'job 3': true,
+              'job 4': true,
+              'job 5': true,
+              'job 6': true,
+              'job 7': true,
+            },
+            indexByName: {
+              'Value #A': 2,
+              'Value #B': 3,
+              'Value #C': 4,
+              'Value #D': 5,
+              'Value #F': 6,
+              bucket: 1,
+              ceph_daemon: 0,
+            },
+            renameByName: {
+              Bucket: '',
+              'Value #A': 'PUTs',
+              'Value #B': 'GETs',
+              'Value #C': 'List',
+              'Value #D': 'Delete',
+              'Value #E': 'Copy',
+              'Value #F': 'Copy',
+              'Value #G': '',
+              bucket: 'Bucket',
+              ceph_daemon: 'Daemon',
+            },
+          },
+        },
+      ]).addTargets([
+        $.addTargetSchema(
+          expr='ceph_rgw_op_per_bucket_list_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"}',
+          datasource={ type: 'prometheus', uid: '${datasource}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          interval='',
+          legendFormat='List Objects',
+          range=false,
+        ),
+        $.addTargetSchema(
+          expr='ceph_rgw_op_per_bucket_put_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"}',
+          datasource={ type: 'prometheus', uid: '${datasource}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          interval='',
+          legendFormat='Upload Objects',
+          range=false,
+        ),
+        $.addTargetSchema(
+          expr='ceph_rgw_op_per_bucket_get_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"}',
+          datasource={ type: 'prometheus', uid: '${datasource}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          interval='',
+          legendFormat='Get Objects',
+          range=false,
+        ),
+        $.addTargetSchema(
+          expr='ceph_rgw_op_per_bucket_del_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"}',
+          datasource={ type: 'prometheus', uid: '${datasource}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          interval='',
+          legendFormat='Delete Objects',
+          range=false,
+        ),
+        $.addTargetSchema(
+          expr='ceph_rgw_op_per_bucket_copy_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"}',
+          datasource={ type: 'prometheus', uid: '${datasource}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          interval='',
+          legendFormat='Copy Objects',
+          range=false,
+        ),
+      ]),
+
+
+      $.addTableExtended(
+        datasource='${datasource}',
+        title='Summary Per User By Bandwidth',
+        gridPosition={ h: 8, w: 12, x: 0, y: 20 },
+        options={
+          footer: {
+            fields: '',
+            reducer: ['sum'],
+            countRows: false,
+            enablePagination: false,
+            show: false,
+          },
+          frameIndex: 1,
+          showHeader: true,
+          sortBy: [
+            {
+              desc: true,
+              displayName: 'PUTs',
+            },
+          ],
+        },
+        custom={ align: 'auto', cellOptions: { type: 'auto' }, filterable: false, inspect: false },
+        thresholds={
+          mode: 'absolute',
+          steps: [
+            { color: 'green', value: null },
+          ],
+        },
+        overrides=[{
+          matcher: { id: 'byType', options: 'number' },
+          properties: [
+            { id: 'unit', value: 'decbytes' },
+          ],
+        }],
+        pluginVersion='9.4.7'
+      )
+      .addTransformations([
+        {
+          id: 'merge',
+          options: {},
+        },
+        {
+          id: 'groupBy',
+          options: {
+            fields: {
+              User: {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              'Value #A': {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              'Value #B': {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              'Value #D': {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              'Value #F': {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              ceph_daemon: {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              instance: {
+                aggregations: [],
+              },
+              user: {
+                aggregations: [],
+                operation: 'groupby',
+              },
+            },
+          },
+        },
+        {
+          id: 'organize',
+          options: {
+            excludeByName: {
+              'Time 1': true,
+              'Time 2': true,
+              'Time 3': true,
+              'Time 4': true,
+              'Time 5': true,
+              'Time 6': true,
+              'Time 7': true,
+              '__name__ 1': true,
+              '__name__ 2': true,
+              '__name__ 3': true,
+              '__name__ 4': true,
+              '__name__ 5': true,
+              '__name__ 6': true,
+              '__name__ 7': true,
+              'ceph_daemon 1': true,
+              'ceph_daemon 2': true,
+              'ceph_daemon 3': true,
+              'ceph_daemon 4': true,
+              'instance 1': true,
+              'instance 2': true,
+              'instance 3': true,
+              'instance 4': true,
+              'instance 5': true,
+              'instance 6': true,
+              'instance 7': true,
+              'instance_id 1': true,
+              'instance_id 2': true,
+              'instance_id 3': true,
+              'instance_id 4': true,
+              'instance_id 5': true,
+              'instance_id 6': true,
+              'instance_id 7': true,
+              'job 1': true,
+              'job 2': true,
+              'job 3': true,
+              'job 4': true,
+              'job 5': true,
+              'job 6': true,
+              'job 7': true,
+            },
+            indexByName: {
+              'Value #A': 2,
+              'Value #B': 3,
+              'Value #D': 4,
+              'Value #F': 5,
+              ceph_daemon: 0,
+              user: 1,
+            },
+            renameByName: {
+              Bucket: '',
+              'Value #A': 'PUTs',
+              'Value #B': 'GETs',
+              'Value #C': 'List',
+              'Value #D': 'Delete',
+              'Value #E': 'Copy',
+              'Value #F': 'Copy',
+              'Value #G': '',
+              ceph_daemon: 'Daemon',
+              user: 'User',
+            },
+          },
+        },
+      ]).addTargets([
+        $.addTargetSchema(
+          expr='ceph_rgw_op_per_user_put_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"}',
+          datasource={ type: 'prometheus', uid: '${datasource}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          interval='',
+          legendFormat='Upload Objects',
+          range=false,
+        ),
+        $.addTargetSchema(
+          expr='ceph_rgw_op_per_user_get_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"}',
+          datasource={ type: 'prometheus', uid: '${datasource}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          interval='',
+          legendFormat='Get Objects',
+          range=false,
+        ),
+        $.addTargetSchema(
+          expr='ceph_rgw_op_per_user_del_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"}',
+          datasource={ type: 'prometheus', uid: '${datasource}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          interval='',
+          legendFormat='Delete Objects',
+          range=false,
+        ),
+        $.addTargetSchema(
+          expr='ceph_rgw_op_per_user_copy_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"}',
+          datasource={ type: 'prometheus', uid: '${datasource}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          interval='',
+          legendFormat='Copy Objects',
+          range=false,
+        ),
+      ]),
+
+
+      $.addTableExtended(
+        datasource='${datasource}',
+        title='Latency(ms) Per User',
+        gridPosition={ h: 8, w: 12, x: 12, y: 20 },
+        options={
+          footer: {
+            fields: '',
+            reducer: ['sum'],
+            countRows: false,
+            enablePagination: false,
+            show: false,
+          },
+          frameIndex: 1,
+          showHeader: true,
+          sortBy: [
+            {
+              desc: true,
+              displayName: 'PUTs',
+            },
+          ],
+        },
+        custom={ align: 'auto', cellOptions: { type: 'auto' }, filterable: false, inspect: false },
+        thresholds={
+          mode: 'absolute',
+          steps: [
+            { color: 'green', value: null },
+          ],
+        },
+        overrides=[{
+          matcher: { id: 'byType', options: 'number' },
+          properties: [
+            { id: 'unit', value: 'ms' },
+          ],
+        }],
+        pluginVersion='9.4.7'
+      )
+      .addTransformations([
+        {
+          id: 'merge',
+          options: {},
+        },
+        {
+          id: 'joinByField',
+          options: {
+            byField: 'User',
+            mode: 'outer',
+          },
+        },
+        {
+          id: 'groupBy',
+          options: {
+            fields: {
+              User: {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              'Value #A': {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              'Value #B': {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              'Value #C': {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              'Value #D': {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              'Value #F': {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              ceph_daemon: {
+                aggregations: [],
+                operation: 'groupby',
+              },
+              user: {
+                aggregations: [],
+                operation: 'groupby',
+              },
+            },
+          },
+        },
+        {
+          id: 'organize',
+          options: {
+            excludeByName: {
+              'Time 1': true,
+              'Time 2': true,
+              'Time 3': true,
+              'Time 4': true,
+              'Time 5': true,
+              'Time 6': true,
+              'Time 7': true,
+              '__name__ 1': true,
+              '__name__ 2': true,
+              '__name__ 3': true,
+              '__name__ 4': true,
+              '__name__ 5': true,
+              '__name__ 6': true,
+              '__name__ 7': true,
+              'ceph_daemon 1': true,
+              'ceph_daemon 2': true,
+              'ceph_daemon 3': true,
+              'ceph_daemon 4': true,
+              'ceph_daemon 5': true,
+              'instance 1': true,
+              'instance 2': true,
+              'instance 3': true,
+              'instance 4': true,
+              'instance 5': true,
+              'instance 6': true,
+              'instance 7': true,
+              'instance_id 1': true,
+              'instance_id 2': true,
+              'instance_id 3': true,
+              'instance_id 4': true,
+              'instance_id 5': true,
+              'instance_id 6': true,
+              'instance_id 7': true,
+              'job 1': true,
+              'job 2': true,
+              'job 3': true,
+              'job 4': true,
+              'job 5': true,
+              'job 6': true,
+              'job 7': true,
+            },
+            indexByName: {
+              'Value #A': 2,
+              'Value #B': 3,
+              'Value #C': 4,
+              'Value #D': 5,
+              'Value #F': 6,
+              ceph_daemon: 0,
+              user: 1,
+            },
+            renameByName: {
+              Bucket: '',
+              'Value #A': 'PUTs',
+              'Value #B': 'GETs',
+              'Value #C': 'List',
+              'Value #D': 'Delete',
+              'Value #E': 'Copy',
+              'Value #F': 'Copy',
+              'Value #G': '',
+              ceph_daemon: 'Daemon',
+              user: 'User',
+            },
+          },
+        },
+      ]).addTargets([
+        $.addTargetSchema(
+          expr='ceph_rgw_op_per_user_list_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"}',
+          datasource={ type: 'prometheus', uid: '${datasource}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          interval='',
+          legendFormat='__auto',
+          range=false,
+        ),
+        $.addTargetSchema(
+          expr='ceph_rgw_op_per_user_put_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"}',
+          datasource={ type: 'prometheus', uid: '${datasource}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          interval='',
+          legendFormat='__auto',
+          range=false,
+        ),
+        $.addTargetSchema(
+          expr='ceph_rgw_op_per_user_get_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"}',
+          datasource={ type: 'prometheus', uid: '${datasource}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          interval='',
+          legendFormat='__auto',
+          range=false,
+        ),
+        $.addTargetSchema(
+          expr='ceph_rgw_op_per_user_del_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"}',
+          datasource={ type: 'prometheus', uid: '${datasource}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          interval='',
+          legendFormat='__auto',
+          range=false,
+        ),
+        $.addTargetSchema(
+          expr='ceph_rgw_op_per_user_copy_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"}',
+          datasource={ type: 'prometheus', uid: '${datasource}' },
+          format='table',
+          hide=false,
+          exemplar=false,
+          instant=true,
+          interval='',
+          legendFormat='__auto',
+          range=false,
+        ),
+      ]),
+
+
+      $.addRowSchema(false, true, 'Buckets', collapsed=true)
+      .addPanels([
+        $.addBarGaugePanel(
+          title='Top 5 Bucket PUTs by Operations',
+          datasource='${datasource}',
+          gridPosition={ x: 0, y: 29, w: 6, h: 8 },
+          unit='none',
+          thresholds={ color: 'green', value: null }
+        )
+        .addTargets([
+          $.addTargetSchema(
+            expr='topk(5, \n    sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_put_obj_ops) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})\n)',
+            datasource='${datasource}',
+            legendFormat='{{ceph_daemon}} - {{bucket}}',
+            range=false,
+            instant=true
+          ),
+        ]) + { fieldConfig: { defaults: { color: { mode: 'thresholds' }, thresholds: { mode: 'absolute', steps: [{ color: 'green' }] } }, overrides: [{ matcher: { id: 'byType', unit: 'number' }, properties: [{ id: 'color' }, { id: 'color', value: { mode: 'palette-classic' } }] }] } }
+        + { options: { orientation: 'horizontal', reduceOptions: { calcs: ['lastNotNull'] }, displayMode: 'gradient' } },
+
+
+        $.addBarGaugePanel(
+          title='Top 5 Bucket GETs by Operations',
+          datasource='${datasource}',
+          gridPosition={ x: 6, y: 29, w: 6, h: 8 },
+          unit='none',
+          thresholds={ color: 'green', value: null }
+        )
+        .addTargets([
+          $.addTargetSchema(
+            expr='topk(5, \n    sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_get_obj_ops) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})\n)',
+            datasource='${datasource}',
+            legendFormat='{{ceph_daemon}} - {{bucket}}',
+            range=false,
+            instant=true
+          ),
+        ]) + { fieldConfig: { defaults: { color: { mode: 'thresholds' }, thresholds: { mode: 'absolute', steps: [{ color: 'green' }] } }, overrides: [{ matcher: { id: 'byType', unit: 'number' }, properties: [{ id: 'color' }, { id: 'color', value: { mode: 'palette-classic' } }] }] } }
+        + { options: { orientation: 'horizontal', reduceOptions: { calcs: ['lastNotNull'] }, displayMode: 'gradient' } },
+
+
+        $.addBarGaugePanel(
+          title='Top 5 Buckets PUTs By Size',
+          datasource='${datasource}',
+          gridPosition={ x: 12, y: 29, w: 6, h: 8 },
+          unit='decbytes',
+          thresholds={ color: 'green', value: null }
+        )
+        .addTargets([
+          $.addTargetSchema(
+            expr='topk(5,\n    sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_put_obj_bytes) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})\n)',
+            datasource='${datasource}',
+            legendFormat='{{ceph_daemon}} - {{bucket}}',
+            range=false,
+            instant=true
+          ),
+        ]) + { fieldConfig: { defaults: { color: { mode: 'thresholds' }, thresholds: { mode: 'absolute', steps: [{ color: 'green' }] } } } }
+        + { options: { orientation: 'horizontal', reduceOptions: { calcs: [] }, displayMode: 'gradient' } },
+
+
+        $.addBarGaugePanel(
+          title='Top 5 Buckets GETs By Size',
+          datasource='${datasource}',
+          gridPosition={ x: 18, y: 29, w: 6, h: 8 },
+          unit='decbytes',
+          thresholds={ color: 'green', value: null }
+        )
+        .addTargets([
+          $.addTargetSchema(
+            expr='topk(5,\n    sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_get_obj_bytes) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})\n)',
+            datasource='${datasource}',
+            legendFormat='{{ceph_daemon}} - {{bucket}}',
+            range=false,
+            instant=true
+          ),
+        ]) + { fieldConfig: { defaults: { color: { mode: 'thresholds' }, thresholds: { mode: 'absolute', steps: [{ color: 'green' }] } } } }
+        + { options: { orientation: 'horizontal', reduceOptions: { calcs: [] }, displayMode: 'gradient' } },
+
+
+        $.timeSeriesPanel(
+          lineInterpolation='linear',
+          lineWidth=1,
+          drawStyle='line',
+          axisPlacement='auto',
+          title='Bucket PUTs by Size',
+          datasource='${datasource}',
+          gridPosition={ h: 8, w: 6, x: 0, y: 37 },
+          fillOpacity=0,
+          pointSize=5,
+          showPoints='auto',
+          unit='decbytes',
+          displayMode='table',
+          showLegend=true,
+          placement='bottom',
+          tooltip={ mode: 'single', sort: 'desc' },
+          stackingMode='none',
+          spanNulls=true,
+          decimals=2,
+          thresholdsMode='absolute',
+          sortBy='Last *',
+          sortDesc=true
+        )
+        .addThresholds([
+          { color: 'green' },
+        ])
+        .addOverrides([
+          { matcher: { id: 'byType', unit: 'number' }, properties: [{ id: 'color' }, { id: 'color', value: { mode: 'palette-classic' } }] },
+        ])
+        .addTargets(
+          [
+            $.addTargetSchema(
+              expr='sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_put_obj_bytes) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+              datasource='${datasource}',
+              format='time_series',
+              instant=false,
+              legendFormat='{{ceph_daemon}} - {{bucket}}',
+              step=300,
+              range=true,
+            ),
+          ]
+        ) + { options: { legend: { calcs: ['lastNotNull'], displayMode: 'table', placement: 'bottom', showLegend: true, sortBy: 'Last *', sortDesc: true }, tooltip: { mode: 'single', sort: 'desc' } } },
+
+
+        $.timeSeriesPanel(
+          lineInterpolation='linear',
+          lineWidth=1,
+          drawStyle='line',
+          axisPlacement='auto',
+          title='Bucket GETs by Size',
+          datasource='${datasource}',
+          gridPosition={ h: 8, w: 6, x: 6, y: 37 },
+          fillOpacity=0,
+          pointSize=5,
+          showPoints='auto',
+          unit='decbytes',
+          displayMode='table',
+          showLegend=true,
+          placement='bottom',
+          tooltip={ mode: 'single', sort: 'desc' },
+          stackingMode='none',
+          spanNulls=true,
+          decimals=2,
+          thresholdsMode='absolute',
+          sortBy='Last *',
+          sortDesc=true
+        )
+        .addThresholds([
+          { color: 'green' },
+        ])
+        .addOverrides([
+          { matcher: { id: 'byType', unit: 'number' }, properties: [{ id: 'color' }, { id: 'color', value: { mode: 'palette-classic' } }] },
+        ])
+        .addTargets(
+          [
+            $.addTargetSchema(
+              expr='sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_get_obj_bytes) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+              datasource='${datasource}',
+              format='time_series',
+              instant=false,
+              legendFormat='{{ceph_daemon}} - {{bucket}}',
+              step=300,
+              range=true,
+            ),
+          ]
+        ) + { options: { legend: { calcs: ['lastNotNull'], displayMode: 'table', placement: 'bottom', showLegend: true, sortBy: 'Last *', sortDesc: true }, tooltip: { mode: 'single', sort: 'desc' } } },
+
+
+        $.timeSeriesPanel(
+          lineInterpolation='linear',
+          lineWidth=1,
+          drawStyle='line',
+          axisPlacement='auto',
+          title='Bucket Copy by Size',
+          datasource='${datasource}',
+          gridPosition={ h: 8, w: 6, x: 12, y: 37 },
+          fillOpacity=0,
+          pointSize=5,
+          showPoints='auto',
+          unit='decbytes',
+          displayMode='table',
+          showLegend=true,
+          placement='bottom',
+          tooltip={ mode: 'single', sort: 'desc' },
+          stackingMode='none',
+          spanNulls=true,
+          decimals=2,
+          thresholdsMode='absolute',
+          sortBy='Last *',
+          sortDesc=true
+        )
+        .addThresholds([
+          { color: 'green' },
+        ])
+        .addOverrides([
+          { matcher: { id: 'byType', unit: 'number' }, properties: [{ id: 'color' }, { id: 'color', value: { mode: 'palette-classic' } }] },
+        ])
+        .addTargets(
+          [
+            $.addTargetSchema(
+              expr='sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_copy_obj_bytes) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+              datasource='${datasource}',
+              format='time_series',
+              instant=false,
+              legendFormat='{{ceph_daemon}} - {{bucket}}',
+              step=300,
+              range=true,
+            ),
+          ]
+        ) + { options: { legend: { calcs: ['lastNotNull'], displayMode: 'table', placement: 'bottom', showLegend: true, sortBy: 'Last *', sortDesc: true }, tooltip: { mode: 'single', sort: 'desc' } } },
+
+
+        $.timeSeriesPanel(
+          lineInterpolation='linear',
+          lineWidth=1,
+          drawStyle='line',
+          axisPlacement='auto',
+          title='Bucket Delete by Size',
+          datasource='${datasource}',
+          gridPosition={ h: 8, w: 6, x: 18, y: 37 },
+          fillOpacity=0,
+          pointSize=5,
+          showPoints='auto',
+          unit='decbytes',
+          displayMode='table',
+          showLegend=true,
+          placement='bottom',
+          tooltip={ mode: 'single', sort: 'desc' },
+          stackingMode='none',
+          spanNulls=true,
+          decimals=2,
+          thresholdsMode='absolute',
+          sortBy='Last *',
+          sortDesc=true
+        )
+        .addThresholds([
+          { color: 'green' },
+        ])
+        .addOverrides([
+          { matcher: { id: 'byType', unit: 'number' }, properties: [{ id: 'color' }, { id: 'color', value: { mode: 'palette-classic' } }] },
+        ])
+        .addTargets(
+          [
+            $.addTargetSchema(
+              expr='sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_del_obj_bytes) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+              datasource='${datasource}',
+              format='time_series',
+              instant=false,
+              legendFormat='{{ceph_daemon}} - {{bucket}}',
+              step=300,
+              range=true,
+            ),
+          ]
+        ) + { options: { legend: { calcs: ['lastNotNull'], displayMode: 'table', placement: 'bottom', showLegend: true, sortBy: 'Last *', sortDesc: true }, tooltip: { mode: 'single', sort: 'desc' } } },
+
+
+        $.timeSeriesPanel(
+          lineInterpolation='linear',
+          lineWidth=1,
+          drawStyle='line',
+          axisPlacement='auto',
+          title='Bucket GETs by Operations',
+          datasource='${datasource}',
+          gridPosition={ h: 8, w: 6, x: 0, y: 45 },
+          fillOpacity=0,
+          pointSize=5,
+          showPoints='auto',
+          unit='none',
+          displayMode='table',
+          showLegend=true,
+          placement='bottom',
+          tooltip={ mode: 'single', sort: 'desc' },
+          stackingMode='none',
+          spanNulls=true,
+          decimals=2,
+          thresholdsMode='absolute',
+          sortBy='Last *',
+          sortDesc=true
+        )
+        .addThresholds([
+          { color: 'green' },
+        ])
+        .addOverrides([
+          { matcher: { id: 'byType', unit: 'number' }, properties: [{ id: 'color' }, { id: 'color', value: { mode: 'palette-classic' } }] },
+        ])
+        .addTargets(
+          [
+            $.addTargetSchema(
+              expr='sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_get_obj_ops) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+              datasource='${datasource}',
+              format='time_series',
+              instant=false,
+              legendFormat='{{ceph_daemon}} - {{bucket}}',
+              step=300,
+              range=true,
+            ),
+          ]
+        ) + { options: { legend: { calcs: ['lastNotNull'], displayMode: 'table', placement: 'bottom', showLegend: true, sortBy: 'Last *', sortDesc: true }, tooltip: { mode: 'single', sort: 'desc' } } },
+
+
+        $.timeSeriesPanel(
+          lineInterpolation='linear',
+          lineWidth=1,
+          drawStyle='line',
+          axisPlacement='auto',
+          title='Bucket PUTs by Operations',
+          datasource='${datasource}',
+          gridPosition={ h: 8, w: 6, x: 6, y: 45 },
+          fillOpacity=0,
+          pointSize=5,
+          showPoints='auto',
+          unit='none',
+          displayMode='table',
+          showLegend=true,
+          placement='bottom',
+          tooltip={ mode: 'single', sort: 'desc' },
+          stackingMode='none',
+          spanNulls=true,
+          decimals=2,
+          thresholdsMode='absolute',
+          sortBy='Last *',
+          sortDesc=true
+        )
+        .addThresholds([
+          { color: 'green' },
+        ])
+        .addOverrides([
+          { matcher: { id: 'byType', unit: 'number' }, properties: [{ id: 'color' }, { id: 'color', value: { mode: 'palette-classic' } }] },
+        ])
+        .addTargets(
+          [
+            $.addTargetSchema(
+              expr='sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_put_obj_ops) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+              datasource='${datasource}',
+              format='time_series',
+              instant=false,
+              legendFormat='{{ceph_daemon}} - {{bucket}}',
+              step=300,
+              range=true,
+            ),
+          ]
+        ) + { options: { legend: { calcs: ['lastNotNull'], displayMode: 'table', placement: 'bottom', showLegend: true, sortBy: 'Last *', sortDesc: true }, tooltip: { mode: 'single', sort: 'desc' } } },
+
+
+        $.timeSeriesPanel(
+          lineInterpolation='linear',
+          lineWidth=1,
+          drawStyle='line',
+          axisPlacement='auto',
+          title='Bucket List by Operations',
+          datasource='${datasource}',
+          gridPosition={ h: 8, w: 6, x: 12, y: 45 },
+          fillOpacity=0,
+          pointSize=5,
+          showPoints='auto',
+          unit='none',
+          displayMode='table',
+          showLegend=true,
+          placement='bottom',
+          tooltip={ mode: 'single', sort: 'desc' },
+          stackingMode='none',
+          spanNulls=true,
+          decimals=2,
+          thresholdsMode='absolute',
+          sortBy='Last *',
+          sortDesc=true
+        )
+        .addThresholds([
+          { color: 'green' },
+        ])
+        .addOverrides([
+          { matcher: { id: 'byType', unit: 'number' }, properties: [{ id: 'color' }, { id: 'color', value: { mode: 'palette-classic' } }] },
+        ])
+        .addTargets(
+          [
+            $.addTargetSchema(
+              expr='sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_list_obj_ops) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+              datasource='${datasource}',
+              format='time_series',
+              instant=false,
+              legendFormat='{{ceph_daemon}} - {{bucket}}',
+              step=300,
+              range=true,
+            ),
+          ]
+        ) + { options: { legend: { calcs: ['lastNotNull'], displayMode: 'table', placement: 'bottom', showLegend: true, sortBy: 'Last *', sortDesc: true }, tooltip: { mode: 'single', sort: 'desc' } } },
+
+
+        $.timeSeriesPanel(
+          lineInterpolation='linear',
+          lineWidth=1,
+          drawStyle='line',
+          axisPlacement='auto',
+          title='Bucket Delete by Operations',
+          datasource='${datasource}',
+          gridPosition={ h: 8, w: 6, x: 18, y: 45 },
+          fillOpacity=0,
+          pointSize=5,
+          showPoints='auto',
+          unit='none',
+          displayMode='table',
+          showLegend=true,
+          placement='bottom',
+          tooltip={ mode: 'single', sort: 'desc' },
+          stackingMode='none',
+          spanNulls=true,
+          decimals=2,
+          thresholdsMode='absolute',
+          sortBy='Last *',
+          sortDesc=true
+        )
+        .addThresholds([
+          { color: 'green' },
+        ])
+        .addOverrides([
+          { matcher: { id: 'byType', unit: 'number' }, properties: [{ id: 'color' }, { id: 'color', value: { mode: 'palette-classic' } }] },
+        ])
+        .addTargets(
+          [
+            $.addTargetSchema(
+              expr='sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_del_obj_ops) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+              datasource='${datasource}',
+              format='time_series',
+              instant=false,
+              legendFormat='{{ceph_daemon}} - {{bucket}}',
+              step=300,
+              range=true,
+            ),
+          ]
+        ) + { options: { legend: { calcs: ['lastNotNull'], displayMode: 'table', placement: 'bottom', showLegend: true, sortBy: 'Last *', sortDesc: true }, tooltip: { mode: 'single', sort: 'desc' } } },
+
+
+        $.timeSeriesPanel(
+          lineInterpolation='linear',
+          lineWidth=1,
+          drawStyle='line',
+          axisPlacement='auto',
+          title='Bucket Copy by Operations',
+          datasource='${datasource}',
+          gridPosition={ h: 8, w: 12, x: 0, y: 53 },
+          fillOpacity=0,
+          pointSize=5,
+          showPoints='auto',
+          unit='none',
+          displayMode='table',
+          showLegend=true,
+          placement='bottom',
+          tooltip={ mode: 'single', sort: 'desc' },
+          stackingMode='none',
+          spanNulls=true,
+          decimals=2,
+          thresholdsMode='absolute',
+          sortBy='Last *',
+          sortDesc=true
+        )
+        .addThresholds([
+          { color: 'green' },
+        ])
+        .addOverrides([
+          { matcher: { id: 'byType', unit: 'number' }, properties: [{ id: 'color' }, { id: 'color', value: { mode: 'palette-classic' } }] },
+        ])
+        .addTargets(
+          [
+            $.addTargetSchema(
+              expr='sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_copy_obj_ops) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+              datasource='${datasource}',
+              format='time_series',
+              instant=false,
+              legendFormat='{{ceph_daemon}} - {{bucket}}',
+              step=300,
+              range=true,
+            ),
+          ]
+        ) + { options: { legend: { calcs: ['lastNotNull'], displayMode: 'table', placement: 'bottom', showLegend: true, sortBy: 'Last *', sortDesc: true }, tooltip: { mode: 'single', sort: 'desc' } } },
+
+
+        $.addTableExtended(
+          datasource='${datasource}',
+          title='Summary Per Bucket by Operations',
+          gridPosition={ h: 8, w: 12, x: 12, y: 53 },
+          options={
+            footer: {
+              fields: '',
+              reducer: ['sum'],
+              countRows: false,
+              enablePagination: false,
+              show: false,
+            },
+            frameIndex: 1,
+            showHeader: true,
+            sortBy: [
+              {
+                desc: true,
+                displayName: 'PUTs',
+              },
+            ],
+          },
+          custom={ align: 'auto', cellOptions: { type: 'auto' }, filterable: false, inspect: false },
+          thresholds={
+            mode: 'absolute',
+            steps: [
+              { color: 'green' },
+            ],
+          },
+          overrides=[{
+            matcher: { id: 'byType', options: 'number' },
+            properties: [
+              { id: 'unit', value: 'none' },
+            ],
+          }],
+          pluginVersion='9.4.7'
+        )
+        .addTransformations([
+          {
+            id: 'merge',
+            options: {},
+          },
+          {
+            id: 'joinByField',
+            options: {
+              byField: 'Bucket',
+              mode: 'outer',
+            },
+          },
+          {
+            id: 'groupBy',
+            options: {
+              fields: {
+                Bucket: {
+                  aggregations: [],
+                  operation: 'groupby',
+                },
+                'Value #A': {
+                  aggregations: [],
+                  operation: 'groupby',
+                },
+                'Value #B': {
+                  aggregations: [],
+                  operation: 'groupby',
+                },
+                'Value #C': {
+                  aggregations: [],
+                  operation: 'groupby',
+                },
+                'Value #D': {
+                  aggregations: [],
+                  operation: 'groupby',
+                },
+                'Value #F': {
+                  aggregations: [],
+                  operation: 'groupby',
+                },
+                bucket: {
+                  aggregations: [],
+                  operation: 'groupby',
+                },
+                ceph_daemon: {
+                  aggregations: [],
+                  operation: 'groupby',
+                },
+              },
+            },
+          },
+          {
+            id: 'organize',
+            options: {
+              excludeByName: {
+                'Time 1': true,
+                'Time 2': true,
+                'Time 3': true,
+                'Time 4': true,
+                'Time 5': true,
+                'Time 6': true,
+                'Time 7': true,
+                __name__: true,
+                '__name__ 1': true,
+                '__name__ 2': true,
+                '__name__ 3': true,
+                '__name__ 4': true,
+                '__name__ 5': true,
+                '__name__ 6': true,
+                '__name__ 7': true,
+                'ceph_daemon 1': true,
+                'ceph_daemon 2': true,
+                'ceph_daemon 3': true,
+                'ceph_daemon 4': true,
+                'instance 1': true,
+                'instance 2': true,
+                'instance 3': true,
+                'instance 4': true,
+                'instance 5': true,
+                'instance 6': true,
+                'instance 7': true,
+                'instance_id 1': true,
+                'instance_id 2': true,
+                'instance_id 3': true,
+                'instance_id 4': true,
+                'instance_id 5': true,
+                'instance_id 6': true,
+                'instance_id 7': true,
+                'job 1': true,
+                'job 2': true,
+                'job 3': true,
+                'job 4': true,
+                'job 5': true,
+                'job 6': true,
+                'job 7': true,
+              },
+              indexByName: {
+                'Value #A': 2,
+                'Value #B': 3,
+                'Value #C': 4,
+                'Value #D': 5,
+                'Value #F': 6,
+                bucket: 1,
+                ceph_daemon: 0,
+              },
+              renameByName: {
+                Bucket: '',
+                'Value #A': 'PUTs',
+                'Value #B': 'GETs',
+                'Value #C': 'List',
+                'Value #D': 'Delete',
+                'Value #E': 'Copy',
+                'Value #F': 'Copy',
+                'Value #G': '',
+                bucket: 'Bucket',
+                ceph_daemon: 'Daemon',
+              },
+            },
+          },
+        ]).addTargets([
+          $.addTargetSchema(
+            expr='sum by (bucket, ceph_daemon) (ceph_rgw_op_per_bucket_put_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+            datasource={ type: 'prometheus', uid: '${datasource}' },
+            format='table',
+            hide=false,
+            exemplar=false,
+            instant=true,
+            interval='',
+            legendFormat='__auto',
+            range=false,
+          ),
+          $.addTargetSchema(
+            expr='sum by (bucket, ceph_daemon) (ceph_rgw_op_per_bucket_get_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+            datasource={ type: 'prometheus', uid: '${datasource}' },
+            format='table',
+            hide=false,
+            exemplar=false,
+            instant=true,
+            interval='',
+            legendFormat='__auto',
+            range=false,
+          ),
+          $.addTargetSchema(
+            expr='sum by (bucket, ceph_daemon) (ceph_rgw_op_per_bucket_del_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+            datasource={ type: 'prometheus', uid: '${datasource}' },
+            format='table',
+            hide=false,
+            exemplar=false,
+            instant=true,
+            interval='',
+            legendFormat='__auto',
+            range=false,
+          ),
+          $.addTargetSchema(
+            expr='sum by (bucket, ceph_daemon) (ceph_rgw_op_per_bucket_copy_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+            datasource={ type: 'prometheus', uid: '${datasource}' },
+            format='table',
+            hide=false,
+            exemplar=false,
+            instant=true,
+            interval='',
+            legendFormat='__auto',
+            range=false,
+          ),
+          $.addTargetSchema(
+            expr='sum by (bucket, ceph_daemon) (ceph_rgw_op_per_bucket_list_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+            datasource={ type: 'prometheus', uid: '${datasource}' },
+            format='table',
+            hide=false,
+            exemplar=false,
+            instant=true,
+            interval='',
+            legendFormat='__auto',
+            range=false,
+          ),
+        ]),
+      ]) + { gridPos: { x: 0, y: 28, w: 24, h: 1 } },
+
+
+      $.addRowSchema(false, true, 'Users', collapsed=true)
+      .addPanels([
+        $.addBarGaugePanel(
+          title='Top 5 Users PUTs By Operations',
+          datasource='${datasource}',
+          gridPosition={ x: 0, y: 62, w: 6, h: 8 },
+          unit='none',
+          thresholds={ color: 'green' }
+        )
+        .addTargets([
+          $.addTargetSchema(
+            expr='topk(5, \n    sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_put_obj_ops ) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})\n)\n',
+            datasource='${datasource}',
+            legendFormat='{{ceph_daemon}} - {{user}}',
+            range=false,
+            instant=true
+          ),
+        ]) + { fieldConfig: { defaults: { color: { mode: 'thresholds' }, thresholds: { mode: 'absolute', steps: [{ color: 'green' }] } }, overrides: [{ matcher: { id: 'byType', unit: 'number' }, properties: [{ id: 'color' }, { id: 'color', value: { mode: 'palette-classic' } }] }] } }
+        + { options: { orientation: 'horizontal', reduceOptions: { calcs: ['lastNotNull'] }, displayMode: 'gradient' } },
+
+
+        $.addBarGaugePanel(
+          title='Top 5 Users GETs by Operations',
+          datasource='${datasource}',
+          gridPosition={ x: 6, y: 62, w: 6, h: 8 },
+          unit='none',
+          thresholds={ color: 'green', value: null }
+        )
+        .addTargets([
+          $.addTargetSchema(
+            expr='topk(5, \n    sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_get_obj_ops ) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})\n)\n',
+            datasource='${datasource}',
+            legendFormat='{{ceph_daemon}} - {{user}}',
+            range=false,
+            instant=true
+          ),
+        ]) + { fieldConfig: { defaults: { color: { mode: 'thresholds' }, thresholds: { mode: 'absolute', steps: [{ color: 'green' }] } }, overrides: [{ matcher: { id: 'byType', unit: 'number' }, properties: [{ id: 'color' }, { id: 'color', value: { mode: 'palette-classic' } }] }] } }
+        + { options: { orientation: 'horizontal', reduceOptions: { calcs: ['lastNotNull'] }, displayMode: 'gradient' } },
+
+
+        $.addBarGaugePanel(
+          title='Top 5 Users PUTs by Size',
+          datasource='${datasource}',
+          gridPosition={ x: 12, y: 62, w: 6, h: 8 },
+          unit='decbytes',
+          thresholds={ color: 'green', value: null }
+        )
+        .addTargets([
+          $.addTargetSchema(
+            expr='topk(5, \n    sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_put_obj_bytes) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})\n)',
+            datasource='${datasource}',
+            legendFormat='{{ceph_daemon}} - {{user}}',
+            range=false,
+            instant=true
+          ),
+        ]) + { fieldConfig: { defaults: { color: { mode: 'thresholds' }, thresholds: { mode: 'absolute', steps: [{ color: 'green' }] } } } }
+        + { options: { orientation: 'horizontal', reduceOptions: { calcs: [] }, displayMode: 'gradient' } },
+
+
+        $.addBarGaugePanel(
+          title='Top 5 Users GETs By Size',
+          datasource='${datasource}',
+          gridPosition={ x: 18, y: 62, w: 6, h: 8 },
+          unit='decbytes',
+          thresholds={ color: 'green', value: null }
+        )
+        .addTargets([
+          $.addTargetSchema(
+            expr='topk(5, \n    sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_get_obj_bytes) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})\n)',
+            datasource='${datasource}',
+            legendFormat='{{ceph_daemon}} - {{user}}',
+            range=false,
+            instant=true
+          ),
+        ]) + { fieldConfig: { defaults: { color: { mode: 'thresholds' }, thresholds: { mode: 'absolute', steps: [{ color: 'green' }] } } } }
+        + { options: { orientation: 'horizontal', reduceOptions: { calcs: [] }, displayMode: 'gradient' } },
+
+
+        $.timeSeriesPanel(
+          lineInterpolation='linear',
+          lineWidth=1,
+          drawStyle='line',
+          axisPlacement='auto',
+          title='User PUTs by Size',
+          datasource='${datasource}',
+          gridPosition={ h: 8, w: 6, x: 0, y: 70 },
+          fillOpacity=0,
+          pointSize=5,
+          showPoints='auto',
+          unit='decbytes',
+          displayMode='table',
+          showLegend=true,
+          placement='bottom',
+          tooltip={ mode: 'single', sort: 'desc' },
+          stackingMode='none',
+          spanNulls=true,
+          decimals=2,
+          thresholdsMode='absolute',
+          sortBy='Last *',
+          sortDesc=true
+        )
+        .addThresholds([
+          { color: 'green' },
+        ])
+        .addOverrides([
+          { matcher: { id: 'byType', unit: 'number' }, properties: [{ id: 'color' }, { id: 'color', value: { mode: 'palette-classic' } }] },
+        ])
+        .addTargets(
+          [
+            $.addTargetSchema(
+              expr='sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_put_obj_bytes) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+              datasource='${datasource}',
+              format='time_series',
+              instant=false,
+              legendFormat='{{ceph_daemon}} - {{user}}',
+              step=300,
+              range=true,
+            ),
+          ]
+        ) + { options: { legend: { calcs: ['lastNotNull'], displayMode: 'table', placement: 'bottom', showLegend: true, sortBy: 'Last *', sortDesc: true }, tooltip: { mode: 'single', sort: 'desc' } } },
+
+
+        $.timeSeriesPanel(
+          lineInterpolation='linear',
+          lineWidth=1,
+          drawStyle='line',
+          axisPlacement='auto',
+          title='User GETs by Size',
+          datasource='${datasource}',
+          gridPosition={ h: 8, w: 6, x: 6, y: 70 },
+          fillOpacity=0,
+          pointSize=5,
+          showPoints='auto',
+          unit='decbytes',
+          displayMode='table',
+          showLegend=true,
+          placement='bottom',
+          tooltip={ mode: 'single', sort: 'desc' },
+          stackingMode='none',
+          spanNulls=true,
+          decimals=2,
+          thresholdsMode='absolute',
+          sortBy='Last *',
+          sortDesc=true
+        )
+        .addThresholds([
+          { color: 'green' },
+        ])
+        .addOverrides([
+          { matcher: { id: 'byType', unit: 'number' }, properties: [{ id: 'color' }, { id: 'color', value: { mode: 'palette-classic' } }] },
+        ])
+        .addTargets(
+          [
+            $.addTargetSchema(
+              expr='sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_get_obj_bytes) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+              datasource='${datasource}',
+              format='time_series',
+              instant=false,
+              legendFormat='{{ceph_daemon}} - {{user}}',
+              step=300,
+              range=true,
+            ),
+          ]
+        ) + { options: { legend: { calcs: ['lastNotNull'], displayMode: 'table', placement: 'bottom', showLegend: true, sortBy: 'Last *', sortDesc: true }, tooltip: { mode: 'single', sort: 'desc' } } },
+
+
+        $.timeSeriesPanel(
+          lineInterpolation='linear',
+          lineWidth=1,
+          drawStyle='line',
+          axisPlacement='auto',
+          title='User Delete by Size',
+          datasource='${datasource}',
+          gridPosition={ h: 8, w: 6, x: 12, y: 70 },
+          fillOpacity=0,
+          pointSize=5,
+          showPoints='auto',
+          unit='decbytes',
+          displayMode='table',
+          showLegend=true,
+          placement='bottom',
+          tooltip={ mode: 'single', sort: 'desc' },
+          stackingMode='none',
+          spanNulls=true,
+          decimals=2,
+          thresholdsMode='absolute',
+          sortBy='Last *',
+          sortDesc=true
+        )
+        .addThresholds([
+          { color: 'green' },
+        ])
+        .addOverrides([
+          { matcher: { id: 'byType', unit: 'number' }, properties: [{ id: 'color' }, { id: 'color', value: { mode: 'palette-classic' } }] },
+        ])
+        .addTargets(
+          [
+            $.addTargetSchema(
+              expr='sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_del_obj_bytes) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+              datasource='${datasource}',
+              format='time_series',
+              instant=false,
+              legendFormat='{{ceph_daemon}} - {{user}}',
+              step=300,
+              range=true,
+            ),
+          ]
+        ) + { options: { legend: { calcs: ['lastNotNull'], displayMode: 'table', placement: 'bottom', showLegend: true, sortBy: 'Last *', sortDesc: true }, tooltip: { mode: 'single', sort: 'desc' } } },
+
+
+        $.timeSeriesPanel(
+          lineInterpolation='linear',
+          lineWidth=1,
+          drawStyle='line',
+          axisPlacement='auto',
+          title='User COPY by Size',
+          datasource='${datasource}',
+          gridPosition={ h: 8, w: 6, x: 18, y: 70 },
+          fillOpacity=0,
+          pointSize=5,
+          showPoints='auto',
+          unit='decbytes',
+          displayMode='table',
+          showLegend=true,
+          placement='bottom',
+          tooltip={ mode: 'single', sort: 'desc' },
+          stackingMode='none',
+          spanNulls=true,
+          decimals=2,
+          thresholdsMode='absolute',
+          sortBy='Last *',
+          sortDesc=true
+        )
+        .addThresholds([
+          { color: 'green' },
+        ])
+        .addOverrides([
+          { matcher: { id: 'byType', unit: 'number' }, properties: [{ id: 'color' }, { id: 'color', value: { mode: 'palette-classic' } }] },
+        ])
+        .addTargets(
+          [
+            $.addTargetSchema(
+              expr='sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_copy_obj_bytes) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+              datasource='${datasource}',
+              format='time_series',
+              instant=false,
+              legendFormat='{{ceph_daemon}} - {{user}}',
+              step=300,
+              range=true,
+            ),
+          ]
+        ) + { options: { legend: { calcs: ['lastNotNull'], displayMode: 'table', placement: 'bottom', showLegend: true, sortBy: 'Last *', sortDesc: true }, tooltip: { mode: 'single', sort: 'desc' } } },
+
+
+        $.timeSeriesPanel(
+          lineInterpolation='linear',
+          lineWidth=1,
+          drawStyle='line',
+          axisPlacement='auto',
+          title='User GETs by Operations',
+          datasource='${datasource}',
+          gridPosition={ h: 8, w: 6, x: 0, y: 78 },
+          fillOpacity=0,
+          pointSize=5,
+          showPoints='auto',
+          unit='none',
+          displayMode='table',
+          showLegend=true,
+          placement='bottom',
+          tooltip={ mode: 'single', sort: 'desc' },
+          stackingMode='none',
+          spanNulls=true,
+          decimals=2,
+          thresholdsMode='absolute',
+          sortBy='Last *',
+          sortDesc=true
+        )
+        .addThresholds([
+          { color: 'green' },
+        ])
+        .addOverrides([
+          { matcher: { id: 'byType', unit: 'number' }, properties: [{ id: 'color' }, { id: 'color', value: { mode: 'palette-classic' } }] },
+        ])
+        .addTargets(
+          [
+            $.addTargetSchema(
+              expr='sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_get_obj_ops) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+              datasource='${datasource}',
+              format='time_series',
+              instant=false,
+              legendFormat='{{ceph_daemon}} - {{user}}',
+              step=300,
+              range=true,
+            ),
+          ]
+        ) + { options: { legend: { calcs: ['lastNotNull'], displayMode: 'table', placement: 'bottom', showLegend: true, sortBy: 'Last *', sortDesc: true }, tooltip: { mode: 'single', sort: 'desc' } } },
+
+
+        $.timeSeriesPanel(
+          lineInterpolation='linear',
+          lineWidth=1,
+          drawStyle='line',
+          axisPlacement='auto',
+          title='User PUTs by Operations',
+          datasource='${datasource}',
+          gridPosition={ h: 8, w: 6, x: 6, y: 78 },
+          fillOpacity=0,
+          pointSize=5,
+          showPoints='auto',
+          unit='none',
+          displayMode='table',
+          showLegend=true,
+          placement='bottom',
+          tooltip={ mode: 'single', sort: 'desc' },
+          stackingMode='none',
+          spanNulls=true,
+          decimals=2,
+          thresholdsMode='absolute',
+          sortBy='Last *',
+          sortDesc=true
+        )
+        .addThresholds([
+          { color: 'green' },
+        ])
+        .addOverrides([
+          { matcher: { id: 'byType', unit: 'number' }, properties: [{ id: 'color' }, { id: 'color', value: { mode: 'palette-classic' } }] },
+        ])
+        .addTargets(
+          [
+            $.addTargetSchema(
+              expr='sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_put_obj_ops) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+              datasource='${datasource}',
+              format='time_series',
+              instant=false,
+              legendFormat='{{ceph_daemon}} - {{user}}',
+              step=300,
+              range=true,
+            ),
+          ]
+        ) + { options: { legend: { calcs: ['lastNotNull'], displayMode: 'table', placement: 'bottom', showLegend: true, sortBy: 'Last *', sortDesc: true }, tooltip: { mode: 'single', sort: 'desc' } } },
+
+
+        $.timeSeriesPanel(
+          lineInterpolation='linear',
+          lineWidth=1,
+          drawStyle='line',
+          axisPlacement='auto',
+          title='User List by Operations',
+          datasource='${datasource}',
+          gridPosition={ h: 8, w: 6, x: 12, y: 78 },
+          fillOpacity=0,
+          pointSize=5,
+          showPoints='auto',
+          unit='none',
+          displayMode='table',
+          showLegend=true,
+          placement='bottom',
+          tooltip={ mode: 'single', sort: 'desc' },
+          stackingMode='none',
+          spanNulls=true,
+          decimals=2,
+          thresholdsMode='absolute',
+          sortBy='Last *',
+          sortDesc=true
+        )
+        .addThresholds([
+          { color: 'green' },
+        ])
+        .addOverrides([
+          { matcher: { id: 'byType', unit: 'number' }, properties: [{ id: 'color' }, { id: 'color', value: { mode: 'palette-classic' } }] },
+        ])
+        .addTargets(
+          [
+            $.addTargetSchema(
+              expr='sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_list_obj_ops) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+              datasource='${datasource}',
+              format='time_series',
+              instant=false,
+              legendFormat='{{ceph_daemon}} - {{user}}',
+              step=300,
+              range=true,
+            ),
+          ]
+        ) + { options: { legend: { calcs: ['lastNotNull'], displayMode: 'table', placement: 'bottom', showLegend: true, sortBy: 'Last *', sortDesc: true }, tooltip: { mode: 'single', sort: 'desc' } } },
+
+
+        $.timeSeriesPanel(
+          lineInterpolation='linear',
+          lineWidth=1,
+          drawStyle='line',
+          axisPlacement='auto',
+          title='User Delete by Operations',
+          datasource='${datasource}',
+          gridPosition={ h: 8, w: 6, x: 18, y: 78 },
+          fillOpacity=0,
+          pointSize=5,
+          showPoints='auto',
+          unit='none',
+          displayMode='table',
+          showLegend=true,
+          placement='bottom',
+          tooltip={ mode: 'single', sort: 'desc' },
+          stackingMode='none',
+          spanNulls=true,
+          decimals=2,
+          thresholdsMode='absolute',
+          sortBy='Last *',
+          sortDesc=true
+        )
+        .addThresholds([
+          { color: 'green' },
+        ])
+        .addOverrides([
+          { matcher: { id: 'byType', unit: 'number' }, properties: [{ id: 'color' }, { id: 'color', value: { mode: 'palette-classic' } }] },
+        ])
+        .addTargets(
+          [
+            $.addTargetSchema(
+              expr='sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_del_obj_ops) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+              datasource='${datasource}',
+              format='time_series',
+              instant=false,
+              legendFormat='{{ceph_daemon}} - {{user}}',
+              step=300,
+              range=true,
+            ),
+          ]
+        ) + { options: { legend: { calcs: ['lastNotNull'], displayMode: 'table', placement: 'bottom', showLegend: true, sortBy: 'Last *', sortDesc: true }, tooltip: { mode: 'single', sort: 'desc' } } },
+
+
+        $.timeSeriesPanel(
+          lineInterpolation='linear',
+          lineWidth=1,
+          drawStyle='line',
+          axisPlacement='auto',
+          title='User Copy by Operations',
+          datasource='${datasource}',
+          gridPosition={ h: 8, w: 12, x: 0, y: 86 },
+          fillOpacity=0,
+          pointSize=5,
+          showPoints='auto',
+          unit='none',
+          displayMode='table',
+          showLegend=true,
+          placement='bottom',
+          tooltip={ mode: 'single', sort: 'desc' },
+          stackingMode='none',
+          spanNulls=true,
+          decimals=2,
+          thresholdsMode='absolute',
+          sortBy='Last *',
+          sortDesc=true
+        )
+        .addThresholds([
+          { color: 'green' },
+        ])
+        .addOverrides([
+          { matcher: { id: 'byType', unit: 'number' }, properties: [{ id: 'color' }, { id: 'color', value: { mode: 'palette-classic' } }] },
+        ])
+        .addTargets(
+          [
+            $.addTargetSchema(
+              expr='sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_copy_obj_ops) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+              datasource='${datasource}',
+              format='time_series',
+              instant=false,
+              legendFormat='{{ceph_daemon}} - {{user}}',
+              step=300,
+              range=true,
+            ),
+          ]
+        ) + { options: { legend: { calcs: ['lastNotNull'], displayMode: 'table', placement: 'bottom', showLegend: true, sortBy: 'Last *', sortDesc: true }, tooltip: { mode: 'single', sort: 'desc' } } },
+
+
+        $.addTableExtended(
+          datasource='${datasource}',
+          title='Summary Per User By Operations',
+          gridPosition={ h: 8, w: 12, x: 12, y: 86 },
+          options={
+            footer: {
+              fields: '',
+              reducer: ['sum'],
+              countRows: false,
+              enablePagination: false,
+              show: false,
+            },
+            frameIndex: 1,
+            showHeader: true,
+            sortBy: [
+              {
+                desc: true,
+                displayName: 'PUTs',
+              },
+            ],
+          },
+          custom={ align: 'auto', cellOptions: { type: 'auto' }, filterable: false, inspect: false },
+          thresholds={
+            mode: 'absolute',
+            steps: [
+              { color: 'green' },
+            ],
+          },
+          overrides=[{
+            matcher: { id: 'byType', options: 'number' },
+            properties: [
+              { id: 'unit', value: 'none' },
+            ],
+          }],
+          pluginVersion='9.4.7'
+        )
+        .addTransformations([
+          {
+            id: 'merge',
+            options: {},
+          },
+          {
+            id: 'joinByField',
+            options: {
+              byField: 'User',
+              mode: 'outer',
+            },
+          },
+          {
+            id: 'groupBy',
+            options: {
+              fields: {
+                User: {
+                  aggregations: [],
+                  operation: 'groupby',
+                },
+                'Value #A': {
+                  aggregations: [],
+                  operation: 'groupby',
+                },
+                'Value #B': {
+                  aggregations: [],
+                  operation: 'groupby',
+                },
+                'Value #C': {
+                  aggregations: [],
+                  operation: 'groupby',
+                },
+                'Value #D': {
+                  aggregations: [],
+                  operation: 'groupby',
+                },
+                'Value #F': {
+                  aggregations: [],
+                  operation: 'groupby',
+                },
+                ceph_daemon: {
+                  aggregations: [],
+                  operation: 'groupby',
+                },
+                user: {
+                  aggregations: [],
+                  operation: 'groupby',
+                },
+              },
+            },
+          },
+          {
+            id: 'organize',
+            options: {
+              excludeByName: {},
+              indexByName: {
+                'Value #A': 2,
+                'Value #B': 3,
+                'Value #C': 4,
+                'Value #D': 5,
+                'Value #F': 6,
+                ceph_daemon: 0,
+                user: 1,
+              },
+              renameByName: {
+                'Value #A': 'PUTs',
+                'Value #B': 'GETs',
+                'Value #C': 'LIST',
+                'Value #D': 'DELETE',
+                'Value #F': 'COPY',
+                ceph_daemon: 'Daemon',
+                user: 'User',
+              },
+            },
+          },
+        ]).addTargets([
+          $.addTargetSchema(
+            expr='sum by (user, ceph_daemon) (ceph_rgw_op_per_user_put_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+            datasource={ type: 'prometheus', uid: '${datasource}' },
+            format='table',
+            hide=false,
+            exemplar=false,
+            instant=true,
+            interval='',
+            legendFormat='__auto',
+            range=false,
+          ),
+          $.addTargetSchema(
+            expr='sum by (user, ceph_daemon) (ceph_rgw_op_per_user_get_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+            datasource={ type: 'prometheus', uid: '${datasource}' },
+            format='table',
+            hide=false,
+            exemplar=false,
+            instant=true,
+            interval='',
+            legendFormat='__auto',
+            range=false,
+          ),
+          $.addTargetSchema(
+            expr='sum by (user, ceph_daemon) (ceph_rgw_op_per_user_del_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+            datasource={ type: 'prometheus', uid: '${datasource}' },
+            format='table',
+            hide=false,
+            exemplar=false,
+            instant=true,
+            interval='',
+            legendFormat='__auto',
+            range=false,
+          ),
+          $.addTargetSchema(
+            expr='sum by (user, ceph_daemon) (ceph_rgw_op_per_user_copy_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+            datasource={ type: 'prometheus', uid: '${datasource}' },
+            format='table',
+            hide=false,
+            exemplar=false,
+            instant=true,
+            interval='',
+            legendFormat='__auto',
+            range=false,
+          ),
+          $.addTargetSchema(
+            expr='sum by (user, ceph_daemon) (ceph_rgw_op_per_user_list_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~"$rgw_servers"})',
+            datasource={ type: 'prometheus', uid: '${datasource}' },
+            format='table',
+            hide=false,
+            exemplar=false,
+            instant=true,
+            interval='',
+            legendFormat='__auto',
+            range=false,
+          ),
+        ]),
+      ]) + { gridPos: { x: 0, y: 29, w: 24, h: 1 } },
+    ]),
+}
diff --git a/monitoring/ceph-mixin/dashboards/utils.libsonnet b/monitoring/ceph-mixin/dashboards/utils.libsonnet
index 16ba771cb3a..397c00fe53a 100644
--- a/monitoring/ceph-mixin/dashboards/utils.libsonnet
+++ b/monitoring/ceph-mixin/dashboards/utils.libsonnet
@@ -524,6 +524,16 @@ local timeSeries = import 'timeseries_panel.libsonnet';
       maxDataPoints: maxDataPoints,
       interval: interval,
     },
+
+  addBarGaugePanel(title='',
+                   description='',
+                   datasource='${DS_PROMETHEUS}',
+                   gridPosition={},
+                   unit='percentunit',
+                   thresholds={})::
+    g.barGaugePanel.new(title, description, datasource, unit, thresholds) + {
+      gridPos: gridPosition,
+    },
   addTableExtended(
     title='',
     datasource=null,
diff --git a/monitoring/ceph-mixin/dashboards_out/rgw-s3-analytics.json b/monitoring/ceph-mixin/dashboards_out/rgw-s3-analytics.json
new file mode 100644
index 00000000000..6a41df1cfaa
--- /dev/null
+++ b/monitoring/ceph-mixin/dashboards_out/rgw-s3-analytics.json
@@ -0,0 +1,4695 @@
+{
+   "__inputs": [ ],
+   "__requires": [ ],
+   "annotations": {
+      "list": [
+         {
+            "builtIn": 1,
+            "datasource": "-- Grafana --",
+            "enable": true,
+            "hide": true,
+            "iconColor": "rgba(0, 211, 255, 1)",
+            "name": "Annotations & Alerts",
+            "showIn": 0,
+            "tags": [ ],
+            "type": "dashboard"
+         }
+      ]
+   },
+   "description": "",
+   "editable": false,
+   "gnetId": null,
+   "graphTooltip": 0,
+   "hideControls": false,
+   "id": null,
+   "links": [ ],
+   "panels": [
+      {
+         "collapse": false,
+         "collapsed": false,
+         "gridPos": {
+            "h": 1,
+            "w": 24,
+            "x": 0,
+            "y": 0
+         },
+         "id": 2,
+         "panels": [ ],
+         "repeat": null,
+         "repeatIteration": null,
+         "repeatRowId": null,
+         "showTitle": true,
+         "title": "Overview",
+         "titleSize": "h6",
+         "type": "row"
+      },
+      {
+         "colors": null,
+         "datasource": "${datasource}",
+         "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "decimals": 0,
+               "links": [ ],
+               "mappings": [ ],
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green",
+                        "value": null
+                     },
+                     {
+                        "color": "red",
+                        "value": 80
+                     }
+                  ]
+               },
+               "unit": "decbytes"
+            }
+         },
+         "gridPos": {
+            "h": 3,
+            "w": 6,
+            "x": 0,
+            "y": 1
+         },
+         "id": 3,
+         "links": [ ],
+         "options": {
+            "colorMode": "none",
+            "graphMode": "none",
+            "justifyMode": "auto",
+            "orientation": "auto",
+            "reduceOptions": {
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "fields": "",
+               "values": false
+            },
+            "textMode": "auto"
+         },
+         "pluginVersion": "9.4.7",
+         "targets": [
+            {
+               "datasource": "${datasource}",
+               "expr": "sum(ceph_rgw_op_put_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "time_series",
+               "intervalFactor": 1,
+               "legendFormat": "__auto",
+               "range": true,
+               "refId": "A"
+            }
+         ],
+         "title": "Total PUTs",
+         "transparent": false,
+         "type": "stat"
+      },
+      {
+         "colors": null,
+         "datasource": "${datasource}",
+         "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "decimals": 0,
+               "links": [ ],
+               "mappings": [ ],
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green",
+                        "value": null
+                     },
+                     {
+                        "color": "red",
+                        "value": 80
+                     }
+                  ]
+               },
+               "unit": "decbytes"
+            }
+         },
+         "gridPos": {
+            "h": 3,
+            "w": 6,
+            "x": 6,
+            "y": 1
+         },
+         "id": 4,
+         "links": [ ],
+         "options": {
+            "colorMode": "none",
+            "graphMode": "none",
+            "justifyMode": "auto",
+            "orientation": "auto",
+            "reduceOptions": {
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "fields": "",
+               "values": false
+            },
+            "textMode": "auto"
+         },
+         "pluginVersion": "9.4.7",
+         "targets": [
+            {
+               "datasource": "${datasource}",
+               "expr": "sum\n(ceph_rgw_op_get_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "time_series",
+               "intervalFactor": 1,
+               "legendFormat": "__auto",
+               "range": true,
+               "refId": "A"
+            }
+         ],
+         "title": "Total GETs",
+         "transparent": false,
+         "type": "stat"
+      },
+      {
+         "colors": null,
+         "datasource": "${datasource}",
+         "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "decimals": 0,
+               "links": [ ],
+               "mappings": [ ],
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green",
+                        "value": null
+                     },
+                     {
+                        "color": "red",
+                        "value": 80
+                     }
+                  ]
+               },
+               "unit": "none"
+            }
+         },
+         "gridPos": {
+            "h": 3,
+            "w": 6,
+            "x": 12,
+            "y": 1
+         },
+         "id": 5,
+         "links": [ ],
+         "options": {
+            "colorMode": "none",
+            "graphMode": "none",
+            "justifyMode": "auto",
+            "orientation": "auto",
+            "reduceOptions": {
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "fields": "",
+               "values": false
+            },
+            "textMode": "auto"
+         },
+         "pluginVersion": "9.4.7",
+         "targets": [
+            {
+               "datasource": "${datasource}",
+               "expr": "sum(ceph_rgw_op_put_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "time_series",
+               "intervalFactor": 1,
+               "legendFormat": "__auto",
+               "range": true,
+               "refId": "A"
+            }
+         ],
+         "title": "Total Objects",
+         "transparent": false,
+         "type": "stat"
+      },
+      {
+         "colors": null,
+         "datasource": "${datasource}",
+         "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "decimals": 0,
+               "links": [ ],
+               "mappings": [ ],
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green",
+                        "value": null
+                     },
+                     {
+                        "color": "red",
+                        "value": 80
+                     }
+                  ]
+               },
+               "unit": "decbytes"
+            }
+         },
+         "gridPos": {
+            "h": 3,
+            "w": 6,
+            "x": 18,
+            "y": 1
+         },
+         "id": 6,
+         "links": [ ],
+         "options": {
+            "colorMode": "none",
+            "graphMode": "none",
+            "justifyMode": "auto",
+            "orientation": "auto",
+            "reduceOptions": {
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "fields": "",
+               "values": false
+            },
+            "textMode": "auto"
+         },
+         "pluginVersion": "9.4.7",
+         "targets": [
+            {
+               "datasource": "${datasource}",
+               "expr": "sum\n((sum by(instance_id)(ceph_rgw_op_put_obj_bytes) > 0) / (sum by(instance_id)(ceph_rgw_op_put_obj_ops) > 0) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "time_series",
+               "intervalFactor": 1,
+               "legendFormat": "__auto",
+               "range": true,
+               "refId": "A"
+            }
+         ],
+         "title": "Average Object Size",
+         "transparent": false,
+         "type": "stat"
+      },
+      {
+         "datasource": "${datasource}",
+         "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "color": {
+                  "mode": "thresholds"
+               },
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green",
+                        "value": null
+                     }
+                  ]
+               }
+            }
+         },
+         "gridPos": {
+            "h": 8,
+            "w": 8,
+            "x": 0,
+            "y": 4
+         },
+         "id": 7,
+         "options": {
+            "displayMode": "gradient",
+            "orientation": "horizontal",
+            "reduceOptions": {
+               "calcs": [
+                  "lastNotNull"
+               ]
+            }
+         },
+         "targets": [
+            {
+               "datasource": "${datasource}",
+               "expr": "sum(ceph_rgw_op_list_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "time_series",
+               "instant": true,
+               "intervalFactor": 1,
+               "legendFormat": "List Objects",
+               "range": false,
+               "refId": "A"
+            },
+            {
+               "datasource": "${datasource}",
+               "expr": "sum(ceph_rgw_op_list_buckets_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "time_series",
+               "intervalFactor": 1,
+               "legendFormat": "List Buckets",
+               "range": true,
+               "refId": "B"
+            },
+            {
+               "datasource": "${datasource}",
+               "expr": "sum(ceph_rgw_op_put_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "time_series",
+               "instant": true,
+               "intervalFactor": 1,
+               "legendFormat": "Put Objects",
+               "range": false,
+               "refId": "C"
+            },
+            {
+               "datasource": "${datasource}",
+               "expr": "sum(ceph_rgw_op_per_bucket_get_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "time_series",
+               "instant": true,
+               "intervalFactor": 1,
+               "legendFormat": "Get Objects",
+               "range": false,
+               "refId": "D"
+            },
+            {
+               "datasource": "${datasource}",
+               "expr": "sum(ceph_rgw_op_del_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "time_series",
+               "instant": true,
+               "intervalFactor": 1,
+               "legendFormat": "Delete Objects",
+               "range": false,
+               "refId": "E"
+            },
+            {
+               "datasource": "${datasource}",
+               "expr": "sum(ceph_rgw_op_del_bucket_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "time_series",
+               "instant": true,
+               "intervalFactor": 1,
+               "legendFormat": "Delete Buckets",
+               "range": false,
+               "refId": "F"
+            },
+            {
+               "datasource": "${datasource}",
+               "expr": "sum(ceph_rgw_op_copy_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "time_series",
+               "intervalFactor": 1,
+               "legendFormat": "Copy Objects",
+               "range": true,
+               "refId": "G"
+            }
+         ],
+         "title": "Total Operations",
+         "type": "bargauge"
+      },
+      {
+         "datasource": "${datasource}",
+         "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "color": {
+                  "mode": "thresholds"
+               },
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green",
+                        "value": null
+                     }
+                  ]
+               }
+            },
+            "overrides": [
+               {
+                  "matcher": {
+                     "id": "byType",
+                     "unit": "number"
+                  },
+                  "properties": [
+                     {
+                        "id": "unit",
+                        "value": "decbytes"
+                     }
+                  ]
+               }
+            ]
+         },
+         "gridPos": {
+            "h": 8,
+            "w": 8,
+            "x": 8,
+            "y": 4
+         },
+         "id": 8,
+         "options": {
+            "displayMode": "gradient",
+            "orientation": "horizontal",
+            "reduceOptions": {
+               "calcs": [
+                  "lastNotNull"
+               ]
+            }
+         },
+         "targets": [
+            {
+               "datasource": "${datasource}",
+               "expr": "sum(ceph_rgw_op_put_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "time_series",
+               "instant": true,
+               "intervalFactor": 1,
+               "legendFormat": "Put Objects",
+               "range": false,
+               "refId": "A"
+            },
+            {
+               "datasource": "${datasource}",
+               "expr": "sum(ceph_rgw_op_per_bucket_get_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "time_series",
+               "instant": true,
+               "intervalFactor": 1,
+               "legendFormat": "Get Objects",
+               "range": false,
+               "refId": "B"
+            },
+            {
+               "datasource": "${datasource}",
+               "expr": "sum(ceph_rgw_op_del_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "time_series",
+               "instant": true,
+               "intervalFactor": 1,
+               "legendFormat": "Delete Objects",
+               "range": false,
+               "refId": "C"
+            },
+            {
+               "datasource": "${datasource}",
+               "expr": "sum(ceph_rgw_op_copy_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "time_series",
+               "intervalFactor": 1,
+               "legendFormat": "Copy Objects",
+               "range": true,
+               "refId": "D"
+            }
+         ],
+         "title": "Total Size",
+         "type": "bargauge"
+      },
+      {
+         "datasource": "${datasource}",
+         "description": "",
+         "fieldConfig": {
+            "defaults": {
+               "color": {
+                  "mode": "thresholds"
+               },
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green",
+                        "value": null
+                     }
+                  ]
+               }
+            },
+            "overrides": [
+               {
+                  "matcher": {
+                     "id": "byType",
+                     "unit": "number"
+                  },
+                  "properties": [
+                     {
+                        "id": "unit",
+                        "value": "ms"
+                     }
+                  ]
+               }
+            ]
+         },
+         "gridPos": {
+            "h": 8,
+            "w": 8,
+            "x": 16,
+            "y": 4
+         },
+         "id": 9,
+         "options": {
+            "displayMode": "gradient",
+            "orientation": "horizontal",
+            "reduceOptions": {
+               "calcs": [
+                  "lastNotNull"
+               ]
+            }
+         },
+         "targets": [
+            {
+               "datasource": "${datasource}",
+               "expr": "sum(ceph_rgw_op_list_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "time_series",
+               "instant": true,
+               "intervalFactor": 1,
+               "legendFormat": "List Object",
+               "range": false,
+               "refId": "A"
+            },
+            {
+               "datasource": "${datasource}",
+               "expr": "sum(ceph_rgw_op_list_buckets_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "time_series",
+               "intervalFactor": 1,
+               "legendFormat": "List Bucket",
+               "range": true,
+               "refId": "B"
+            },
+            {
+               "datasource": "${datasource}",
+               "expr": "sum(ceph_rgw_op_put_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "time_series",
+               "instant": true,
+               "intervalFactor": 1,
+               "legendFormat": "Put Object",
+               "range": false,
+               "refId": "C"
+            },
+            {
+               "datasource": "${datasource}",
+               "expr": "sum(ceph_rgw_op_get_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "time_series",
+               "instant": true,
+               "intervalFactor": 1,
+               "legendFormat": "Get Object",
+               "range": false,
+               "refId": "D"
+            },
+            {
+               "datasource": "${datasource}",
+               "expr": "sum(ceph_rgw_op_del_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "time_series",
+               "instant": true,
+               "intervalFactor": 1,
+               "legendFormat": "Delete Object",
+               "range": false,
+               "refId": "E"
+            },
+            {
+               "datasource": "${datasource}",
+               "expr": "sum(ceph_rgw_op_del_bucket_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "time_series",
+               "instant": true,
+               "intervalFactor": 1,
+               "legendFormat": "Delete Bucket",
+               "range": false,
+               "refId": "F"
+            },
+            {
+               "datasource": "${datasource}",
+               "expr": "sum(ceph_rgw_op_copy_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "time_series",
+               "intervalFactor": 1,
+               "legendFormat": "Copy Object",
+               "range": true,
+               "refId": "G"
+            }
+         ],
+         "title": "Total Latencies",
+         "type": "bargauge"
+      },
+      {
+         "columns": [ ],
+         "datasource": "${datasource}",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "align": "auto",
+                  "cellOptions": {
+                     "type": "color-text"
+                  },
+                  "filterable": false,
+                  "inspect": false
+               },
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green",
+                        "value": null
+                     }
+                  ]
+               }
+            },
+            "overrides": [
+               {
+                  "matcher": {
+                     "id": "byType",
+                     "options": "number"
+                  },
+                  "properties": [
+                     {
+                        "id": "unit",
+                        "value": "decbytes"
+                     }
+                  ]
+               }
+            ]
+         },
+         "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 12
+         },
+         "id": 10,
+         "links": [ ],
+         "options": {
+            "footer": {
+               "countRows": false,
+               "enablePagination": false,
+               "fields": "",
+               "reducer": [
+                  "sum"
+               ],
+               "show": false
+            },
+            "frameIndex": 1,
+            "showHeader": true,
+            "sortBy": [
+               {
+                  "desc": true,
+                  "displayName": "PUTs"
+               }
+            ]
+         },
+         "pluginVersion": "9.4.7",
+         "styles": "",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "exemplar": false,
+               "expr": "sum by (bucket, ceph_daemon) (ceph_rgw_op_per_bucket_put_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "Upload Objects",
+               "range": false,
+               "refId": "A"
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "exemplar": false,
+               "expr": "sum by (bucket, ceph_daemon) (ceph_rgw_op_per_bucket_get_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "Get Objects",
+               "range": false,
+               "refId": "B"
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "exemplar": false,
+               "expr": "sum by (bucket, ceph_daemon) (ceph_rgw_op_per_bucket_del_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "Delete Objects",
+               "range": false,
+               "refId": "C"
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "exemplar": false,
+               "expr": "sum by (bucket, ceph_daemon) (ceph_rgw_op_per_bucket_copy_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "Copy Objects",
+               "range": false,
+               "refId": "D"
+            }
+         ],
+         "timeFrom": null,
+         "timeShift": null,
+         "title": "Summary Per Bucket by Bandwidth",
+         "transformations": [
+            {
+               "id": "merge",
+               "options": { }
+            },
+            {
+               "id": "groupBy",
+               "options": {
+                  "fields": {
+                     "Bucket": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "Value #A": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "Value #B": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "Value #D": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "Value #F": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "bucket": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "ceph_daemon": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     }
+                  }
+               }
+            },
+            {
+               "id": "organize",
+               "options": {
+                  "excludeByName": {
+                     "Time 1": true,
+                     "Time 2": true,
+                     "Time 3": true,
+                     "Time 4": true,
+                     "Time 5": true,
+                     "Time 6": true,
+                     "Time 7": true,
+                     "__name__ 1": true,
+                     "__name__ 2": true,
+                     "__name__ 3": true,
+                     "__name__ 4": true,
+                     "__name__ 5": true,
+                     "__name__ 6": true,
+                     "__name__ 7": true,
+                     "ceph_daemon 1": false,
+                     "ceph_daemon 2": true,
+                     "ceph_daemon 3": true,
+                     "ceph_daemon 4": true,
+                     "instance 1": true,
+                     "instance 2": true,
+                     "instance 3": true,
+                     "instance 4": true,
+                     "instance 5": true,
+                     "instance 6": true,
+                     "instance 7": true,
+                     "instance_id 1": true,
+                     "instance_id 2": true,
+                     "instance_id 3": true,
+                     "instance_id 4": true,
+                     "instance_id 5": true,
+                     "instance_id 6": true,
+                     "instance_id 7": true,
+                     "job 1": true,
+                     "job 2": true,
+                     "job 3": true,
+                     "job 4": true,
+                     "job 5": true,
+                     "job 6": true,
+                     "job 7": true
+                  },
+                  "indexByName": {
+                     "Value #A": 2,
+                     "Value #B": 3,
+                     "Value #D": 4,
+                     "Value #F": 5,
+                     "bucket": 1,
+                     "ceph_daemon": 0
+                  },
+                  "renameByName": {
+                     "Bucket": "",
+                     "Value #A": "PUTs",
+                     "Value #B": "GETs",
+                     "Value #C": "List",
+                     "Value #D": "Delete",
+                     "Value #E": "Copy",
+                     "Value #F": "Copy",
+                     "Value #G": "",
+                     "bucket": "Bucket",
+                     "ceph_daemon": "Daemon",
+                     "ceph_daemon 1": "Daemon"
+                  }
+               }
+            }
+         ],
+         "type": "table"
+      },
+      {
+         "columns": [ ],
+         "datasource": "${datasource}",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "align": "auto",
+                  "cellOptions": {
+                     "type": "auto"
+                  },
+                  "filterable": false,
+                  "inspect": false
+               },
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green",
+                        "value": null
+                     }
+                  ]
+               }
+            },
+            "overrides": [
+               {
+                  "matcher": {
+                     "id": "byType",
+                     "options": "number"
+                  },
+                  "properties": [
+                     {
+                        "id": "unit",
+                        "value": "ms"
+                     }
+                  ]
+               }
+            ]
+         },
+         "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 12
+         },
+         "id": 11,
+         "links": [ ],
+         "options": {
+            "footer": {
+               "countRows": false,
+               "enablePagination": false,
+               "fields": "",
+               "reducer": [
+                  "sum"
+               ],
+               "show": false
+            },
+            "frameIndex": 1,
+            "showHeader": true,
+            "sortBy": [
+               {
+                  "desc": true,
+                  "displayName": "PUTs"
+               }
+            ]
+         },
+         "pluginVersion": "9.4.7",
+         "styles": "",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "exemplar": false,
+               "expr": "ceph_rgw_op_per_bucket_list_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"}",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "List Objects",
+               "range": false,
+               "refId": "A"
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "exemplar": false,
+               "expr": "ceph_rgw_op_per_bucket_put_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"}",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "Upload Objects",
+               "range": false,
+               "refId": "B"
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "exemplar": false,
+               "expr": "ceph_rgw_op_per_bucket_get_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"}",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "Get Objects",
+               "range": false,
+               "refId": "C"
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "exemplar": false,
+               "expr": "ceph_rgw_op_per_bucket_del_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"}",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "Delete Objects",
+               "range": false,
+               "refId": "D"
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "exemplar": false,
+               "expr": "ceph_rgw_op_per_bucket_copy_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"}",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "Copy Objects",
+               "range": false,
+               "refId": "E"
+            }
+         ],
+         "timeFrom": null,
+         "timeShift": null,
+         "title": "Latency(ms) Per Bucket",
+         "transformations": [
+            {
+               "id": "merge",
+               "options": { }
+            },
+            {
+               "id": "joinByField",
+               "options": {
+                  "byField": "Bucket",
+                  "mode": "outer"
+               }
+            },
+            {
+               "id": "groupBy",
+               "options": {
+                  "fields": {
+                     "Bucket": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "Value #A": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "Value #B": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "Value #C": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "Value #D": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "Value #F": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "bucket": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "ceph_daemon": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     }
+                  }
+               }
+            },
+            {
+               "id": "organize",
+               "options": {
+                  "excludeByName": {
+                     "Time 1": true,
+                     "Time 2": true,
+                     "Time 3": true,
+                     "Time 4": true,
+                     "Time 5": true,
+                     "Time 6": true,
+                     "Time 7": true,
+                     "__name__ 1": true,
+                     "__name__ 2": true,
+                     "__name__ 3": true,
+                     "__name__ 4": true,
+                     "__name__ 5": true,
+                     "__name__ 6": true,
+                     "__name__ 7": true,
+                     "ceph_daemon 1": true,
+                     "ceph_daemon 2": true,
+                     "ceph_daemon 3": true,
+                     "ceph_daemon 4": true,
+                     "ceph_daemon 5": true,
+                     "instance 1": true,
+                     "instance 2": true,
+                     "instance 3": true,
+                     "instance 4": true,
+                     "instance 5": true,
+                     "instance 6": true,
+                     "instance 7": true,
+                     "instance_id 1": true,
+                     "instance_id 2": true,
+                     "instance_id 3": true,
+                     "instance_id 4": true,
+                     "instance_id 5": true,
+                     "instance_id 6": true,
+                     "instance_id 7": true,
+                     "job 1": true,
+                     "job 2": true,
+                     "job 3": true,
+                     "job 4": true,
+                     "job 5": true,
+                     "job 6": true,
+                     "job 7": true
+                  },
+                  "indexByName": {
+                     "Value #A": 2,
+                     "Value #B": 3,
+                     "Value #C": 4,
+                     "Value #D": 5,
+                     "Value #F": 6,
+                     "bucket": 1,
+                     "ceph_daemon": 0
+                  },
+                  "renameByName": {
+                     "Bucket": "",
+                     "Value #A": "PUTs",
+                     "Value #B": "GETs",
+                     "Value #C": "List",
+                     "Value #D": "Delete",
+                     "Value #E": "Copy",
+                     "Value #F": "Copy",
+                     "Value #G": "",
+                     "bucket": "Bucket",
+                     "ceph_daemon": "Daemon"
+                  }
+               }
+            }
+         ],
+         "type": "table"
+      },
+      {
+         "columns": [ ],
+         "datasource": "${datasource}",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "align": "auto",
+                  "cellOptions": {
+                     "type": "auto"
+                  },
+                  "filterable": false,
+                  "inspect": false
+               },
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green",
+                        "value": null
+                     }
+                  ]
+               }
+            },
+            "overrides": [
+               {
+                  "matcher": {
+                     "id": "byType",
+                     "options": "number"
+                  },
+                  "properties": [
+                     {
+                        "id": "unit",
+                        "value": "decbytes"
+                     }
+                  ]
+               }
+            ]
+         },
+         "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 20
+         },
+         "id": 12,
+         "links": [ ],
+         "options": {
+            "footer": {
+               "countRows": false,
+               "enablePagination": false,
+               "fields": "",
+               "reducer": [
+                  "sum"
+               ],
+               "show": false
+            },
+            "frameIndex": 1,
+            "showHeader": true,
+            "sortBy": [
+               {
+                  "desc": true,
+                  "displayName": "PUTs"
+               }
+            ]
+         },
+         "pluginVersion": "9.4.7",
+         "styles": "",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "exemplar": false,
+               "expr": "ceph_rgw_op_per_user_put_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"}",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "Upload Objects",
+               "range": false,
+               "refId": "A"
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "exemplar": false,
+               "expr": "ceph_rgw_op_per_user_get_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"}",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "Get Objects",
+               "range": false,
+               "refId": "B"
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "exemplar": false,
+               "expr": "ceph_rgw_op_per_user_del_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"}",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "Delete Objects",
+               "range": false,
+               "refId": "C"
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "exemplar": false,
+               "expr": "ceph_rgw_op_per_user_copy_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"}",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "Copy Objects",
+               "range": false,
+               "refId": "D"
+            }
+         ],
+         "timeFrom": null,
+         "timeShift": null,
+         "title": "Summary Per User By Bandwidth",
+         "transformations": [
+            {
+               "id": "merge",
+               "options": { }
+            },
+            {
+               "id": "groupBy",
+               "options": {
+                  "fields": {
+                     "User": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "Value #A": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "Value #B": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "Value #D": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "Value #F": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "ceph_daemon": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "instance": {
+                        "aggregations": [ ]
+                     },
+                     "user": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     }
+                  }
+               }
+            },
+            {
+               "id": "organize",
+               "options": {
+                  "excludeByName": {
+                     "Time 1": true,
+                     "Time 2": true,
+                     "Time 3": true,
+                     "Time 4": true,
+                     "Time 5": true,
+                     "Time 6": true,
+                     "Time 7": true,
+                     "__name__ 1": true,
+                     "__name__ 2": true,
+                     "__name__ 3": true,
+                     "__name__ 4": true,
+                     "__name__ 5": true,
+                     "__name__ 6": true,
+                     "__name__ 7": true,
+                     "ceph_daemon 1": true,
+                     "ceph_daemon 2": true,
+                     "ceph_daemon 3": true,
+                     "ceph_daemon 4": true,
+                     "instance 1": true,
+                     "instance 2": true,
+                     "instance 3": true,
+                     "instance 4": true,
+                     "instance 5": true,
+                     "instance 6": true,
+                     "instance 7": true,
+                     "instance_id 1": true,
+                     "instance_id 2": true,
+                     "instance_id 3": true,
+                     "instance_id 4": true,
+                     "instance_id 5": true,
+                     "instance_id 6": true,
+                     "instance_id 7": true,
+                     "job 1": true,
+                     "job 2": true,
+                     "job 3": true,
+                     "job 4": true,
+                     "job 5": true,
+                     "job 6": true,
+                     "job 7": true
+                  },
+                  "indexByName": {
+                     "Value #A": 2,
+                     "Value #B": 3,
+                     "Value #D": 4,
+                     "Value #F": 5,
+                     "ceph_daemon": 0,
+                     "user": 1
+                  },
+                  "renameByName": {
+                     "Bucket": "",
+                     "Value #A": "PUTs",
+                     "Value #B": "GETs",
+                     "Value #C": "List",
+                     "Value #D": "Delete",
+                     "Value #E": "Copy",
+                     "Value #F": "Copy",
+                     "Value #G": "",
+                     "ceph_daemon": "Daemon",
+                     "user": "User"
+                  }
+               }
+            }
+         ],
+         "type": "table"
+      },
+      {
+         "columns": [ ],
+         "datasource": "${datasource}",
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "align": "auto",
+                  "cellOptions": {
+                     "type": "auto"
+                  },
+                  "filterable": false,
+                  "inspect": false
+               },
+               "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                     {
+                        "color": "green",
+                        "value": null
+                     }
+                  ]
+               }
+            },
+            "overrides": [
+               {
+                  "matcher": {
+                     "id": "byType",
+                     "options": "number"
+                  },
+                  "properties": [
+                     {
+                        "id": "unit",
+                        "value": "ms"
+                     }
+                  ]
+               }
+            ]
+         },
+         "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 20
+         },
+         "id": 13,
+         "links": [ ],
+         "options": {
+            "footer": {
+               "countRows": false,
+               "enablePagination": false,
+               "fields": "",
+               "reducer": [
+                  "sum"
+               ],
+               "show": false
+            },
+            "frameIndex": 1,
+            "showHeader": true,
+            "sortBy": [
+               {
+                  "desc": true,
+                  "displayName": "PUTs"
+               }
+            ]
+         },
+         "pluginVersion": "9.4.7",
+         "styles": "",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "exemplar": false,
+               "expr": "ceph_rgw_op_per_user_list_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"}",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "__auto",
+               "range": false,
+               "refId": "A"
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "exemplar": false,
+               "expr": "ceph_rgw_op_per_user_put_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"}",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "__auto",
+               "range": false,
+               "refId": "B"
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "exemplar": false,
+               "expr": "ceph_rgw_op_per_user_get_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"}",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "__auto",
+               "range": false,
+               "refId": "C"
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "exemplar": false,
+               "expr": "ceph_rgw_op_per_user_del_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"}",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "__auto",
+               "range": false,
+               "refId": "D"
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "exemplar": false,
+               "expr": "ceph_rgw_op_per_user_copy_obj_lat_sum *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"}",
+               "format": "table",
+               "hide": false,
+               "instant": true,
+               "interval": "",
+               "intervalFactor": 1,
+               "legendFormat": "__auto",
+               "range": false,
+               "refId": "E"
+            }
+         ],
+         "timeFrom": null,
+         "timeShift": null,
+         "title": "Latency(ms) Per User",
+         "transformations": [
+            {
+               "id": "merge",
+               "options": { }
+            },
+            {
+               "id": "joinByField",
+               "options": {
+                  "byField": "User",
+                  "mode": "outer"
+               }
+            },
+            {
+               "id": "groupBy",
+               "options": {
+                  "fields": {
+                     "User": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "Value #A": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "Value #B": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "Value #C": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "Value #D": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "Value #F": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "ceph_daemon": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     },
+                     "user": {
+                        "aggregations": [ ],
+                        "operation": "groupby"
+                     }
+                  }
+               }
+            },
+            {
+               "id": "organize",
+               "options": {
+                  "excludeByName": {
+                     "Time 1": true,
+                     "Time 2": true,
+                     "Time 3": true,
+                     "Time 4": true,
+                     "Time 5": true,
+                     "Time 6": true,
+                     "Time 7": true,
+                     "__name__ 1": true,
+                     "__name__ 2": true,
+                     "__name__ 3": true,
+                     "__name__ 4": true,
+                     "__name__ 5": true,
+                     "__name__ 6": true,
+                     "__name__ 7": true,
+                     "ceph_daemon 1": true,
+                     "ceph_daemon 2": true,
+                     "ceph_daemon 3": true,
+                     "ceph_daemon 4": true,
+                     "ceph_daemon 5": true,
+                     "instance 1": true,
+                     "instance 2": true,
+                     "instance 3": true,
+                     "instance 4": true,
+                     "instance 5": true,
+                     "instance 6": true,
+                     "instance 7": true,
+                     "instance_id 1": true,
+                     "instance_id 2": true,
+                     "instance_id 3": true,
+                     "instance_id 4": true,
+                     "instance_id 5": true,
+                     "instance_id 6": true,
+                     "instance_id 7": true,
+                     "job 1": true,
+                     "job 2": true,
+                     "job 3": true,
+                     "job 4": true,
+                     "job 5": true,
+                     "job 6": true,
+                     "job 7": true
+                  },
+                  "indexByName": {
+                     "Value #A": 2,
+                     "Value #B": 3,
+                     "Value #C": 4,
+                     "Value #D": 5,
+                     "Value #F": 6,
+                     "ceph_daemon": 0,
+                     "user": 1
+                  },
+                  "renameByName": {
+                     "Bucket": "",
+                     "Value #A": "PUTs",
+                     "Value #B": "GETs",
+                     "Value #C": "List",
+                     "Value #D": "Delete",
+                     "Value #E": "Copy",
+                     "Value #F": "Copy",
+                     "Value #G": "",
+                     "ceph_daemon": "Daemon",
+                     "user": "User"
+                  }
+               }
+            }
+         ],
+         "type": "table"
+      },
+      {
+         "collapse": false,
+         "collapsed": true,
+         "gridPos": {
+            "h": 1,
+            "w": 24,
+            "x": 0,
+            "y": 28
+         },
+         "id": 14,
+         "panels": [
+            {
+               "datasource": "${datasource}",
+               "description": "",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "thresholds"
+                     },
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     }
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byType",
+                           "unit": "number"
+                        },
+                        "properties": [
+                           {
+                              "id": "color"
+                           },
+                           {
+                              "id": "color",
+                              "value": {
+                                 "mode": "palette-classic"
+                              }
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 6,
+                  "x": 0,
+                  "y": 29
+               },
+               "id": 15,
+               "options": {
+                  "displayMode": "gradient",
+                  "orientation": "horizontal",
+                  "reduceOptions": {
+                     "calcs": [
+                        "lastNotNull"
+                     ]
+                  }
+               },
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "topk(5, \n    sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_put_obj_ops) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})\n)",
+                     "format": "time_series",
+                     "instant": true,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{bucket}}",
+                     "range": false,
+                     "refId": "A"
+                  }
+               ],
+               "title": "Top 5 Bucket PUTs by Operations",
+               "type": "bargauge"
+            },
+            {
+               "datasource": "${datasource}",
+               "description": "",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "thresholds"
+                     },
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     }
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byType",
+                           "unit": "number"
+                        },
+                        "properties": [
+                           {
+                              "id": "color"
+                           },
+                           {
+                              "id": "color",
+                              "value": {
+                                 "mode": "palette-classic"
+                              }
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 6,
+                  "x": 6,
+                  "y": 29
+               },
+               "id": 16,
+               "options": {
+                  "displayMode": "gradient",
+                  "orientation": "horizontal",
+                  "reduceOptions": {
+                     "calcs": [
+                        "lastNotNull"
+                     ]
+                  }
+               },
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "topk(5, \n    sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_get_obj_ops) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})\n)",
+                     "format": "time_series",
+                     "instant": true,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{bucket}}",
+                     "range": false,
+                     "refId": "A"
+                  }
+               ],
+               "title": "Top 5 Bucket GETs by Operations",
+               "type": "bargauge"
+            },
+            {
+               "datasource": "${datasource}",
+               "description": "",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "thresholds"
+                     },
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     }
+                  }
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 6,
+                  "x": 12,
+                  "y": 29
+               },
+               "id": 17,
+               "options": {
+                  "displayMode": "gradient",
+                  "orientation": "horizontal",
+                  "reduceOptions": {
+                     "calcs": [ ]
+                  }
+               },
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "topk(5,\n    sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_put_obj_bytes) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})\n)",
+                     "format": "time_series",
+                     "instant": true,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{bucket}}",
+                     "range": false,
+                     "refId": "A"
+                  }
+               ],
+               "title": "Top 5 Buckets PUTs By Size",
+               "type": "bargauge"
+            },
+            {
+               "datasource": "${datasource}",
+               "description": "",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "thresholds"
+                     },
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     }
+                  }
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 6,
+                  "x": 18,
+                  "y": 29
+               },
+               "id": 18,
+               "options": {
+                  "displayMode": "gradient",
+                  "orientation": "horizontal",
+                  "reduceOptions": {
+                     "calcs": [ ]
+                  }
+               },
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "topk(5,\n    sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_get_obj_bytes) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})\n)",
+                     "format": "time_series",
+                     "instant": true,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{bucket}}",
+                     "range": false,
+                     "refId": "A"
+                  }
+               ],
+               "title": "Top 5 Buckets GETs By Size",
+               "type": "bargauge"
+            },
+            {
+               "datasource": "${datasource}",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "palette-classic"
+                     },
+                     "custom": {
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "",
+                        "axisPlacement": "auto",
+                        "barAlignment": 0,
+                        "drawStyle": "line",
+                        "fillOpacity": 0,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                           "legend": false,
+                           "tooltip": false,
+                           "viz": false
+                        },
+                        "lineInterpolation": "linear",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                           "type": "linear"
+                        },
+                        "showPoints": "auto",
+                        "spanNulls": true,
+                        "stacking": {
+                           "group": "A",
+                           "mode": "none"
+                        },
+                        "thresholdsStyle": {
+                           "mode": "off"
+                        }
+                     },
+                     "decimals": 2,
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     },
+                     "unit": "decbytes"
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byType",
+                           "unit": "number"
+                        },
+                        "properties": [
+                           {
+                              "id": "color"
+                           },
+                           {
+                              "id": "color",
+                              "value": {
+                                 "mode": "palette-classic"
+                              }
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 6,
+                  "x": 0,
+                  "y": 37
+               },
+               "id": 19,
+               "options": {
+                  "legend": {
+                     "calcs": [
+                        "lastNotNull"
+                     ],
+                     "displayMode": "table",
+                     "placement": "bottom",
+                     "showLegend": true,
+                     "sortBy": "Last *",
+                     "sortDesc": true
+                  },
+                  "tooltip": {
+                     "mode": "single",
+                     "sort": "desc"
+                  }
+               },
+               "pluginVersion": "9.1.3",
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_put_obj_bytes) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "time_series",
+                     "instant": false,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{bucket}}",
+                     "range": true,
+                     "refId": "A",
+                     "step": 300
+                  }
+               ],
+               "title": "Bucket PUTs by Size",
+               "type": "timeseries"
+            },
+            {
+               "datasource": "${datasource}",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "palette-classic"
+                     },
+                     "custom": {
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "",
+                        "axisPlacement": "auto",
+                        "barAlignment": 0,
+                        "drawStyle": "line",
+                        "fillOpacity": 0,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                           "legend": false,
+                           "tooltip": false,
+                           "viz": false
+                        },
+                        "lineInterpolation": "linear",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                           "type": "linear"
+                        },
+                        "showPoints": "auto",
+                        "spanNulls": true,
+                        "stacking": {
+                           "group": "A",
+                           "mode": "none"
+                        },
+                        "thresholdsStyle": {
+                           "mode": "off"
+                        }
+                     },
+                     "decimals": 2,
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     },
+                     "unit": "decbytes"
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byType",
+                           "unit": "number"
+                        },
+                        "properties": [
+                           {
+                              "id": "color"
+                           },
+                           {
+                              "id": "color",
+                              "value": {
+                                 "mode": "palette-classic"
+                              }
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 6,
+                  "x": 6,
+                  "y": 37
+               },
+               "id": 20,
+               "options": {
+                  "legend": {
+                     "calcs": [
+                        "lastNotNull"
+                     ],
+                     "displayMode": "table",
+                     "placement": "bottom",
+                     "showLegend": true,
+                     "sortBy": "Last *",
+                     "sortDesc": true
+                  },
+                  "tooltip": {
+                     "mode": "single",
+                     "sort": "desc"
+                  }
+               },
+               "pluginVersion": "9.1.3",
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_get_obj_bytes) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "time_series",
+                     "instant": false,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{bucket}}",
+                     "range": true,
+                     "refId": "A",
+                     "step": 300
+                  }
+               ],
+               "title": "Bucket GETs by Size",
+               "type": "timeseries"
+            },
+            {
+               "datasource": "${datasource}",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "palette-classic"
+                     },
+                     "custom": {
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "",
+                        "axisPlacement": "auto",
+                        "barAlignment": 0,
+                        "drawStyle": "line",
+                        "fillOpacity": 0,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                           "legend": false,
+                           "tooltip": false,
+                           "viz": false
+                        },
+                        "lineInterpolation": "linear",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                           "type": "linear"
+                        },
+                        "showPoints": "auto",
+                        "spanNulls": true,
+                        "stacking": {
+                           "group": "A",
+                           "mode": "none"
+                        },
+                        "thresholdsStyle": {
+                           "mode": "off"
+                        }
+                     },
+                     "decimals": 2,
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     },
+                     "unit": "decbytes"
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byType",
+                           "unit": "number"
+                        },
+                        "properties": [
+                           {
+                              "id": "color"
+                           },
+                           {
+                              "id": "color",
+                              "value": {
+                                 "mode": "palette-classic"
+                              }
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 6,
+                  "x": 12,
+                  "y": 37
+               },
+               "id": 21,
+               "options": {
+                  "legend": {
+                     "calcs": [
+                        "lastNotNull"
+                     ],
+                     "displayMode": "table",
+                     "placement": "bottom",
+                     "showLegend": true,
+                     "sortBy": "Last *",
+                     "sortDesc": true
+                  },
+                  "tooltip": {
+                     "mode": "single",
+                     "sort": "desc"
+                  }
+               },
+               "pluginVersion": "9.1.3",
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_copy_obj_bytes) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "time_series",
+                     "instant": false,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{bucket}}",
+                     "range": true,
+                     "refId": "A",
+                     "step": 300
+                  }
+               ],
+               "title": "Bucket Copy by Size",
+               "type": "timeseries"
+            },
+            {
+               "datasource": "${datasource}",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "palette-classic"
+                     },
+                     "custom": {
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "",
+                        "axisPlacement": "auto",
+                        "barAlignment": 0,
+                        "drawStyle": "line",
+                        "fillOpacity": 0,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                           "legend": false,
+                           "tooltip": false,
+                           "viz": false
+                        },
+                        "lineInterpolation": "linear",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                           "type": "linear"
+                        },
+                        "showPoints": "auto",
+                        "spanNulls": true,
+                        "stacking": {
+                           "group": "A",
+                           "mode": "none"
+                        },
+                        "thresholdsStyle": {
+                           "mode": "off"
+                        }
+                     },
+                     "decimals": 2,
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     },
+                     "unit": "decbytes"
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byType",
+                           "unit": "number"
+                        },
+                        "properties": [
+                           {
+                              "id": "color"
+                           },
+                           {
+                              "id": "color",
+                              "value": {
+                                 "mode": "palette-classic"
+                              }
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 6,
+                  "x": 18,
+                  "y": 37
+               },
+               "id": 22,
+               "options": {
+                  "legend": {
+                     "calcs": [
+                        "lastNotNull"
+                     ],
+                     "displayMode": "table",
+                     "placement": "bottom",
+                     "showLegend": true,
+                     "sortBy": "Last *",
+                     "sortDesc": true
+                  },
+                  "tooltip": {
+                     "mode": "single",
+                     "sort": "desc"
+                  }
+               },
+               "pluginVersion": "9.1.3",
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_del_obj_bytes) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "time_series",
+                     "instant": false,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{bucket}}",
+                     "range": true,
+                     "refId": "A",
+                     "step": 300
+                  }
+               ],
+               "title": "Bucket Delete by Size",
+               "type": "timeseries"
+            },
+            {
+               "datasource": "${datasource}",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "palette-classic"
+                     },
+                     "custom": {
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "",
+                        "axisPlacement": "auto",
+                        "barAlignment": 0,
+                        "drawStyle": "line",
+                        "fillOpacity": 0,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                           "legend": false,
+                           "tooltip": false,
+                           "viz": false
+                        },
+                        "lineInterpolation": "linear",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                           "type": "linear"
+                        },
+                        "showPoints": "auto",
+                        "spanNulls": true,
+                        "stacking": {
+                           "group": "A",
+                           "mode": "none"
+                        },
+                        "thresholdsStyle": {
+                           "mode": "off"
+                        }
+                     },
+                     "decimals": 2,
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     },
+                     "unit": "none"
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byType",
+                           "unit": "number"
+                        },
+                        "properties": [
+                           {
+                              "id": "color"
+                           },
+                           {
+                              "id": "color",
+                              "value": {
+                                 "mode": "palette-classic"
+                              }
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 6,
+                  "x": 0,
+                  "y": 45
+               },
+               "id": 23,
+               "options": {
+                  "legend": {
+                     "calcs": [
+                        "lastNotNull"
+                     ],
+                     "displayMode": "table",
+                     "placement": "bottom",
+                     "showLegend": true,
+                     "sortBy": "Last *",
+                     "sortDesc": true
+                  },
+                  "tooltip": {
+                     "mode": "single",
+                     "sort": "desc"
+                  }
+               },
+               "pluginVersion": "9.1.3",
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_get_obj_ops) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "time_series",
+                     "instant": false,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{bucket}}",
+                     "range": true,
+                     "refId": "A",
+                     "step": 300
+                  }
+               ],
+               "title": "Bucket GETs by Operations",
+               "type": "timeseries"
+            },
+            {
+               "datasource": "${datasource}",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "palette-classic"
+                     },
+                     "custom": {
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "",
+                        "axisPlacement": "auto",
+                        "barAlignment": 0,
+                        "drawStyle": "line",
+                        "fillOpacity": 0,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                           "legend": false,
+                           "tooltip": false,
+                           "viz": false
+                        },
+                        "lineInterpolation": "linear",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                           "type": "linear"
+                        },
+                        "showPoints": "auto",
+                        "spanNulls": true,
+                        "stacking": {
+                           "group": "A",
+                           "mode": "none"
+                        },
+                        "thresholdsStyle": {
+                           "mode": "off"
+                        }
+                     },
+                     "decimals": 2,
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     },
+                     "unit": "none"
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byType",
+                           "unit": "number"
+                        },
+                        "properties": [
+                           {
+                              "id": "color"
+                           },
+                           {
+                              "id": "color",
+                              "value": {
+                                 "mode": "palette-classic"
+                              }
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 6,
+                  "x": 6,
+                  "y": 45
+               },
+               "id": 24,
+               "options": {
+                  "legend": {
+                     "calcs": [
+                        "lastNotNull"
+                     ],
+                     "displayMode": "table",
+                     "placement": "bottom",
+                     "showLegend": true,
+                     "sortBy": "Last *",
+                     "sortDesc": true
+                  },
+                  "tooltip": {
+                     "mode": "single",
+                     "sort": "desc"
+                  }
+               },
+               "pluginVersion": "9.1.3",
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_put_obj_ops) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "time_series",
+                     "instant": false,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{bucket}}",
+                     "range": true,
+                     "refId": "A",
+                     "step": 300
+                  }
+               ],
+               "title": "Bucket PUTs by Operations",
+               "type": "timeseries"
+            },
+            {
+               "datasource": "${datasource}",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "palette-classic"
+                     },
+                     "custom": {
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "",
+                        "axisPlacement": "auto",
+                        "barAlignment": 0,
+                        "drawStyle": "line",
+                        "fillOpacity": 0,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                           "legend": false,
+                           "tooltip": false,
+                           "viz": false
+                        },
+                        "lineInterpolation": "linear",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                           "type": "linear"
+                        },
+                        "showPoints": "auto",
+                        "spanNulls": true,
+                        "stacking": {
+                           "group": "A",
+                           "mode": "none"
+                        },
+                        "thresholdsStyle": {
+                           "mode": "off"
+                        }
+                     },
+                     "decimals": 2,
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     },
+                     "unit": "none"
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byType",
+                           "unit": "number"
+                        },
+                        "properties": [
+                           {
+                              "id": "color"
+                           },
+                           {
+                              "id": "color",
+                              "value": {
+                                 "mode": "palette-classic"
+                              }
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 6,
+                  "x": 12,
+                  "y": 45
+               },
+               "id": 25,
+               "options": {
+                  "legend": {
+                     "calcs": [
+                        "lastNotNull"
+                     ],
+                     "displayMode": "table",
+                     "placement": "bottom",
+                     "showLegend": true,
+                     "sortBy": "Last *",
+                     "sortDesc": true
+                  },
+                  "tooltip": {
+                     "mode": "single",
+                     "sort": "desc"
+                  }
+               },
+               "pluginVersion": "9.1.3",
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_list_obj_ops) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "time_series",
+                     "instant": false,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{bucket}}",
+                     "range": true,
+                     "refId": "A",
+                     "step": 300
+                  }
+               ],
+               "title": "Bucket List by Operations",
+               "type": "timeseries"
+            },
+            {
+               "datasource": "${datasource}",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "palette-classic"
+                     },
+                     "custom": {
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "",
+                        "axisPlacement": "auto",
+                        "barAlignment": 0,
+                        "drawStyle": "line",
+                        "fillOpacity": 0,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                           "legend": false,
+                           "tooltip": false,
+                           "viz": false
+                        },
+                        "lineInterpolation": "linear",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                           "type": "linear"
+                        },
+                        "showPoints": "auto",
+                        "spanNulls": true,
+                        "stacking": {
+                           "group": "A",
+                           "mode": "none"
+                        },
+                        "thresholdsStyle": {
+                           "mode": "off"
+                        }
+                     },
+                     "decimals": 2,
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     },
+                     "unit": "none"
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byType",
+                           "unit": "number"
+                        },
+                        "properties": [
+                           {
+                              "id": "color"
+                           },
+                           {
+                              "id": "color",
+                              "value": {
+                                 "mode": "palette-classic"
+                              }
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 6,
+                  "x": 18,
+                  "y": 45
+               },
+               "id": 26,
+               "options": {
+                  "legend": {
+                     "calcs": [
+                        "lastNotNull"
+                     ],
+                     "displayMode": "table",
+                     "placement": "bottom",
+                     "showLegend": true,
+                     "sortBy": "Last *",
+                     "sortDesc": true
+                  },
+                  "tooltip": {
+                     "mode": "single",
+                     "sort": "desc"
+                  }
+               },
+               "pluginVersion": "9.1.3",
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_del_obj_ops) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "time_series",
+                     "instant": false,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{bucket}}",
+                     "range": true,
+                     "refId": "A",
+                     "step": 300
+                  }
+               ],
+               "title": "Bucket Delete by Operations",
+               "type": "timeseries"
+            },
+            {
+               "datasource": "${datasource}",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "palette-classic"
+                     },
+                     "custom": {
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "",
+                        "axisPlacement": "auto",
+                        "barAlignment": 0,
+                        "drawStyle": "line",
+                        "fillOpacity": 0,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                           "legend": false,
+                           "tooltip": false,
+                           "viz": false
+                        },
+                        "lineInterpolation": "linear",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                           "type": "linear"
+                        },
+                        "showPoints": "auto",
+                        "spanNulls": true,
+                        "stacking": {
+                           "group": "A",
+                           "mode": "none"
+                        },
+                        "thresholdsStyle": {
+                           "mode": "off"
+                        }
+                     },
+                     "decimals": 2,
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     },
+                     "unit": "none"
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byType",
+                           "unit": "number"
+                        },
+                        "properties": [
+                           {
+                              "id": "color"
+                           },
+                           {
+                              "id": "color",
+                              "value": {
+                                 "mode": "palette-classic"
+                              }
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 12,
+                  "x": 0,
+                  "y": 53
+               },
+               "id": 27,
+               "options": {
+                  "legend": {
+                     "calcs": [
+                        "lastNotNull"
+                     ],
+                     "displayMode": "table",
+                     "placement": "bottom",
+                     "showLegend": true,
+                     "sortBy": "Last *",
+                     "sortDesc": true
+                  },
+                  "tooltip": {
+                     "mode": "single",
+                     "sort": "desc"
+                  }
+               },
+               "pluginVersion": "9.1.3",
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "sum by (bucket, ceph_daemon) ((ceph_rgw_op_per_bucket_copy_obj_ops) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "time_series",
+                     "instant": false,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{bucket}}",
+                     "range": true,
+                     "refId": "A",
+                     "step": 300
+                  }
+               ],
+               "title": "Bucket Copy by Operations",
+               "type": "timeseries"
+            },
+            {
+               "columns": [ ],
+               "datasource": "${datasource}",
+               "fieldConfig": {
+                  "defaults": {
+                     "custom": {
+                        "align": "auto",
+                        "cellOptions": {
+                           "type": "auto"
+                        },
+                        "filterable": false,
+                        "inspect": false
+                     },
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     }
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byType",
+                           "options": "number"
+                        },
+                        "properties": [
+                           {
+                              "id": "unit",
+                              "value": "none"
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 12,
+                  "x": 12,
+                  "y": 53
+               },
+               "id": 28,
+               "links": [ ],
+               "options": {
+                  "footer": {
+                     "countRows": false,
+                     "enablePagination": false,
+                     "fields": "",
+                     "reducer": [
+                        "sum"
+                     ],
+                     "show": false
+                  },
+                  "frameIndex": 1,
+                  "showHeader": true,
+                  "sortBy": [
+                     {
+                        "desc": true,
+                        "displayName": "PUTs"
+                     }
+                  ]
+               },
+               "pluginVersion": "9.4.7",
+               "styles": "",
+               "targets": [
+                  {
+                     "datasource": {
+                        "type": "prometheus",
+                        "uid": "${datasource}"
+                     },
+                     "exemplar": false,
+                     "expr": "sum by (bucket, ceph_daemon) (ceph_rgw_op_per_bucket_put_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "table",
+                     "hide": false,
+                     "instant": true,
+                     "interval": "",
+                     "intervalFactor": 1,
+                     "legendFormat": "__auto",
+                     "range": false,
+                     "refId": "A"
+                  },
+                  {
+                     "datasource": {
+                        "type": "prometheus",
+                        "uid": "${datasource}"
+                     },
+                     "exemplar": false,
+                     "expr": "sum by (bucket, ceph_daemon) (ceph_rgw_op_per_bucket_get_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "table",
+                     "hide": false,
+                     "instant": true,
+                     "interval": "",
+                     "intervalFactor": 1,
+                     "legendFormat": "__auto",
+                     "range": false,
+                     "refId": "B"
+                  },
+                  {
+                     "datasource": {
+                        "type": "prometheus",
+                        "uid": "${datasource}"
+                     },
+                     "exemplar": false,
+                     "expr": "sum by (bucket, ceph_daemon) (ceph_rgw_op_per_bucket_del_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "table",
+                     "hide": false,
+                     "instant": true,
+                     "interval": "",
+                     "intervalFactor": 1,
+                     "legendFormat": "__auto",
+                     "range": false,
+                     "refId": "C"
+                  },
+                  {
+                     "datasource": {
+                        "type": "prometheus",
+                        "uid": "${datasource}"
+                     },
+                     "exemplar": false,
+                     "expr": "sum by (bucket, ceph_daemon) (ceph_rgw_op_per_bucket_copy_obj_bytes *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "table",
+                     "hide": false,
+                     "instant": true,
+                     "interval": "",
+                     "intervalFactor": 1,
+                     "legendFormat": "__auto",
+                     "range": false,
+                     "refId": "D"
+                  },
+                  {
+                     "datasource": {
+                        "type": "prometheus",
+                        "uid": "${datasource}"
+                     },
+                     "exemplar": false,
+                     "expr": "sum by (bucket, ceph_daemon) (ceph_rgw_op_per_bucket_list_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "table",
+                     "hide": false,
+                     "instant": true,
+                     "interval": "",
+                     "intervalFactor": 1,
+                     "legendFormat": "__auto",
+                     "range": false,
+                     "refId": "E"
+                  }
+               ],
+               "timeFrom": null,
+               "timeShift": null,
+               "title": "Summary Per Bucket by Operations",
+               "transformations": [
+                  {
+                     "id": "merge",
+                     "options": { }
+                  },
+                  {
+                     "id": "joinByField",
+                     "options": {
+                        "byField": "Bucket",
+                        "mode": "outer"
+                     }
+                  },
+                  {
+                     "id": "groupBy",
+                     "options": {
+                        "fields": {
+                           "Bucket": {
+                              "aggregations": [ ],
+                              "operation": "groupby"
+                           },
+                           "Value #A": {
+                              "aggregations": [ ],
+                              "operation": "groupby"
+                           },
+                           "Value #B": {
+                              "aggregations": [ ],
+                              "operation": "groupby"
+                           },
+                           "Value #C": {
+                              "aggregations": [ ],
+                              "operation": "groupby"
+                           },
+                           "Value #D": {
+                              "aggregations": [ ],
+                              "operation": "groupby"
+                           },
+                           "Value #F": {
+                              "aggregations": [ ],
+                              "operation": "groupby"
+                           },
+                           "bucket": {
+                              "aggregations": [ ],
+                              "operation": "groupby"
+                           },
+                           "ceph_daemon": {
+                              "aggregations": [ ],
+                              "operation": "groupby"
+                           }
+                        }
+                     }
+                  },
+                  {
+                     "id": "organize",
+                     "options": {
+                        "excludeByName": {
+                           "Time 1": true,
+                           "Time 2": true,
+                           "Time 3": true,
+                           "Time 4": true,
+                           "Time 5": true,
+                           "Time 6": true,
+                           "Time 7": true,
+                           "__name__": true,
+                           "__name__ 1": true,
+                           "__name__ 2": true,
+                           "__name__ 3": true,
+                           "__name__ 4": true,
+                           "__name__ 5": true,
+                           "__name__ 6": true,
+                           "__name__ 7": true,
+                           "ceph_daemon 1": true,
+                           "ceph_daemon 2": true,
+                           "ceph_daemon 3": true,
+                           "ceph_daemon 4": true,
+                           "instance 1": true,
+                           "instance 2": true,
+                           "instance 3": true,
+                           "instance 4": true,
+                           "instance 5": true,
+                           "instance 6": true,
+                           "instance 7": true,
+                           "instance_id 1": true,
+                           "instance_id 2": true,
+                           "instance_id 3": true,
+                           "instance_id 4": true,
+                           "instance_id 5": true,
+                           "instance_id 6": true,
+                           "instance_id 7": true,
+                           "job 1": true,
+                           "job 2": true,
+                           "job 3": true,
+                           "job 4": true,
+                           "job 5": true,
+                           "job 6": true,
+                           "job 7": true
+                        },
+                        "indexByName": {
+                           "Value #A": 2,
+                           "Value #B": 3,
+                           "Value #C": 4,
+                           "Value #D": 5,
+                           "Value #F": 6,
+                           "bucket": 1,
+                           "ceph_daemon": 0
+                        },
+                        "renameByName": {
+                           "Bucket": "",
+                           "Value #A": "PUTs",
+                           "Value #B": "GETs",
+                           "Value #C": "List",
+                           "Value #D": "Delete",
+                           "Value #E": "Copy",
+                           "Value #F": "Copy",
+                           "Value #G": "",
+                           "bucket": "Bucket",
+                           "ceph_daemon": "Daemon"
+                        }
+                     }
+                  }
+               ],
+               "type": "table"
+            }
+         ],
+         "repeat": null,
+         "repeatIteration": null,
+         "repeatRowId": null,
+         "showTitle": true,
+         "title": "Buckets",
+         "titleSize": "h6",
+         "type": "row"
+      },
+      {
+         "collapse": false,
+         "collapsed": true,
+         "gridPos": {
+            "h": 1,
+            "w": 24,
+            "x": 0,
+            "y": 29
+         },
+         "id": 29,
+         "panels": [
+            {
+               "datasource": "${datasource}",
+               "description": "",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "thresholds"
+                     },
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     }
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byType",
+                           "unit": "number"
+                        },
+                        "properties": [
+                           {
+                              "id": "color"
+                           },
+                           {
+                              "id": "color",
+                              "value": {
+                                 "mode": "palette-classic"
+                              }
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 6,
+                  "x": 0,
+                  "y": 62
+               },
+               "id": 30,
+               "options": {
+                  "displayMode": "gradient",
+                  "orientation": "horizontal",
+                  "reduceOptions": {
+                     "calcs": [
+                        "lastNotNull"
+                     ]
+                  }
+               },
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "topk(5, \n    sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_put_obj_ops ) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})\n)\n",
+                     "format": "time_series",
+                     "instant": true,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{user}}",
+                     "range": false,
+                     "refId": "A"
+                  }
+               ],
+               "title": "Top 5 Users PUTs By Operations",
+               "type": "bargauge"
+            },
+            {
+               "datasource": "${datasource}",
+               "description": "",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "thresholds"
+                     },
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     }
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byType",
+                           "unit": "number"
+                        },
+                        "properties": [
+                           {
+                              "id": "color"
+                           },
+                           {
+                              "id": "color",
+                              "value": {
+                                 "mode": "palette-classic"
+                              }
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 6,
+                  "x": 6,
+                  "y": 62
+               },
+               "id": 31,
+               "options": {
+                  "displayMode": "gradient",
+                  "orientation": "horizontal",
+                  "reduceOptions": {
+                     "calcs": [
+                        "lastNotNull"
+                     ]
+                  }
+               },
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "topk(5, \n    sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_get_obj_ops ) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})\n)\n",
+                     "format": "time_series",
+                     "instant": true,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{user}}",
+                     "range": false,
+                     "refId": "A"
+                  }
+               ],
+               "title": "Top 5 Users GETs by Operations",
+               "type": "bargauge"
+            },
+            {
+               "datasource": "${datasource}",
+               "description": "",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "thresholds"
+                     },
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     }
+                  }
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 6,
+                  "x": 12,
+                  "y": 62
+               },
+               "id": 32,
+               "options": {
+                  "displayMode": "gradient",
+                  "orientation": "horizontal",
+                  "reduceOptions": {
+                     "calcs": [ ]
+                  }
+               },
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "topk(5, \n    sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_put_obj_bytes) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})\n)",
+                     "format": "time_series",
+                     "instant": true,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{user}}",
+                     "range": false,
+                     "refId": "A"
+                  }
+               ],
+               "title": "Top 5 Users PUTs by Size",
+               "type": "bargauge"
+            },
+            {
+               "datasource": "${datasource}",
+               "description": "",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "thresholds"
+                     },
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     }
+                  }
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 6,
+                  "x": 18,
+                  "y": 62
+               },
+               "id": 33,
+               "options": {
+                  "displayMode": "gradient",
+                  "orientation": "horizontal",
+                  "reduceOptions": {
+                     "calcs": [ ]
+                  }
+               },
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "topk(5, \n    sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_get_obj_bytes) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})\n)",
+                     "format": "time_series",
+                     "instant": true,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{user}}",
+                     "range": false,
+                     "refId": "A"
+                  }
+               ],
+               "title": "Top 5 Users GETs By Size",
+               "type": "bargauge"
+            },
+            {
+               "datasource": "${datasource}",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "palette-classic"
+                     },
+                     "custom": {
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "",
+                        "axisPlacement": "auto",
+                        "barAlignment": 0,
+                        "drawStyle": "line",
+                        "fillOpacity": 0,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                           "legend": false,
+                           "tooltip": false,
+                           "viz": false
+                        },
+                        "lineInterpolation": "linear",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                           "type": "linear"
+                        },
+                        "showPoints": "auto",
+                        "spanNulls": true,
+                        "stacking": {
+                           "group": "A",
+                           "mode": "none"
+                        },
+                        "thresholdsStyle": {
+                           "mode": "off"
+                        }
+                     },
+                     "decimals": 2,
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     },
+                     "unit": "decbytes"
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byType",
+                           "unit": "number"
+                        },
+                        "properties": [
+                           {
+                              "id": "color"
+                           },
+                           {
+                              "id": "color",
+                              "value": {
+                                 "mode": "palette-classic"
+                              }
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 6,
+                  "x": 0,
+                  "y": 70
+               },
+               "id": 34,
+               "options": {
+                  "legend": {
+                     "calcs": [
+                        "lastNotNull"
+                     ],
+                     "displayMode": "table",
+                     "placement": "bottom",
+                     "showLegend": true,
+                     "sortBy": "Last *",
+                     "sortDesc": true
+                  },
+                  "tooltip": {
+                     "mode": "single",
+                     "sort": "desc"
+                  }
+               },
+               "pluginVersion": "9.1.3",
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_put_obj_bytes) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "time_series",
+                     "instant": false,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{user}}",
+                     "range": true,
+                     "refId": "A",
+                     "step": 300
+                  }
+               ],
+               "title": "User PUTs by Size",
+               "type": "timeseries"
+            },
+            {
+               "datasource": "${datasource}",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "palette-classic"
+                     },
+                     "custom": {
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "",
+                        "axisPlacement": "auto",
+                        "barAlignment": 0,
+                        "drawStyle": "line",
+                        "fillOpacity": 0,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                           "legend": false,
+                           "tooltip": false,
+                           "viz": false
+                        },
+                        "lineInterpolation": "linear",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                           "type": "linear"
+                        },
+                        "showPoints": "auto",
+                        "spanNulls": true,
+                        "stacking": {
+                           "group": "A",
+                           "mode": "none"
+                        },
+                        "thresholdsStyle": {
+                           "mode": "off"
+                        }
+                     },
+                     "decimals": 2,
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     },
+                     "unit": "decbytes"
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byType",
+                           "unit": "number"
+                        },
+                        "properties": [
+                           {
+                              "id": "color"
+                           },
+                           {
+                              "id": "color",
+                              "value": {
+                                 "mode": "palette-classic"
+                              }
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 6,
+                  "x": 6,
+                  "y": 70
+               },
+               "id": 35,
+               "options": {
+                  "legend": {
+                     "calcs": [
+                        "lastNotNull"
+                     ],
+                     "displayMode": "table",
+                     "placement": "bottom",
+                     "showLegend": true,
+                     "sortBy": "Last *",
+                     "sortDesc": true
+                  },
+                  "tooltip": {
+                     "mode": "single",
+                     "sort": "desc"
+                  }
+               },
+               "pluginVersion": "9.1.3",
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_get_obj_bytes) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "time_series",
+                     "instant": false,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{user}}",
+                     "range": true,
+                     "refId": "A",
+                     "step": 300
+                  }
+               ],
+               "title": "User GETs by Size",
+               "type": "timeseries"
+            },
+            {
+               "datasource": "${datasource}",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "palette-classic"
+                     },
+                     "custom": {
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "",
+                        "axisPlacement": "auto",
+                        "barAlignment": 0,
+                        "drawStyle": "line",
+                        "fillOpacity": 0,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                           "legend": false,
+                           "tooltip": false,
+                           "viz": false
+                        },
+                        "lineInterpolation": "linear",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                           "type": "linear"
+                        },
+                        "showPoints": "auto",
+                        "spanNulls": true,
+                        "stacking": {
+                           "group": "A",
+                           "mode": "none"
+                        },
+                        "thresholdsStyle": {
+                           "mode": "off"
+                        }
+                     },
+                     "decimals": 2,
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     },
+                     "unit": "decbytes"
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byType",
+                           "unit": "number"
+                        },
+                        "properties": [
+                           {
+                              "id": "color"
+                           },
+                           {
+                              "id": "color",
+                              "value": {
+                                 "mode": "palette-classic"
+                              }
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 6,
+                  "x": 12,
+                  "y": 70
+               },
+               "id": 36,
+               "options": {
+                  "legend": {
+                     "calcs": [
+                        "lastNotNull"
+                     ],
+                     "displayMode": "table",
+                     "placement": "bottom",
+                     "showLegend": true,
+                     "sortBy": "Last *",
+                     "sortDesc": true
+                  },
+                  "tooltip": {
+                     "mode": "single",
+                     "sort": "desc"
+                  }
+               },
+               "pluginVersion": "9.1.3",
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_del_obj_bytes) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "time_series",
+                     "instant": false,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{user}}",
+                     "range": true,
+                     "refId": "A",
+                     "step": 300
+                  }
+               ],
+               "title": "User Delete by Size",
+               "type": "timeseries"
+            },
+            {
+               "datasource": "${datasource}",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "palette-classic"
+                     },
+                     "custom": {
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "",
+                        "axisPlacement": "auto",
+                        "barAlignment": 0,
+                        "drawStyle": "line",
+                        "fillOpacity": 0,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                           "legend": false,
+                           "tooltip": false,
+                           "viz": false
+                        },
+                        "lineInterpolation": "linear",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                           "type": "linear"
+                        },
+                        "showPoints": "auto",
+                        "spanNulls": true,
+                        "stacking": {
+                           "group": "A",
+                           "mode": "none"
+                        },
+                        "thresholdsStyle": {
+                           "mode": "off"
+                        }
+                     },
+                     "decimals": 2,
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     },
+                     "unit": "decbytes"
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byType",
+                           "unit": "number"
+                        },
+                        "properties": [
+                           {
+                              "id": "color"
+                           },
+                           {
+                              "id": "color",
+                              "value": {
+                                 "mode": "palette-classic"
+                              }
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 6,
+                  "x": 18,
+                  "y": 70
+               },
+               "id": 37,
+               "options": {
+                  "legend": {
+                     "calcs": [
+                        "lastNotNull"
+                     ],
+                     "displayMode": "table",
+                     "placement": "bottom",
+                     "showLegend": true,
+                     "sortBy": "Last *",
+                     "sortDesc": true
+                  },
+                  "tooltip": {
+                     "mode": "single",
+                     "sort": "desc"
+                  }
+               },
+               "pluginVersion": "9.1.3",
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_copy_obj_bytes) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "time_series",
+                     "instant": false,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{user}}",
+                     "range": true,
+                     "refId": "A",
+                     "step": 300
+                  }
+               ],
+               "title": "User COPY by Size",
+               "type": "timeseries"
+            },
+            {
+               "datasource": "${datasource}",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "palette-classic"
+                     },
+                     "custom": {
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "",
+                        "axisPlacement": "auto",
+                        "barAlignment": 0,
+                        "drawStyle": "line",
+                        "fillOpacity": 0,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                           "legend": false,
+                           "tooltip": false,
+                           "viz": false
+                        },
+                        "lineInterpolation": "linear",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                           "type": "linear"
+                        },
+                        "showPoints": "auto",
+                        "spanNulls": true,
+                        "stacking": {
+                           "group": "A",
+                           "mode": "none"
+                        },
+                        "thresholdsStyle": {
+                           "mode": "off"
+                        }
+                     },
+                     "decimals": 2,
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     },
+                     "unit": "none"
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byType",
+                           "unit": "number"
+                        },
+                        "properties": [
+                           {
+                              "id": "color"
+                           },
+                           {
+                              "id": "color",
+                              "value": {
+                                 "mode": "palette-classic"
+                              }
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 6,
+                  "x": 0,
+                  "y": 78
+               },
+               "id": 38,
+               "options": {
+                  "legend": {
+                     "calcs": [
+                        "lastNotNull"
+                     ],
+                     "displayMode": "table",
+                     "placement": "bottom",
+                     "showLegend": true,
+                     "sortBy": "Last *",
+                     "sortDesc": true
+                  },
+                  "tooltip": {
+                     "mode": "single",
+                     "sort": "desc"
+                  }
+               },
+               "pluginVersion": "9.1.3",
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_get_obj_ops) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "time_series",
+                     "instant": false,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{user}}",
+                     "range": true,
+                     "refId": "A",
+                     "step": 300
+                  }
+               ],
+               "title": "User GETs by Operations",
+               "type": "timeseries"
+            },
+            {
+               "datasource": "${datasource}",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "palette-classic"
+                     },
+                     "custom": {
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "",
+                        "axisPlacement": "auto",
+                        "barAlignment": 0,
+                        "drawStyle": "line",
+                        "fillOpacity": 0,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                           "legend": false,
+                           "tooltip": false,
+                           "viz": false
+                        },
+                        "lineInterpolation": "linear",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                           "type": "linear"
+                        },
+                        "showPoints": "auto",
+                        "spanNulls": true,
+                        "stacking": {
+                           "group": "A",
+                           "mode": "none"
+                        },
+                        "thresholdsStyle": {
+                           "mode": "off"
+                        }
+                     },
+                     "decimals": 2,
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     },
+                     "unit": "none"
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byType",
+                           "unit": "number"
+                        },
+                        "properties": [
+                           {
+                              "id": "color"
+                           },
+                           {
+                              "id": "color",
+                              "value": {
+                                 "mode": "palette-classic"
+                              }
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 6,
+                  "x": 6,
+                  "y": 78
+               },
+               "id": 39,
+               "options": {
+                  "legend": {
+                     "calcs": [
+                        "lastNotNull"
+                     ],
+                     "displayMode": "table",
+                     "placement": "bottom",
+                     "showLegend": true,
+                     "sortBy": "Last *",
+                     "sortDesc": true
+                  },
+                  "tooltip": {
+                     "mode": "single",
+                     "sort": "desc"
+                  }
+               },
+               "pluginVersion": "9.1.3",
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_put_obj_ops) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "time_series",
+                     "instant": false,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{user}}",
+                     "range": true,
+                     "refId": "A",
+                     "step": 300
+                  }
+               ],
+               "title": "User PUTs by Operations",
+               "type": "timeseries"
+            },
+            {
+               "datasource": "${datasource}",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "palette-classic"
+                     },
+                     "custom": {
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "",
+                        "axisPlacement": "auto",
+                        "barAlignment": 0,
+                        "drawStyle": "line",
+                        "fillOpacity": 0,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                           "legend": false,
+                           "tooltip": false,
+                           "viz": false
+                        },
+                        "lineInterpolation": "linear",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                           "type": "linear"
+                        },
+                        "showPoints": "auto",
+                        "spanNulls": true,
+                        "stacking": {
+                           "group": "A",
+                           "mode": "none"
+                        },
+                        "thresholdsStyle": {
+                           "mode": "off"
+                        }
+                     },
+                     "decimals": 2,
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     },
+                     "unit": "none"
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byType",
+                           "unit": "number"
+                        },
+                        "properties": [
+                           {
+                              "id": "color"
+                           },
+                           {
+                              "id": "color",
+                              "value": {
+                                 "mode": "palette-classic"
+                              }
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 6,
+                  "x": 12,
+                  "y": 78
+               },
+               "id": 40,
+               "options": {
+                  "legend": {
+                     "calcs": [
+                        "lastNotNull"
+                     ],
+                     "displayMode": "table",
+                     "placement": "bottom",
+                     "showLegend": true,
+                     "sortBy": "Last *",
+                     "sortDesc": true
+                  },
+                  "tooltip": {
+                     "mode": "single",
+                     "sort": "desc"
+                  }
+               },
+               "pluginVersion": "9.1.3",
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_list_obj_ops) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "time_series",
+                     "instant": false,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{user}}",
+                     "range": true,
+                     "refId": "A",
+                     "step": 300
+                  }
+               ],
+               "title": "User List by Operations",
+               "type": "timeseries"
+            },
+            {
+               "datasource": "${datasource}",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "palette-classic"
+                     },
+                     "custom": {
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "",
+                        "axisPlacement": "auto",
+                        "barAlignment": 0,
+                        "drawStyle": "line",
+                        "fillOpacity": 0,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                           "legend": false,
+                           "tooltip": false,
+                           "viz": false
+                        },
+                        "lineInterpolation": "linear",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                           "type": "linear"
+                        },
+                        "showPoints": "auto",
+                        "spanNulls": true,
+                        "stacking": {
+                           "group": "A",
+                           "mode": "none"
+                        },
+                        "thresholdsStyle": {
+                           "mode": "off"
+                        }
+                     },
+                     "decimals": 2,
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     },
+                     "unit": "none"
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byType",
+                           "unit": "number"
+                        },
+                        "properties": [
+                           {
+                              "id": "color"
+                           },
+                           {
+                              "id": "color",
+                              "value": {
+                                 "mode": "palette-classic"
+                              }
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 6,
+                  "x": 18,
+                  "y": 78
+               },
+               "id": 41,
+               "options": {
+                  "legend": {
+                     "calcs": [
+                        "lastNotNull"
+                     ],
+                     "displayMode": "table",
+                     "placement": "bottom",
+                     "showLegend": true,
+                     "sortBy": "Last *",
+                     "sortDesc": true
+                  },
+                  "tooltip": {
+                     "mode": "single",
+                     "sort": "desc"
+                  }
+               },
+               "pluginVersion": "9.1.3",
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_del_obj_ops) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "time_series",
+                     "instant": false,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{user}}",
+                     "range": true,
+                     "refId": "A",
+                     "step": 300
+                  }
+               ],
+               "title": "User Delete by Operations",
+               "type": "timeseries"
+            },
+            {
+               "datasource": "${datasource}",
+               "fieldConfig": {
+                  "defaults": {
+                     "color": {
+                        "mode": "palette-classic"
+                     },
+                     "custom": {
+                        "axisCenteredZero": false,
+                        "axisColorMode": "text",
+                        "axisLabel": "",
+                        "axisPlacement": "auto",
+                        "barAlignment": 0,
+                        "drawStyle": "line",
+                        "fillOpacity": 0,
+                        "gradientMode": "none",
+                        "hideFrom": {
+                           "legend": false,
+                           "tooltip": false,
+                           "viz": false
+                        },
+                        "lineInterpolation": "linear",
+                        "lineWidth": 1,
+                        "pointSize": 5,
+                        "scaleDistribution": {
+                           "type": "linear"
+                        },
+                        "showPoints": "auto",
+                        "spanNulls": true,
+                        "stacking": {
+                           "group": "A",
+                           "mode": "none"
+                        },
+                        "thresholdsStyle": {
+                           "mode": "off"
+                        }
+                     },
+                     "decimals": 2,
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     },
+                     "unit": "none"
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byType",
+                           "unit": "number"
+                        },
+                        "properties": [
+                           {
+                              "id": "color"
+                           },
+                           {
+                              "id": "color",
+                              "value": {
+                                 "mode": "palette-classic"
+                              }
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 12,
+                  "x": 0,
+                  "y": 86
+               },
+               "id": 42,
+               "options": {
+                  "legend": {
+                     "calcs": [
+                        "lastNotNull"
+                     ],
+                     "displayMode": "table",
+                     "placement": "bottom",
+                     "showLegend": true,
+                     "sortBy": "Last *",
+                     "sortDesc": true
+                  },
+                  "tooltip": {
+                     "mode": "single",
+                     "sort": "desc"
+                  }
+               },
+               "pluginVersion": "9.1.3",
+               "targets": [
+                  {
+                     "datasource": "${datasource}",
+                     "expr": "sum by (user, ceph_daemon) ((ceph_rgw_op_per_user_copy_obj_ops) *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "time_series",
+                     "instant": false,
+                     "intervalFactor": 1,
+                     "legendFormat": "{{ceph_daemon}} - {{user}}",
+                     "range": true,
+                     "refId": "A",
+                     "step": 300
+                  }
+               ],
+               "title": "User Copy by Operations",
+               "type": "timeseries"
+            },
+            {
+               "columns": [ ],
+               "datasource": "${datasource}",
+               "fieldConfig": {
+                  "defaults": {
+                     "custom": {
+                        "align": "auto",
+                        "cellOptions": {
+                           "type": "auto"
+                        },
+                        "filterable": false,
+                        "inspect": false
+                     },
+                     "thresholds": {
+                        "mode": "absolute",
+                        "steps": [
+                           {
+                              "color": "green"
+                           }
+                        ]
+                     }
+                  },
+                  "overrides": [
+                     {
+                        "matcher": {
+                           "id": "byType",
+                           "options": "number"
+                        },
+                        "properties": [
+                           {
+                              "id": "unit",
+                              "value": "none"
+                           }
+                        ]
+                     }
+                  ]
+               },
+               "gridPos": {
+                  "h": 8,
+                  "w": 12,
+                  "x": 12,
+                  "y": 86
+               },
+               "id": 43,
+               "links": [ ],
+               "options": {
+                  "footer": {
+                     "countRows": false,
+                     "enablePagination": false,
+                     "fields": "",
+                     "reducer": [
+                        "sum"
+                     ],
+                     "show": false
+                  },
+                  "frameIndex": 1,
+                  "showHeader": true,
+                  "sortBy": [
+                     {
+                        "desc": true,
+                        "displayName": "PUTs"
+                     }
+                  ]
+               },
+               "pluginVersion": "9.4.7",
+               "styles": "",
+               "targets": [
+                  {
+                     "datasource": {
+                        "type": "prometheus",
+                        "uid": "${datasource}"
+                     },
+                     "exemplar": false,
+                     "expr": "sum by (user, ceph_daemon) (ceph_rgw_op_per_user_put_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "table",
+                     "hide": false,
+                     "instant": true,
+                     "interval": "",
+                     "intervalFactor": 1,
+                     "legendFormat": "__auto",
+                     "range": false,
+                     "refId": "A"
+                  },
+                  {
+                     "datasource": {
+                        "type": "prometheus",
+                        "uid": "${datasource}"
+                     },
+                     "exemplar": false,
+                     "expr": "sum by (user, ceph_daemon) (ceph_rgw_op_per_user_get_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "table",
+                     "hide": false,
+                     "instant": true,
+                     "interval": "",
+                     "intervalFactor": 1,
+                     "legendFormat": "__auto",
+                     "range": false,
+                     "refId": "B"
+                  },
+                  {
+                     "datasource": {
+                        "type": "prometheus",
+                        "uid": "${datasource}"
+                     },
+                     "exemplar": false,
+                     "expr": "sum by (user, ceph_daemon) (ceph_rgw_op_per_user_del_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "table",
+                     "hide": false,
+                     "instant": true,
+                     "interval": "",
+                     "intervalFactor": 1,
+                     "legendFormat": "__auto",
+                     "range": false,
+                     "refId": "C"
+                  },
+                  {
+                     "datasource": {
+                        "type": "prometheus",
+                        "uid": "${datasource}"
+                     },
+                     "exemplar": false,
+                     "expr": "sum by (user, ceph_daemon) (ceph_rgw_op_per_user_copy_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "table",
+                     "hide": false,
+                     "instant": true,
+                     "interval": "",
+                     "intervalFactor": 1,
+                     "legendFormat": "__auto",
+                     "range": false,
+                     "refId": "D"
+                  },
+                  {
+                     "datasource": {
+                        "type": "prometheus",
+                        "uid": "${datasource}"
+                     },
+                     "exemplar": false,
+                     "expr": "sum by (user, ceph_daemon) (ceph_rgw_op_per_user_list_obj_ops *\n    on (instance_id) group_left (ceph_daemon) ceph_rgw_metadata{ceph_daemon=~\"$rgw_servers\"})",
+                     "format": "table",
+                     "hide": false,
+                     "instant": true,
+                     "interval": "",
+                     "intervalFactor": 1,
+                     "legendFormat": "__auto",
+                     "range": false,
+                     "refId": "E"
+                  }
+               ],
+               "timeFrom": null,
+               "timeShift": null,
+               "title": "Summary Per User By Operations",
+               "transformations": [
+                  {
+                     "id": "merge",
+                     "options": { }
+                  },
+                  {
+                     "id": "joinByField",
+                     "options": {
+                        "byField": "User",
+                        "mode": "outer"
+                     }
+                  },
+                  {
+                     "id": "groupBy",
+                     "options": {
+                        "fields": {
+                           "User": {
+                              "aggregations": [ ],
+                              "operation": "groupby"
+                           },
+                           "Value #A": {
+                              "aggregations": [ ],
+                              "operation": "groupby"
+                           },
+                           "Value #B": {
+                              "aggregations": [ ],
+                              "operation": "groupby"
+                           },
+                           "Value #C": {
+                              "aggregations": [ ],
+                              "operation": "groupby"
+                           },
+                           "Value #D": {
+                              "aggregations": [ ],
+                              "operation": "groupby"
+                           },
+                           "Value #F": {
+                              "aggregations": [ ],
+                              "operation": "groupby"
+                           },
+                           "ceph_daemon": {
+                              "aggregations": [ ],
+                              "operation": "groupby"
+                           },
+                           "user": {
+                              "aggregations": [ ],
+                              "operation": "groupby"
+                           }
+                        }
+                     }
+                  },
+                  {
+                     "id": "organize",
+                     "options": {
+                        "excludeByName": { },
+                        "indexByName": {
+                           "Value #A": 2,
+                           "Value #B": 3,
+                           "Value #C": 4,
+                           "Value #D": 5,
+                           "Value #F": 6,
+                           "ceph_daemon": 0,
+                           "user": 1
+                        },
+                        "renameByName": {
+                           "Value #A": "PUTs",
+                           "Value #B": "GETs",
+                           "Value #C": "LIST",
+                           "Value #D": "DELETE",
+                           "Value #F": "COPY",
+                           "ceph_daemon": "Daemon",
+                           "user": "User"
+                        }
+                     }
+                  }
+               ],
+               "type": "table"
+            }
+         ],
+         "repeat": null,
+         "repeatIteration": null,
+         "repeatRowId": null,
+         "showTitle": true,
+         "title": "Users",
+         "titleSize": "h6",
+         "type": "row"
+      }
+   ],
+   "refresh": "30s",
+   "rows": [ ],
+   "schemaVersion": 22,
+   "style": "dark",
+   "tags": [
+      "ceph-mixin"
+   ],
+   "templating": {
+      "list": [
+         {
+            "current": {
+               "text": "default",
+               "value": "default"
+            },
+            "hide": 0,
+            "label": "Data Source",
+            "name": "datasource",
+            "options": [ ],
+            "query": "prometheus",
+            "refresh": 1,
+            "regex": "",
+            "type": "datasource"
+         },
+         {
+            "allValue": null,
+            "current": { },
+            "datasource": "$datasource",
+            "hide": 0,
+            "includeAll": true,
+            "label": null,
+            "multi": false,
+            "name": "rgw_servers",
+            "options": [ ],
+            "query": "label_values(ceph_rgw_metadata{}, ceph_daemon)",
+            "refresh": 2,
+            "regex": "",
+            "sort": 0,
+            "tagValuesQuery": "",
+            "tags": [ ],
+            "tagsQuery": "",
+            "type": "query",
+            "useTags": false
+         },
+         {
+            "datasource": "$datasource",
+            "hide": 2,
+            "label": "filters",
+            "name": "Filters",
+            "type": "adhoc"
+         }
+      ]
+   },
+   "time": {
+      "from": "now-1h",
+      "to": "now"
+   },
+   "timepicker": {
+      "refresh_intervals": [
+         "5s",
+         "10s",
+         "30s",
+         "1m",
+         "5m",
+         "15m",
+         "30m",
+         "1h",
+         "2h",
+         "1d"
+      ],
+      "time_options": [
+         "5m",
+         "15m",
+         "1h",
+         "6h",
+         "12h",
+         "24h",
+         "2d",
+         "7d",
+         "30d"
+      ]
+   },
+   "timezone": "",
+   "title": "RGW S3 Analytics",
+   "uid": "BnxelG7Sz",
+   "version": 0
+}
diff --git a/monitoring/ceph-mixin/tests_dashboards/util.py b/monitoring/ceph-mixin/tests_dashboards/util.py
index 4213ff6d5a1..c6369c5f834 100644
--- a/monitoring/ceph-mixin/tests_dashboards/util.py
+++ b/monitoring/ceph-mixin/tests_dashboards/util.py
@@ -55,7 +55,7 @@ def add_dashboard_queries(data: Dict[str, Any], dashboard_data: Dict[str, Any],
                 title = panel['title']
                 legend_format = target['legendFormat'] if 'legendFormat' in target else ""
                 query_id = f'{title}-{legend_format}'
-                if query_id in panel_ids_in_file:
+                if query_id in panel_ids_in_file and legend_format != '__auto':
                     cprint((f'ERROR: Query in panel "{title}" with legend "{legend_format}"'
                             f' already exists in the same file: "{path}"'), 'red')
                     error = 1