crimson/os/seastore: track shard io stats below transaction manager

Including background transactions.

Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>

9 files changed, 99 insertions, 15 deletions

diff --git a/src/crimson/os/seastore/async_cleaner.cc b/src/crimson/os/seastore/async_cleaner.cc
index 4ca5d1a2069..00e7fa38956 100644
--- a/src/crimson/os/seastore/async_cleaner.cc
+++ b/src/crimson/os/seastore/async_cleaner.cc
@@ -598,7 +598,14 @@ JournalTrimmerImpl::trim_alloc()
 {
   LOG_PREFIX(JournalTrimmerImpl::trim_alloc);
   assert(background_callback->is_ready());
-  return repeat_eagain([this, FNAME] {
+
+  auto& shard_stats = extent_callback->get_shard_stats();
+  ++(shard_stats.trim_alloc_num);
+  ++(shard_stats.pending_bg_num);
+
+  return repeat_eagain([this, FNAME, &shard_stats] {
+    ++(shard_stats.repeat_trim_alloc_num);
+
     return extent_callback->with_transaction_intr(
       Transaction::src_t::TRIM_ALLOC,
       "trim_alloc",
@@ -622,8 +629,11 @@ JournalTrimmerImpl::trim_alloc()
         return seastar::now();
       });
     });
-  }).safe_then([this, FNAME] {
+  }).finally([this, FNAME, &shard_stats] {
     DEBUG("finish, alloc_tail={}", journal_alloc_tail);
+
+    assert(shard_stats.pending_bg_num);
+    --(shard_stats.pending_bg_num);
   });
 }
 
@@ -632,7 +642,14 @@ JournalTrimmerImpl::trim_dirty()
 {
   LOG_PREFIX(JournalTrimmerImpl::trim_dirty);
   assert(background_callback->is_ready());
-  return repeat_eagain([this, FNAME] {
+
+  auto& shard_stats = extent_callback->get_shard_stats();
+  ++(shard_stats.trim_dirty_num);
+  ++(shard_stats.pending_bg_num);
+
+  return repeat_eagain([this, FNAME, &shard_stats] {
+    ++(shard_stats.repeat_trim_dirty_num);
+
     return extent_callback->with_transaction_intr(
       Transaction::src_t::TRIM_DIRTY,
       "trim_dirty",
@@ -662,8 +679,11 @@ JournalTrimmerImpl::trim_dirty()
         return extent_callback->submit_transaction_direct(t);
       });
     });
-  }).safe_then([this, FNAME] {
+  }).finally([this, FNAME, &shard_stats] {
     DEBUG("finish, dirty_tail={}", journal_dirty_tail);
+
+    assert(shard_stats.pending_bg_num);
+    --(shard_stats.pending_bg_num);
   });
 }
 
@@ -1073,6 +1093,14 @@ SegmentCleaner::do_reclaim_space(
     std::size_t &reclaimed,
     std::size_t &runs)
 {
+  auto& shard_stats = extent_callback->get_shard_stats();
+  if (is_cold) {
+    ++(shard_stats.cleaner_cold_num);
+  } else {
+    ++(shard_stats.cleaner_main_num);
+  }
+  ++(shard_stats.pending_bg_num);
+
   // Extents satisfying any of the following requirements
   // are considered DEAD:
   // 1. can't find the corresponding mapping in both the
@@ -1082,13 +1110,17 @@ SegmentCleaner::do_reclaim_space(
   // 	tree doesn't match the extent's paddr
   // 3. the extent is physical and doesn't exist in the
   // 	lba tree, backref tree or backref cache;
-  return repeat_eagain([this, &backref_extents,
+  return repeat_eagain([this, &backref_extents, &shard_stats,
                         &pin_list, &reclaimed, &runs] {
     reclaimed = 0;
     runs++;
-    auto src = Transaction::src_t::CLEANER_MAIN;
+    transaction_type_t src;
     if (is_cold) {
       src = Transaction::src_t::CLEANER_COLD;
+      ++(shard_stats.repeat_cleaner_cold_num);
+    } else {
+      src = Transaction::src_t::CLEANER_MAIN;
+      ++(shard_stats.repeat_cleaner_main_num);
     }
     return extent_callback->with_transaction_intr(
       src,
@@ -1167,6 +1199,9 @@ SegmentCleaner::do_reclaim_space(
         return extent_callback->submit_transaction_direct(t);
       });
     });
+  }).finally([&shard_stats] {
+    assert(shard_stats.pending_bg_num);
+    --(shard_stats.pending_bg_num);
   });
 }
 
@@ -1202,6 +1237,7 @@ SegmentCleaner::clean_space_ret SegmentCleaner::clean_space()
     std::pair<std::vector<CachedExtentRef>, backref_pin_list_t>(),
     [this](auto &weak_read_ret) {
     return repeat_eagain([this, &weak_read_ret] {
+      // Note: not tracked by shard_stats_t intentionally.
       return extent_callback->with_transaction_intr(
 	  Transaction::src_t::READ,
 	  "retrieve_from_backref_tree",
diff --git a/src/crimson/os/seastore/async_cleaner.h b/src/crimson/os/seastore/async_cleaner.h
index 3b041d23064..adf9fb177ad 100644
--- a/src/crimson/os/seastore/async_cleaner.h
+++ b/src/crimson/os/seastore/async_cleaner.h
@@ -277,6 +277,8 @@ public:
 
   virtual ~ExtentCallbackInterface() = default;
 
+  virtual shard_stats_t& get_shard_stats() = 0;
+
   /// Creates empty transaction
   /// weak transaction should be type READ
   virtual TransactionRef create_transaction(
diff --git a/src/crimson/os/seastore/seastore.cc b/src/crimson/os/seastore/seastore.cc
index d764b4d4f36..011cadb612d 100644
--- a/src/crimson/os/seastore/seastore.cc
+++ b/src/crimson/os/seastore/seastore.cc
@@ -2469,7 +2469,7 @@ void SeaStore::Shard::init_managers()
   shard_stats = {};
 
   transaction_manager = make_transaction_manager(
-      device, secondaries, is_test);
+      device, secondaries, shard_stats, is_test);
   collection_manager = std::make_unique<collection_manager::FlatCollectionManager>(
       *transaction_manager);
   onode_manager = std::make_unique<crimson::os::seastore::onode::FLTreeOnodeManager>(
diff --git a/src/crimson/os/seastore/seastore_types.h b/src/crimson/os/seastore/seastore_types.h
index f98a720bd79..0412ed7e8a6 100644
--- a/src/crimson/os/seastore/seastore_types.h
+++ b/src/crimson/os/seastore/seastore_types.h
@@ -2383,6 +2383,17 @@ struct shard_stats_t {
   uint64_t repeat_read_num = 0;
   uint64_t pending_read_num = 0;
 
+  // transaction_type_t::TRIM_DIRTY~CLEANER_COLD
+  uint64_t pending_bg_num = 0;
+  uint64_t trim_alloc_num = 0;
+  uint64_t repeat_trim_alloc_num = 0;
+  uint64_t trim_dirty_num = 0;
+  uint64_t repeat_trim_dirty_num = 0;
+  uint64_t cleaner_main_num = 0;
+  uint64_t repeat_cleaner_main_num = 0;
+  uint64_t cleaner_cold_num = 0;
+  uint64_t repeat_cleaner_cold_num = 0;
+
   uint64_t flush_num = 0;
   uint64_t pending_flush_num = 0;
 };
diff --git a/src/crimson/os/seastore/transaction_manager.cc b/src/crimson/os/seastore/transaction_manager.cc
index 93a22a88331..517deb3f408 100644
--- a/src/crimson/os/seastore/transaction_manager.cc
+++ b/src/crimson/os/seastore/transaction_manager.cc
@@ -29,7 +29,8 @@ TransactionManager::TransactionManager(
   CacheRef _cache,
   LBAManagerRef _lba_manager,
   ExtentPlacementManagerRef &&_epm,
-  BackrefManagerRef&& _backref_manager)
+  BackrefManagerRef&& _backref_manager,
+  shard_stats_t& _shard_stats)
   : cache(std::move(_cache)),
     lba_manager(std::move(_lba_manager)),
     journal(std::move(_journal)),
@@ -37,7 +38,8 @@ TransactionManager::TransactionManager(
     backref_manager(std::move(_backref_manager)),
     full_extent_integrity_check(
       crimson::common::get_conf<bool>(
-        "seastore_full_integrity_check"))
+        "seastore_full_integrity_check")),
+    shard_stats(_shard_stats)
 {
   epm->set_extent_callback(this);
   journal->set_write_pipeline(&write_pipeline);
@@ -55,6 +57,12 @@ TransactionManager::mkfs_ertr::future<> TransactionManager::mkfs()
     journal->get_trimmer().set_journal_head(start_seq);
     return epm->open_for_write();
   }).safe_then([this, FNAME]() {
+    ++(shard_stats.io_num);
+    ++(shard_stats.pending_io_num);
+    // For submit_transaction_direct()
+    ++(shard_stats.processing_inlock_io_num);
+    ++(shard_stats.repeat_io_num);
+
     return with_transaction_intr(
       Transaction::src_t::MUTATE,
       "mkfs_tm",
@@ -76,7 +84,13 @@ TransactionManager::mkfs_ertr::future<> TransactionManager::mkfs()
         return mkfs_ertr::now();
       }),
       mkfs_ertr::pass_further{}
-    );
+    ).finally([this] {
+      assert(shard_stats.pending_io_num);
+      --(shard_stats.pending_io_num);
+      // XXX: it's wrong to assume no failure,
+      // but failure leads to fatal error
+      --(shard_stats.processing_postlock_io_num);
+    });
   }).safe_then([this] {
     return close();
   }).safe_then([FNAME] {
@@ -419,6 +433,10 @@ TransactionManager::do_submit_transaction(
       journal->get_trimmer().get_dirty_tail());
 
     tref.get_handle().maybe_release_collection_lock();
+    if (tref.get_src() == Transaction::src_t::MUTATE) {
+      --(shard_stats.processing_inlock_io_num);
+      ++(shard_stats.processing_postlock_io_num);
+    }
 
     SUBTRACET(seastore_t, "submitting record", tref);
     return journal->submit_record(std::move(record), tref.get_handle()
@@ -734,6 +752,7 @@ TransactionManager::~TransactionManager() {}
 TransactionManagerRef make_transaction_manager(
     Device *primary_device,
     const std::vector<Device*> &secondary_devices,
+    shard_stats_t& shard_stats,
     bool is_test)
 {
   auto epm = std::make_unique<ExtentPlacementManager>();
@@ -873,7 +892,8 @@ TransactionManagerRef make_transaction_manager(
     std::move(cache),
     std::move(lba_manager),
     std::move(epm),
-    std::move(backref_manager));
+    std::move(backref_manager),
+    shard_stats);
 }
 
 }
diff --git a/src/crimson/os/seastore/transaction_manager.h b/src/crimson/os/seastore/transaction_manager.h
index 7181a32b9f1..8db88628ed9 100644
--- a/src/crimson/os/seastore/transaction_manager.h
+++ b/src/crimson/os/seastore/transaction_manager.h
@@ -65,7 +65,8 @@ public:
     CacheRef cache,
     LBAManagerRef lba_manager,
     ExtentPlacementManagerRef &&epm,
-    BackrefManagerRef&& backref_manager);
+    BackrefManagerRef&& backref_manager,
+    shard_stats_t& shard_stats);
 
   /// Writes initial metadata to disk
   using mkfs_ertr = base_ertr;
@@ -662,6 +663,10 @@ public:
    * ExtentCallbackInterface
    */
 
+  shard_stats_t& get_shard_stats() {
+    return shard_stats;
+  }
+
   /// weak transaction should be type READ
   TransactionRef create_transaction(
       Transaction::src_t src,
@@ -834,6 +839,8 @@ private:
 
   bool full_extent_integrity_check = true;
 
+  shard_stats_t& shard_stats;
+
   rewrite_extent_ret rewrite_logical_extent(
     Transaction& t,
     LogicalCachedExtentRef extent);
@@ -1010,5 +1017,6 @@ using TransactionManagerRef = std::unique_ptr<TransactionManager>;
 TransactionManagerRef make_transaction_manager(
     Device *primary_device,
     const std::vector<Device*> &secondary_devices,
+    shard_stats_t& shard_stats,
     bool is_test);
 }
diff --git a/src/crimson/tools/store_nbd/tm_driver.cc b/src/crimson/tools/store_nbd/tm_driver.cc
index 967a46ccd45..078e33bf8c4 100644
--- a/src/crimson/tools/store_nbd/tm_driver.cc
+++ b/src/crimson/tools/store_nbd/tm_driver.cc
@@ -139,11 +139,13 @@ seastar::future<bufferlist> TMDriver::read(
 
 void TMDriver::init()
 {
+  shard_stats = {};
+
   std::vector<Device*> sec_devices;
 #ifndef NDEBUG
-  tm = make_transaction_manager(device.get(), sec_devices, true);
+  tm = make_transaction_manager(device.get(), sec_devices, shard_stats, true);
 #else
-  tm = make_transaction_manager(device.get(), sec_devices, false);
+  tm = make_transaction_manager(device.get(), sec_devices, shard_stats, false);
 #endif
 }
 
diff --git a/src/crimson/tools/store_nbd/tm_driver.h b/src/crimson/tools/store_nbd/tm_driver.h
index 24aabdeb603..6433c050e44 100644
--- a/src/crimson/tools/store_nbd/tm_driver.h
+++ b/src/crimson/tools/store_nbd/tm_driver.h
@@ -41,6 +41,9 @@ private:
   using TransactionManagerRef = crimson::os::seastore::TransactionManagerRef;
   TransactionManagerRef tm;
 
+  using shard_stats_t = crimson::os::seastore::shard_stats_t;
+  shard_stats_t shard_stats;
+
   seastar::future<> mkfs();
   void init();
   void clear();
diff --git a/src/test/crimson/seastore/transaction_manager_test_state.h b/src/test/crimson/seastore/transaction_manager_test_state.h
index 451d795a317..f339b2baf69 100644
--- a/src/test/crimson/seastore/transaction_manager_test_state.h
+++ b/src/test/crimson/seastore/transaction_manager_test_state.h
@@ -273,6 +273,7 @@ protected:
   Cache* cache;
   ExtentPlacementManager *epm;
   uint64_t seq = 0;
+  shard_stats_t shard_stats;
 
   TMTestState() : EphemeralTestState(1, 0) {}
 
@@ -292,7 +293,8 @@ protected:
 	"seastore_full_integrity_check", "false");
     }
 #endif
-    tm = make_transaction_manager(p_dev, sec_devices, true);
+    shard_stats = {};
+    tm = make_transaction_manager(p_dev, sec_devices, shard_stats, true);
     epm = tm->get_epm();
     lba_manager = tm->get_lba_manager();
     cache = tm->get_cache();