diff options
Diffstat (limited to 'src/mds/ScrubStack.cc')
| -rw-r--r-- | src/mds/ScrubStack.cc | 183 |
1 files changed, 181 insertions, 2 deletions
diff --git a/src/mds/ScrubStack.cc b/src/mds/ScrubStack.cc index 28392f53366..7ec77a31de3 100644 --- a/src/mds/ScrubStack.cc +++ b/src/mds/ScrubStack.cc @@ -17,6 +17,7 @@ #include "mds/MDSRank.h" #include "mds/MDCache.h" #include "mds/MDSContinuation.h" +#include "osdc/Objecter.h" #define dout_context g_ceph_context #define dout_subsys ceph_subsys_mds @@ -75,6 +76,7 @@ int ScrubStack::_enqueue(MDSCacheObject *obj, ScrubHeaderRef& header, bool top) dout(10) << __func__ << " with {" << *in << "}" << ", top=" << top << dendl; in->scrub_initialize(header); + in->uninline_initialize(); } else if (CDir *dir = dynamic_cast<CDir*>(obj)) { if (dir->scrub_is_in_progress()) { dout(10) << __func__ << " with {" << *dir << "}" << ", already in scrubbing" << dendl; @@ -106,6 +108,55 @@ int ScrubStack::_enqueue(MDSCacheObject *obj, ScrubHeaderRef& header, bool top) return 0; } +void ScrubStack::purge_scrub_counters(std::string_view tag) +{ + for (auto& stat : mds_scrub_stats) { + if (tag == "all") { + stat.counters.clear(); + } else { + auto it = stat.counters.find(std::string(tag)); + if (it != stat.counters.end()) { + stat.counters.erase(it); + } + } + } +} + +// called from tick +void ScrubStack::purge_old_scrub_counters() +{ + // "mds_scrub_stats_review_period" must be in number of days + auto review_period = ceph::make_timespan(_mds_scrub_stats_review_period * 24 * 60 * 60); + auto now = coarse_real_clock::now(); + + dout(20) << __func__ << " review_period:" << review_period << dendl; + + for (mds_rank_t rank = 0; rank < (mds_rank_t)mds_scrub_stats.size(); rank++) { + auto& counters = mds_scrub_stats[rank].counters; + for (auto it = counters.begin(); it != counters.end(); ) { + auto curr = it; + auto c = (*it).second; + auto elapsed = now - c.start_time; + dout(20) << __func__ + << " rank(" << rank << ") :" + << " elapsed:" << elapsed + << dendl; + ++it; + if (elapsed >= review_period) { + counters.erase(curr); + } + } + } +} + +void ScrubStack::init_scrub_counters(std::string_view path, std::string_view tag) +{ + scrub_counters_t sc{coarse_real_clock::now(), std::string(path), 0, 0, 0}; + for (auto& stat : mds_scrub_stats) { + stat.counters[std::string(tag)] = sc; + } +} + int ScrubStack::enqueue(CInode *in, ScrubHeaderRef& header, bool top) { // abort in progress @@ -133,6 +184,10 @@ int ScrubStack::enqueue(CInode *in, ScrubHeaderRef& header, bool top) //to make sure mdsdir is always on the top top = false; } + + std::string path; + in->make_path_string(path); + init_scrub_counters(path, header->get_tag()); int r = _enqueue(in, header, top); if (r < 0) return r; @@ -227,6 +282,7 @@ void ScrubStack::kick_off_scrubs() // it's a regular file, symlink, or hard link dequeue(in); // we only touch it this once, so remove from stack + uninline_data(in, new C_MDSInternalNoop); scrub_file_inode(in); } else { bool added_children = false; @@ -235,6 +291,7 @@ void ScrubStack::kick_off_scrubs() if (done) { dout(20) << __func__ << " dir inode, done" << dendl; dequeue(in); + in->uninline_finished(); } if (added_children) { // dirfrags were queued at top of stack @@ -707,6 +764,43 @@ void ScrubStack::scrub_status(Formatter *f) { f->close_section(); // scrub id } f->close_section(); // scrubs + + if (mds_scrub_stats.size()) { + f->open_object_section("scrub_stats"); + for (auto& [tag, ctrs] : mds_scrub_stats[0].counters) { + uint64_t started = 0; + uint64_t passed = 0; + uint64_t failed = 0; + uint64_t skipped = 0; + for (auto& stats : mds_scrub_stats) { + if (auto it = stats.counters.find(tag); it != stats.counters.end()) { + auto& [t, c] = *it; + started += c.uninline_started; + passed += c.uninline_passed; + failed += c.uninline_failed; + skipped += c.uninline_skipped; + } + } + f->open_object_section(tag); + { + f->dump_stream("start_time") << ctrs.start_time; + std::string path = ctrs.origin_path; + if (path == "") { + path = "/"; + } else if (path.starts_with("~mds")) { + path = "~mdsdir"; + } + f->dump_string("path", path); + f->dump_int("uninline_started", started); + f->dump_int("uninline_passed", passed); + f->dump_int("uninline_failed", failed); + f->dump_int("uninline_skipped", skipped); + } + f->close_section(); // tag + } + f->close_section(); // scrub_stats + } + f->close_section(); // result } @@ -936,6 +1030,7 @@ void ScrubStack::handle_scrub(const cref_t<MMDSScrub> &m) header->set_origin(m->get_origin()); scrubbing_map.emplace(header->get_tag(), header); } + for (auto dir : dfs) { queued.insert_raw(dir->get_frag()); _enqueue(dir, header, true); @@ -1016,6 +1111,7 @@ void ScrubStack::handle_scrub(const cref_t<MMDSScrub> &m) const auto& header = in->get_scrub_header(); header->set_epoch_last_forwarded(scrub_epoch); in->scrub_finished(); + in->uninline_finished(); kick_off_scrubs(); } @@ -1052,6 +1148,10 @@ void ScrubStack::handle_scrub_stats(const cref_t<MMDSScrubStats> &m) bool any_finished = false; bool any_repaired = false; std::set<std::string> scrubbing_tags; + std::unordered_map<std::string, unordered_map<int, std::vector<_inodeno_t>>> uninline_failed_meta_info; + std::unordered_map<_inodeno_t, std::string> paths; + std::unordered_map<std::string, std::vector<uint64_t>> counters; + for (auto it = scrubbing_map.begin(); it != scrubbing_map.end(); ) { auto& header = it->second; if (header->get_num_pending() || @@ -1062,6 +1162,17 @@ void ScrubStack::handle_scrub_stats(const cref_t<MMDSScrubStats> &m) any_finished = true; if (header->get_repaired()) any_repaired = true; + auto& ufi = header->get_uninline_failed_info(); + uninline_failed_meta_info[it->first] = ufi; + ufi.clear(); + paths.merge(header->get_paths()); + ceph_assert(header->get_paths().size() == 0); + std::vector<uint64_t> c{header->get_uninline_started(), + header->get_uninline_passed(), + header->get_uninline_failed(), + header->get_uninline_skipped() + }; + counters[header->get_tag()] = c; scrubbing_map.erase(it++); } else { ++it; @@ -1071,7 +1182,11 @@ void ScrubStack::handle_scrub_stats(const cref_t<MMDSScrubStats> &m) scrub_epoch = m->get_epoch(); auto ack = make_message<MMDSScrubStats>(scrub_epoch, - std::move(scrubbing_tags), clear_stack); + std::move(scrubbing_tags), + std::move(uninline_failed_meta_info), + std::move(paths), + std::move(counters), + clear_stack); mdcache->mds->send_message_mds(ack, 0); if (any_finished) @@ -1085,7 +1200,40 @@ void ScrubStack::handle_scrub_stats(const cref_t<MMDSScrubStats> &m) stat.epoch_acked = m->get_epoch(); stat.scrubbing_tags = m->get_scrubbing_tags(); stat.aborting = m->is_aborting(); + for (auto& [scrub_tag, errno_map] : m->get_uninline_failed_meta_info()) { + stat.uninline_failed_meta_info[scrub_tag] = errno_map; + } + stat.paths.insert(m->get_paths().begin(), m->get_paths().end());; + for (auto& [tag, v] : m->get_counters()) { + stat.counters[tag].uninline_started = v[0]; + stat.counters[tag].uninline_passed = v[1]; + stat.counters[tag].uninline_failed = v[2]; + stat.counters[tag].uninline_skipped = v[3]; + } + } + } +} + +void ScrubStack::move_uninline_failures_to_damage_table() +{ + auto mds = mdcache->mds; + + for (mds_rank_t rank = 0; rank < (mds_rank_t)mds_scrub_stats.size(); rank++) { + auto& ufmi = mds_scrub_stats[rank].uninline_failed_meta_info; + auto& paths = mds_scrub_stats[rank].paths; + + for (const auto& [scrub_tag, errno_ino_vec_map] : ufmi) { + for (const auto& [errno_, ino_vec] : errno_ino_vec_map) { + for (auto ino : ino_vec) { + mds->damage_table.notify_uninline_failed(ino, rank, errno_, scrub_tag, paths[ino]); + } + } } + ufmi.clear(); + paths.clear(); + // do not clear the counters map; we'll clear them later: + // - on user request or + // - after a grace period } } @@ -1152,6 +1300,18 @@ void ScrubStack::advance_scrub_status() any_finished = true; if (header->get_repaired()) any_repaired = true; + auto& ufmi = mds_scrub_stats[0].uninline_failed_meta_info; + ufmi[it->first] = header->get_uninline_failed_info(); + mds_scrub_stats[0].paths.merge(header->get_paths()); + move_uninline_failures_to_damage_table(); + + auto& c = mds_scrub_stats[0].counters; + auto& sc = c[header->get_tag()]; + sc.uninline_started = header->get_uninline_started(); + sc.uninline_passed = header->get_uninline_passed(); + sc.uninline_failed = header->get_uninline_failed(); + sc.uninline_skipped = header->get_uninline_skipped(); + scrubbing_map.erase(it++); } else { ++it; @@ -1159,7 +1319,6 @@ void ScrubStack::advance_scrub_status() } ++scrub_epoch; - for (auto& r : up_mds) { if (r == 0) continue; @@ -1197,3 +1356,23 @@ void ScrubStack::handle_mds_failure(mds_rank_t mds) if (kick) kick_off_scrubs(); } + +void ScrubStack::uninline_data(CInode *in, Context *fin) +{ + dout(10) << "(uninline_data) starting data uninlining for " << *in << dendl; + + MDRequestRef mdr = in->mdcache->request_start_internal(CEPH_MDS_OP_UNINLINE_DATA); + mdr->set_filepath(filepath(in->ino())); + mdr->snapid = CEPH_NOSNAP; + mdr->no_early_reply = true; + mdr->internal_op_finish = fin; + + in->mdcache->dispatch_request(mdr); +} + +void ScrubStack::handle_conf_change(const std::set<std::string>& changed) +{ + if (changed.count("mds_scrub_stats_review_period")) { + _mds_scrub_stats_review_period = g_conf().get_val<uint64_t>("mds_scrub_stats_review_period"); + } +} |