osd: account for used kb separately

Account for used separately from available, as the underlying fs may do tricks
reserving space for root and such.

4 files changed, 21 insertions, 10 deletions

diff --git a/src/mon/PGMap.h b/src/mon/PGMap.h
index e1d43bda995..68fb28d033e 100644
--- a/src/mon/PGMap.h
+++ b/src/mon/PGMap.h
@@ -104,6 +104,7 @@ public:
   int64_t total_pg_num_objects;
   int64_t num_osd;
   int64_t total_osd_kb;
+  int64_t total_osd_kb_used;
   int64_t total_osd_kb_avail;
   int64_t total_osd_num_objects;
 
@@ -117,6 +118,7 @@ public:
     total_pg_num_objects = 0;
     num_osd = 0;
     total_osd_kb = 0;
+    total_osd_kb_used = 0;
     total_osd_kb_avail = 0;
     total_osd_num_objects = 0;
   }
@@ -142,19 +144,21 @@ public:
   void stat_osd_add(osd_stat_t &s) {
     num_osd++;
     total_osd_kb += s.kb;
+    total_osd_kb_used += s.kb_used;
     total_osd_kb_avail += s.kb_avail;
     total_osd_num_objects += s.num_objects;
   }
   void stat_osd_sub(osd_stat_t &s) {
     num_osd--;
     total_osd_kb -= s.kb;
+    total_osd_kb_used -= s.kb_used;
     total_osd_kb_avail -= s.kb_avail;
     total_osd_num_objects -= s.num_objects;
   }
 
   uint64_t total_kb() { return total_osd_kb; }
   uint64_t total_avail_kb() { return total_osd_kb_avail; }
-  uint64_t total_used_kb() { return total_kb() - total_avail_kb(); }
+  uint64_t total_used_kb() { return total_osd_kb_used; }
 
   PGMap() : version(0),
 	    last_osdmap_epoch(0), last_pg_scan(0),
@@ -164,6 +168,7 @@ public:
 	    total_pg_num_objects(0), 
 	    num_osd(0),
 	    total_osd_kb(0),
+	    total_osd_kb_used(0),
 	    total_osd_kb_avail(0),
 	    total_osd_num_objects(0) {}
 
diff --git a/src/mon/PGMonitor.cc b/src/mon/PGMonitor.cc
index 44f37ee1a5e..ae7c20208a3 100644
--- a/src/mon/PGMonitor.cc
+++ b/src/mon/PGMonitor.cc
@@ -45,13 +45,14 @@ struct kb_t {
 };
 ostream& operator<<(ostream& out, const kb_t& kb)
 {
-  if (kb.v > 10ull*1048ull*1024ull*1024ull*1024ULL)
+  __u64 bump_after = 100;
+  if (kb.v > bump_after << 40)
     return out << (kb.v >> 40) << " PB";    
-  if (kb.v > 10ull*1048ull*1024ull*1024ULL)
+  if (kb.v > bump_after << 30)
     return out << (kb.v >> 30) << " TB";    
-  if (kb.v > 10ull*1048ull*1024ULL)
+  if (kb.v > bump_after << 20)
     return out << (kb.v >> 20) << " GB";    
-  if (kb.v > 10ull*1048ULL)
+  if (kb.v > bump_after << 10)
     return out << (kb.v >> 10) << " MB";
   return out << kb.v << " KB";
 }
@@ -270,10 +271,10 @@ bool PGMonitor::prepare_pg_stats(MPGStats *stats)
   
   // apply to live map too (screw consistency)
   if (pg_map.osd_stat.count(from)) {
-    dout(10) << " got " << stats->osd_stat << " (was " << pg_map.osd_stat[from] << ")" << dendl;
+    dout(10) << " got osd" << from << " " << stats->osd_stat << " (was " << pg_map.osd_stat[from] << ")" << dendl;
     pg_map.stat_osd_sub(pg_map.osd_stat[from]);
   } else {
-    dout(10) << " got " << stats->osd_stat << " (first report)" << dendl;
+    dout(10) << " got osd " << from << " " << stats->osd_stat << " (first report)" << dendl;
   }
   pg_map.osd_stat[from] = stats->osd_stat;
   pg_map.stat_osd_add(stats->osd_stat);
@@ -571,6 +572,7 @@ bool PGMonitor::preprocess_command(MMonCommand *m)
 	   p != pg_map.osd_stat.end();
 	   p++)
 	ss << p->first << "\t" << p->second.kb
+	   << "\t" << p->second.kb_used
 	   << "\t" << p->second.kb_avail 
 	   << "\t" << p->second.num_objects
 	   << std::endl;
diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index c010b5f2a7f..f2ba2435446 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -1164,6 +1164,7 @@ void OSD::send_pg_stats()
     struct statfs stbuf;
     store->statfs(&stbuf);
     m->osd_stat.kb = stbuf.f_blocks * stbuf.f_bsize / 1024;
+    m->osd_stat.kb_used = (stbuf.f_blocks - stbuf.f_bfree) * stbuf.f_bsize / 1024;
     m->osd_stat.kb_avail = stbuf.f_bavail * stbuf.f_bsize / 1024;
     m->osd_stat.num_objects = stbuf.f_files;
     dout(20) << " osd_stat " << m->osd_stat << dendl;
diff --git a/src/osd/osd_types.h b/src/osd/osd_types.h
index a22685a7530..08fe680751f 100644
--- a/src/osd/osd_types.h
+++ b/src/osd/osd_types.h
@@ -244,18 +244,20 @@ inline ostream& operator<<(ostream& out, const eversion_t e) {
  */
 struct osd_stat_t {
   int64_t kb;
-  int64_t kb_avail;
+  int64_t kb_used, kb_avail;
   int64_t num_objects;
 
-  osd_stat_t() : kb(0), kb_avail(0), num_objects(0) {}
+  osd_stat_t() : kb(0), kb_used(0), kb_avail(0), num_objects(0) {}
 
   void encode(bufferlist &bl) const {
     ::encode(kb, bl);
+    ::encode(kb_used, bl);
     ::encode(kb_avail, bl);
     ::encode(num_objects, bl);
   }
   void decode(bufferlist::iterator &bl) {
     ::decode(kb, bl);
+    ::decode(kb_used, bl);
     ::decode(kb_avail, bl);
     ::decode(num_objects, bl);
   }
@@ -264,7 +266,8 @@ WRITE_CLASS_ENCODER(osd_stat_t)
 
 
 inline ostream& operator<<(ostream& out, const osd_stat_t& s) {
-  return out << "osd_stat(" << (s.kb-s.kb_avail) << "/" << s.kb << " KB used, " 
+  return out << "osd_stat(" << (s.kb_used) << "/" << s.kb << " KB used, " 
+	     << s.kb_avail << " avail, "
 	     << s.num_objects << " objects)";
 }