common/perf_counters: use second atomic to make counters safe to read

Add a second counter so that we can detect a race with an add/inc during
read, and retry.

Signed-off-by: Sage Weil <sage@inktank.com>

1 files changed, 23 insertions, 6 deletions

diff --git a/src/common/perf_counters.h b/src/common/perf_counters.h
index f4f650771bc..78c8684a2a5 100644
--- a/src/common/perf_counters.h
+++ b/src/common/perf_counters.h
@@ -115,14 +115,18 @@ private:
       : name(NULL),
 	type(PERFCOUNTER_NONE),
 	u64(0),
-	avgcount(0)
+	avgcount(0),
+	avgcount2(0)
     {}
     perf_counter_data_any_d(const perf_counter_data_any_d& other)
       : name(other.name),
 	type(other.type),
-	u64(other.u64.read()),
-	avgcount(other.avgcount.read())
-    {}
+	u64(other.u64.read()) {
+      pair<uint64_t,uint64_t> a = other.read_avg();
+      u64.set(a.first);
+      avgcount.set(a.second);
+      avgcount2.set(a.second);
+    }
 
     void write_schema_json(char *buf, size_t buf_sz) const;
     void  write_json(char *buf, size_t buf_sz) const;
@@ -131,14 +135,27 @@ private:
     enum perfcounter_type_d type;
     atomic64_t u64;
     atomic64_t avgcount;
+    atomic64_t avgcount2;
 
     perf_counter_data_any_d& operator=(const perf_counter_data_any_d& other) {
       name = other.name;
       type = other.type;
-      u64.set(other.u64.read());
-      avgcount.set(other.avgcount.read());
+      pair<uint64_t,uint64_t> a = other.read_avg();
+      u64.set(a.first);
+      avgcount.set(a.second);
+      avgcount2.set(a.second);
       return *this;
     }
+
+    /// read <sum, count> safely
+    pair<uint64_t,uint64_t> read_avg() const {
+      uint64_t sum, count;
+      do {
+	count = avgcount.read();
+	sum = u64.read();
+      } while (avgcount2.read() != count);
+      return make_pair(sum, count);
+    }
   };
   typedef std::vector<perf_counter_data_any_d> perf_counter_data_vec_t;