crush: compile/decompile crush_choose_arg_map

A map of crush_choose_arg_map is added to the crushmap text syntax. The
key is an integer matching a pool number.

Signed-off-by: Loic Dachary <loic@dachary.org>

8 files changed, 445 insertions, 4 deletions

diff --git a/src/crush/CrushCompiler.cc b/src/crush/CrushCompiler.cc
index 8957ed7bd84..4e0df956b65 100644
--- a/src/crush/CrushCompiler.cc
+++ b/src/crush/CrushCompiler.cc
@@ -215,6 +215,89 @@ int CrushCompiler::decompile_bucket(int cur,
   return 0;
 }
 
+int CrushCompiler::decompile_weight_set_weights(crush_weight_set weight_set,
+                                                ostream &out)
+{
+  out << "      [ ";
+  for (__u32 i = 0; i < weight_set.size; i++) {
+    print_fixedpoint(out, weight_set.weights[i]);
+    out << " ";
+  }
+  out << "]\n";
+  return 0;
+}
+
+int CrushCompiler::decompile_weight_set(crush_weight_set *weight_set,
+                                        __u32 size,
+                                        ostream &out)
+{
+  out << "    weight_set [\n";
+  for (__u32 i = 0; i < size; i++) {
+    int r = decompile_weight_set_weights(weight_set[i], out);
+    if (r < 0)
+      return r;
+  }
+  out << "    ]\n";
+  return 0;
+}
+
+int CrushCompiler::decompile_ids(int *ids,
+                                 __u32 size,
+                                 ostream &out)
+{
+  out << "    ids [ ";
+  for (__u32 i = 0; i < size; i++)
+    out << ids[i] << " ";
+  out << "]\n";
+  return 0;
+}
+
+int CrushCompiler::decompile_choose_arg(crush_choose_arg *arg,
+                                        int bucket_id,
+                                        ostream &out)
+{
+  int r;
+  out << "  {\n";
+  out << "    bucket_id " << bucket_id << "\n";
+  if (arg->weight_set_size > 0) {
+    r = decompile_weight_set(arg->weight_set, arg->weight_set_size, out);
+    if (r < 0)
+      return r;
+  }
+  if (arg->ids_size > 0) {
+    r = decompile_ids(arg->ids, arg->ids_size, out);
+    if (r < 0)
+      return r;
+  }
+  out << "  }\n";
+  return 0;
+}
+
+int CrushCompiler::decompile_choose_arg_map(crush_choose_arg_map arg_map,
+                                            ostream &out)
+{
+  for (__u32 i = 0; i < arg_map.size; i++) {
+    if ((arg_map.args[i].ids_size == 0) &&
+        (arg_map.args[i].weight_set_size == 0))
+      continue;
+    int r = decompile_choose_arg(&arg_map.args[i], -1-i, out);
+    if (r < 0)
+      return r;
+  }
+  return 0;
+}
+
+int CrushCompiler::decompile_choose_args(std::pair<const long unsigned int, crush_choose_arg_map> &i,
+                                         ostream &out)
+{
+  out << "choose_args " << i.first << " {\n";
+  int r = decompile_choose_arg_map(i.second, out);
+  if (r < 0)
+    return r;
+  out << "}\n";
+  return 0;
+}
+
 int CrushCompiler::decompile(ostream &out)
 {
   crush.cleanup_classes();
@@ -373,6 +456,14 @@ int CrushCompiler::decompile(ostream &out)
     }
     out << "}\n";
   }
+  if (crush.choose_args.size() > 0) {
+    out << "\n# choose_args\n";
+    for (auto i : crush.choose_args) {
+      int ret = decompile_choose_args(i, out);
+      if (ret)
+        return ret;
+    }
+  }
   out << "\n# end crush map" << std::endl;
   return 0;
 }
@@ -836,6 +927,121 @@ int CrushCompiler::parse_rule(iter_t const& i)
   return 0;
 }
 
+int CrushCompiler::parse_weight_set_weights(iter_t const& i, int bucket_id, crush_weight_set *weight_set)
+{
+  // -2 for the enclosing [ ]
+  __u32 size = i->children.size() - 2;
+  __u32 bucket_size = crush.get_bucket_size(bucket_id);
+  if (size != bucket_size) {
+    err << bucket_id << " needs exactly " << bucket_size
+        << " weights but got " << size << std::endl;
+    return -1;
+  }
+  weight_set->size = size;
+  weight_set->weights = (__u32 *)calloc(weight_set->size, sizeof(__u32));
+  __u32 pos = 0;
+  for (iter_t p = i->children.begin() + 1; p != i->children.end(); p++, pos++)
+    if (pos < size)
+      weight_set->weights[pos] = float_node(*p) * (float)0x10000;
+  return 0;
+}
+
+int CrushCompiler::parse_weight_set(iter_t const& i, int bucket_id, crush_choose_arg *arg)
+{
+  // -3 stands for the leading "weight_set" keyword and the enclosing [ ]
+  arg->weight_set_size = i->children.size() - 3;
+  arg->weight_set = (crush_weight_set *)calloc(arg->weight_set_size, sizeof(crush_weight_set));
+  __u32 pos = 0;
+  for (iter_t p = i->children.begin(); p != i->children.end(); p++) {
+    int r = 0;
+    switch((int)p->value.id().to_long()) {
+    case crush_grammar::_weight_set_weights:
+      if (pos < arg->weight_set_size) {
+        r = parse_weight_set_weights(p, bucket_id, &arg->weight_set[pos]);
+        pos++;
+      } else {
+        err << "invalid weight_set syntax" << std::endl;
+        r = -1;
+      }
+    }
+    if (r < 0)
+      return r;
+  }
+  return 0;
+}
+
+int CrushCompiler::parse_choose_arg_ids(iter_t const& i, int bucket_id, crush_choose_arg *arg)
+{
+  // -3 for the leading "ids" keyword and the enclosing [ ]
+  __u32 size = i->children.size() - 3;
+  __u32 bucket_size = crush.get_bucket_size(bucket_id);
+  if (size != bucket_size) {
+    err << bucket_id << " needs exactly " << bucket_size
+        << " ids but got " << size << std::endl;
+    return -1;
+  }
+  arg->ids_size = size;
+  arg->ids = (int *)calloc(arg->ids_size, sizeof(int));
+  __u32 pos = 0;
+  for (iter_t p = i->children.begin() + 2; pos < size; p++, pos++)
+    arg->ids[pos] = int_node(*p);
+  return 0;
+}
+
+int CrushCompiler::parse_choose_arg(iter_t const& i, crush_choose_arg *args)
+{
+  int bucket_id = int_node(i->children[2]);
+  if (-1-bucket_id < 0 || -1-bucket_id >= crush.get_max_buckets()) {
+    err << bucket_id << " is out of range" << std::endl;
+    return -1;
+  }
+  if (!crush.bucket_exists(bucket_id)) {
+    err << bucket_id << " does not exist" << std::endl;
+    return -1;
+  }
+  crush_choose_arg *arg = &args[-1-bucket_id];
+  for (iter_t p = i->children.begin(); p != i->children.end(); p++) {
+    int r = 0;
+    switch((int)p->value.id().to_long()) {
+    case crush_grammar::_weight_set:
+      r = parse_weight_set(p, bucket_id, arg);
+      break;
+    case crush_grammar::_choose_arg_ids:
+      r = parse_choose_arg_ids(p, bucket_id, arg);
+      break;
+    }
+    if (r < 0)
+      return r;
+  }
+  return 0;
+}
+
+int CrushCompiler::parse_choose_args(iter_t const& i)
+{
+  int choose_arg_index = int_node(i->children[1]);
+  if (crush.choose_args.find(choose_arg_index) != crush.choose_args.end()) {
+    err << choose_arg_index << " duplicated" << std::endl;
+    return -1;
+  }
+  crush_choose_arg_map arg_map;
+  arg_map.size = crush.get_max_buckets();
+  arg_map.args = (crush_choose_arg *)calloc(arg_map.size, sizeof(crush_choose_arg));
+  for (iter_t p = i->children.begin() + 2; p != i->children.end(); p++) {
+    int r = 0;
+    switch((int)p->value.id().to_long()) {
+    case crush_grammar::_choose_arg:
+      r = parse_choose_arg(p, arg_map.args);
+      break;
+    }
+    if (r < 0) {
+      crush.destroy_choose_args(arg_map);
+      return r;
+    }
+  }
+  crush.choose_args[choose_arg_index] = arg_map;
+  return 0;
+}
+
 void CrushCompiler::find_used_bucket_ids(iter_t const& i)
 {
   for (iter_t p = i->children.begin(); p != i->children.end(); p++) {
@@ -873,6 +1079,9 @@ int CrushCompiler::parse_crush(iter_t const& i)
     case crush_grammar::_crushrule: 
       r = parse_rule(p);
       break;
+    case crush_grammar::_choose_args:
+      r = parse_choose_args(p);
+      break;
     default:
       ceph_abort();
     }
@@ -884,7 +1093,7 @@ int CrushCompiler::parse_crush(iter_t const& i)
   //err << "max_devices " << crush.get_max_devices() << std::endl;
   crush.cleanup_classes();
   crush.finalize();
-  
+
   return 0;
 } 
 
diff --git a/src/crush/CrushCompiler.h b/src/crush/CrushCompiler.h
index 4e1542f5b09..3a93085597c 100644
--- a/src/crush/CrushCompiler.h
+++ b/src/crush/CrushCompiler.h
@@ -23,6 +23,21 @@ class CrushCompiler {
     DCB_STATE_DONE
   };
 
+  int decompile_weight_set_weights(crush_weight_set weight_set,
+				   ostream &out);
+  int decompile_weight_set(crush_weight_set *weight_set,
+			   __u32 size,
+			   ostream &out);
+  int decompile_choose_arg(crush_choose_arg *arg,
+			   int bucket_id,
+			   ostream &out);
+  int decompile_ids(int *ids,
+		    __u32 size,
+		    ostream &out);
+  int decompile_choose_arg_map(crush_choose_arg_map arg_map,
+			       ostream &out);
+  int decompile_choose_args(std::pair<const long unsigned int, crush_choose_arg_map> &i,
+			    ostream &out);
   int decompile_bucket_impl(int i, ostream &out);
   int decompile_bucket(int cur,
 		       std::map<int, dcb_state_t>& dcb_states,
@@ -49,6 +64,11 @@ class CrushCompiler {
   int parse_bucket_type(iter_t const& i);
   int parse_bucket(iter_t const& i);
   int parse_rule(iter_t const& i);
+  int parse_weight_set_weights(iter_t const& i, int bucket_id, crush_weight_set *weight_set);
+  int parse_weight_set(iter_t const& i, int bucket_id, crush_choose_arg *arg);
+  int parse_choose_arg_ids(iter_t const& i, int bucket_id, crush_choose_arg *args);
+  int parse_choose_arg(iter_t const& i, crush_choose_arg *args);
+  int parse_choose_args(iter_t const& i);
   void find_used_bucket_ids(iter_t const& i);
   int parse_crush(iter_t const& i);  
   void dump(iter_t const& i, int ind=1);
diff --git a/src/crush/CrushWrapper.cc b/src/crush/CrushWrapper.cc
index f1093070538..45f95f7a5c8 100644
--- a/src/crush/CrushWrapper.cc
+++ b/src/crush/CrushWrapper.cc
@@ -1440,6 +1440,38 @@ void CrushWrapper::encode(bufferlist& bl, uint64_t features) const
     ::encode(class_map, bl);
     ::encode(class_name, bl);
     ::encode(class_bucket, bl);
+
+    ::encode(choose_args.size(), bl);
+    for (auto c : choose_args) {
+      ::encode(c.first, bl);
+      crush_choose_arg_map arg_map = c.second;
+      __u32 size = 0;
+      for (__u32 i = 0; i < arg_map.size; i++) {
+	crush_choose_arg *arg = &arg_map.args[i];
+	if (arg->weight_set_size == 0 &&
+	    arg->ids_size == 0)
+	  continue;
+	size++;
+      }
+      ::encode(size, bl);
+      for (__u32 i = 0; i < arg_map.size; i++) {
+	crush_choose_arg *arg = &arg_map.args[i];
+	if (arg->weight_set_size == 0 &&
+	    arg->ids_size == 0)
+	  continue;
+	::encode(i, bl);
+	::encode(arg->weight_set_size, bl);
+	for (__u32 j = 0; j < arg->weight_set_size; j++) {
+	  crush_weight_set *weight_set = &arg->weight_set[j];
+	  ::encode(weight_set->size, bl);
+	  for (__u32 k = 0; k < weight_set->size; k++)
+	    ::encode(weight_set->weights[k], bl);
+	}
+	::encode(arg->ids_size, bl);
+	for (__u32 j = 0; j < arg->ids_size; j++)
+	  ::encode(arg->ids[j], bl);
+      }
+    }
   }
 }
 
@@ -1541,6 +1573,39 @@ void CrushWrapper::decode(bufferlist::iterator& blp)
       ::decode(class_bucket, blp);
       cleanup_classes();
     }
+    if (!blp.end()) {
+      size_t choose_args_size;
+      ::decode(choose_args_size, blp);
+      for (size_t i = 0; i < choose_args_size; i++) {
+	uint64_t choose_args_index;
+	::decode(choose_args_index, blp);
+	crush_choose_arg_map arg_map;
+	arg_map.size = crush->max_buckets;
+	arg_map.args = (crush_choose_arg*)calloc(arg_map.size, sizeof(crush_choose_arg));
+	__u32 size;
+	::decode(size, blp);
+	for (__u32 j = 0; j < size; j++) {
+	  __u32 bucket_index;
+	  ::decode(bucket_index, blp);
+	  assert(bucket_index < arg_map.size);
+	  crush_choose_arg *arg = &arg_map.args[bucket_index];
+	  ::decode(arg->weight_set_size, blp);
+	  arg->weight_set = (crush_weight_set*)calloc(arg->weight_set_size, sizeof(crush_weight_set));
+	  for (__u32 k = 0; k < arg->weight_set_size; k++) {
+	    crush_weight_set *weight_set = &arg->weight_set[k];
+	    ::decode(weight_set->size, blp);
+	    weight_set->weights = (__u32*)calloc(weight_set->size, sizeof(__u32));
+	    for (__u32 l = 0; l < weight_set->size; l++)
+	      ::decode(weight_set->weights[l], blp);
+	  }
+	  ::decode(arg->ids_size, blp);
+	  arg->ids = (int*)calloc(arg->ids_size, sizeof(int));
+	  for (__u32 k = 0; k < arg->ids_size; k++)
+	    ::decode(arg->ids[k], blp);
+	}
+	choose_args[choose_args_index] = arg_map;
+      }
+    }
     finalize();
   }
   catch (...) {
diff --git a/src/crush/CrushWrapper.h b/src/crush/CrushWrapper.h
index 40c70793b9f..773824e4eba 100644
--- a/src/crush/CrushWrapper.h
+++ b/src/crush/CrushWrapper.h
@@ -98,6 +98,7 @@ public:
     if (crush)
       crush_destroy(crush);
     crush = crush_create();
+    choose_args_clear();
     assert(crush);
     have_rmaps = false;
 
@@ -1101,7 +1102,6 @@ public:
   void finalize() {
     assert(crush);
     crush_finalize(crush);
-    choose_args_clear();
   }
 
   int update_device_class(CephContext *cct, int id, const string& class_name, const string& name);
diff --git a/src/crush/grammar.h b/src/crush/grammar.h
index 910e74d6f42..8fb28c877a4 100644
--- a/src/crush/grammar.h
+++ b/src/crush/grammar.h
@@ -55,6 +55,11 @@ struct crush_grammar : public grammar<crush_grammar>
     _step_emit,
     _step,
     _crushrule,
+    _weight_set_weights,
+    _weight_set,
+    _choose_arg_ids,
+    _choose_arg,
+    _choose_args,
     _crushmap,
     _tunable,
   };
@@ -91,6 +96,11 @@ struct crush_grammar : public grammar<crush_grammar>
     rule<ScannerT, parser_context<>, parser_tag<_step_emit> >      step_emit;
     rule<ScannerT, parser_context<>, parser_tag<_step> >      step;
     rule<ScannerT, parser_context<>, parser_tag<_crushrule> >      crushrule;
+    rule<ScannerT, parser_context<>, parser_tag<_weight_set_weights> >     weight_set_weights;
+    rule<ScannerT, parser_context<>, parser_tag<_weight_set> >     weight_set;
+    rule<ScannerT, parser_context<>, parser_tag<_choose_arg_ids> >     choose_arg_ids;
+    rule<ScannerT, parser_context<>, parser_tag<_choose_arg> >     choose_arg;
+    rule<ScannerT, parser_context<>, parser_tag<_choose_args> >     choose_args;
 
     rule<ScannerT, parser_context<>, parser_tag<_crushmap> >      crushmap;
 
@@ -158,8 +168,19 @@ struct crush_grammar : public grammar<crush_grammar>
 			   >> +step
 			   >> '}';
 
+      weight_set_weights = str_p("[") >> *real_p >> str_p("]");
+      weight_set = str_p("weight_set") >> str_p("[")
+				       >> *weight_set_weights
+				       >> str_p("]");
+      choose_arg_ids = str_p("ids") >> str_p("[") >> *integer >> str_p("]");
+      choose_arg = str_p("{") >> str_p("bucket_id") >> negint
+			      >> !weight_set
+			      >> !choose_arg_ids
+			      >> str_p("}");
+      choose_args = str_p("choose_args") >> posint >> str_p("{") >> *choose_arg >> str_p("}");
+
       // the whole crush map
-      crushmap = *(tunable | device | bucket_type) >> *(bucket | crushrule);
+      crushmap = *(tunable | device | bucket_type) >> *(bucket | crushrule) >> *choose_args;
     }
 
     rule<ScannerT, parser_context<>, parser_tag<_crushmap> > const&
diff --git a/src/test/cli/crushtool/choose-args.crush b/src/test/cli/crushtool/choose-args.crush
new file mode 100644
index 00000000000..2ed6c2315d0
--- /dev/null
+++ b/src/test/cli/crushtool/choose-args.crush
@@ -0,0 +1,120 @@
+# begin crush map
+
+# devices
+device 0 device0
+device 1 device1
+device 2 device2
+
+# types
+type 0 device
+type 1 host
+type 2 rack
+type 3 root
+
+# buckets
+host host0 {
+	id -1		# do not change unnecessarily
+	# weight 1.000
+	alg straw
+	hash 0	# rjenkins1
+	item device0 weight 1.000
+}
+host host1 {
+	id -2		# do not change unnecessarily
+	# weight 1.000
+	alg straw
+	hash 0	# rjenkins1
+	item device1 weight 1.000
+}
+host host2 {
+	id -5		# do not change unnecessarily
+	# weight 1.000
+	alg straw
+	hash 0	# rjenkins1
+	item device2 weight 1.000
+}
+rack rack0 {
+	id -3		# do not change unnecessarily
+	# weight 3.000
+	alg straw
+	hash 0	# rjenkins1
+	item host0 weight 1.000
+	item host1 weight 1.000
+	item host2 weight 1.000
+}
+root root {
+	id -4		# do not change unnecessarily
+	# weight 4.000
+	alg straw
+	hash 0	# rjenkins1
+	item rack0 weight 4.000
+}
+
+# rules
+rule data {
+	ruleset 3
+	type replicated
+	min_size 2
+	max_size 2
+	step take root
+	step chooseleaf firstn 0 type rack
+	step emit
+}
+
+# choose_args
+choose_args 1 {
+}
+choose_args 2 {
+  {
+    bucket_id -3
+    ids [ -20 30 -25 ]
+  }
+}
+choose_args 3 {
+  {
+    bucket_id -3
+    weight_set [
+      [ 1.000 2.000 5.000 ]
+      [ 3.000 2.000 5.000 ]
+    ]
+    ids [ -20 -30 -25 ]
+  }
+}
+choose_args 4 {
+  {
+    bucket_id -2
+    weight_set [
+      [ 1.000 ]
+      [ 3.000 ]
+    ]
+  }
+}
+choose_args 5 {
+  {
+    bucket_id -1
+    ids [ -450 ]
+  }
+}
+choose_args 6 {
+  {
+    bucket_id -1
+    ids [ -450 ]
+  }
+  {
+    bucket_id -2
+    weight_set [
+      [ 1.000 ]
+      [ 3.000 ]
+    ]
+  }
+  {
+    bucket_id -3
+    weight_set [
+      [ 1.000 2.000 5.000 ]
+      [ 3.000 2.000 5.000 ]
+    ]
+    ids [ -20 -30 -25 ]
+  }
+}
+
+# end crush map
diff --git a/src/test/cli/crushtool/choose-args.t b/src/test/cli/crushtool/choose-args.t
new file mode 100644
index 00000000000..0c7e3220441
--- /dev/null
+++ b/src/test/cli/crushtool/choose-args.t
@@ -0,0 +1,6 @@
+  $ cp "$TESTDIR/choose-args.crush" .
+  $ crushtool -c choose-args.crush -o choose-args.compiled
+  $ crushtool -d choose-args.compiled -o choose-args.conf
+  $ crushtool -c choose-args.conf -o choose-args.recompiled
+  $ cmp choose-args.crush choose-args.conf
+  $ cmp choose-args.compiled choose-args.recompiled
diff --git a/src/test/cli/osdmaptool/crush.t b/src/test/cli/osdmaptool/crush.t
index 048285df551..0cd9ae61968 100644
--- a/src/test/cli/osdmaptool/crush.t
+++ b/src/test/cli/osdmaptool/crush.t
@@ -6,5 +6,5 @@
   osdmaptool: exported crush map to oc
   $ osdmaptool --import-crush oc myosdmap
   osdmaptool: osdmap file 'myosdmap'
-  osdmaptool: imported 504 byte crush map from oc
+  osdmaptool: imported 512 byte crush map from oc
   osdmaptool: writing epoch 3 to myosdmap