With admin metadata rest API implementation and unit test cases for it

Signed-off-by: Babu Shanmugam <anbu@enovance.com>

9 files changed, 1154 insertions, 7 deletions

diff --git a/src/Makefile.am b/src/Makefile.am
index a45dd4e9830..dc11c9a5df8 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -412,6 +412,7 @@ radosgw_SOURCES = \
         rgw/rgw_rest_usage.cc \
         rgw/rgw_rest_user.cc \
         rgw/rgw_rest_bucket.cc \
+        rgw/rgw_rest_metadata.cc \
         rgw/rgw_http_client.cc \
         rgw/rgw_swift.cc \
 	rgw/rgw_swift_auth.cc \
@@ -934,6 +935,13 @@ ceph_test_cors_LDFLAGS = libglobal.la
 ceph_test_cors_LDADD = librados.la librgw.a ${UNITTEST_LDADD} ${UNITTEST_STATIC_LDADD} $(CRYPTO_LIBS) -lcurl -luuid -lexpat
 ceph_test_cors_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS}
 bin_DEBUGPROGRAMS += ceph_test_cors
+
+unittest_rgw_meta_SOURCES = test/test_rgw_admin_meta.cc
+unittest_rgw_meta_LDFLAGS = libglobal.la 
+unittest_rgw_meta_LDADD = librgw.a ${UNITTEST_LDADD} ${UNITTEST_STATIC_LDADD} -lcryptopp -lcurl -luuid -lexpat  librados.la libcls_version_client.a \
+ libcls_log_client.a libcls_refcount_client.a libcls_rgw_client.a libcls_lock_client.a
+unittest_rgw_meta_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS}
+check_PROGRAMS += unittest_rgw_meta
 endif
 
 ceph_test_librbd_SOURCES = test/librbd/test_librbd.cc test/librados/test.cc
@@ -2069,7 +2077,8 @@ noinst_HEADERS = \
 	rgw/rgw_rest_usage.h\
 	rgw/rgw_rest_user.h\
 	rgw/rgw_rest_bucket.h\
-        rgw/rgw_tools.h\
+	rgw/rgw_tools.h\
+	rgw/rgw_rest_metadata.h\
 	rgw/rgw_usage.h\
 	rgw/rgw_user.h\
 	rgw/rgw_bucket.h\
diff --git a/src/common/ceph_json.cc b/src/common/ceph_json.cc
index ad3d70846d2..502ffb58e70 100644
--- a/src/common/ceph_json.cc
+++ b/src/common/ceph_json.cc
@@ -468,17 +468,17 @@ void encode_json(const char *name, long val, Formatter *f)
 
 void encode_json(const char *name, unsigned val, Formatter *f)
 {
-  f->dump_int(name, val);
+  f->dump_unsigned(name, val);
 }
 
 void encode_json(const char *name, unsigned long val, Formatter *f)
 {
-  f->dump_int(name, val);
+  f->dump_unsigned(name, val);
 }
 
 void encode_json(const char *name, unsigned long long val, Formatter *f)
 {
-  f->dump_int(name, val);
+  f->dump_unsigned(name, val);
 }
 
 void encode_json(const char *name, long long val, Formatter *f)
diff --git a/src/common/ceph_json.h b/src/common/ceph_json.h
index 8085a7ca6cc..7be8ce5bca1 100644
--- a/src/common/ceph_json.h
+++ b/src/common/ceph_json.h
@@ -257,6 +257,7 @@ void encode_json(const char *name, int val, Formatter *f);
 void encode_json(const char *name, unsigned val, Formatter *f);
 void encode_json(const char *name, long val, Formatter *f);
 void encode_json(const char *name, unsigned long val, Formatter *f);
+void encode_json(const char *name, long long val, Formatter *f);
 void encode_json(const char *name, const utime_t& val, Formatter *f);
 void encode_json(const char *name, const bufferlist& bl, Formatter *f);
 void encode_json(const char *name, long long val, Formatter *f);
diff --git a/src/rgw/rgw_cache.h b/src/rgw/rgw_cache.h
index e32a921d1c6..6514a6171de 100644
--- a/src/rgw/rgw_cache.h
+++ b/src/rgw/rgw_cache.h
@@ -57,7 +57,7 @@ struct ObjectCacheInfo {
   ObjectMetaInfo meta;
   obj_version version;
 
-  ObjectCacheInfo() : status(0), flags(0), epoch(0) {}
+  ObjectCacheInfo() : status(0), flags(0), epoch(0), version() {}
 
   void encode(bufferlist& bl) const {
     ENCODE_START(5, 3, bl);
@@ -550,7 +550,7 @@ int RGWCache<T>::watch_cb(int opcode, uint64_t ver, bufferlist& bl)
   string oid;
   normalize_bucket_and_obj(info.obj.bucket, info.obj.object, bucket, oid);
   string name = normal_name(bucket, oid);
-
+  
   switch (info.op) {
   case UPDATE_OBJ:
     cache.put(name, info.obj_info);
diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h
index 1d8c32da98c..3ab0c97ec4f 100644
--- a/src/rgw/rgw_common.h
+++ b/src/rgw/rgw_common.h
@@ -234,7 +234,9 @@ class XMLArgs
     return (iter != sub_resources.end());
   }
   map<string, string>& get_sub_resources() { return sub_resources; }
-
+  unsigned get_num_params() {
+    return val_map.size();
+  }
   bool has_response_modifier() {
     return has_resp_modifier;
   }
@@ -669,6 +671,7 @@ struct req_state {
    rgw_bucket bucket;
    string bucket_name_str;
    string object_str;
+   string meta_section_str;
    ACLOwner bucket_owner;
    ACLOwner owner;
 
diff --git a/src/rgw/rgw_main.cc b/src/rgw/rgw_main.cc
index 57c6c5781ce..bcf740d32b1 100644
--- a/src/rgw/rgw_main.cc
+++ b/src/rgw/rgw_main.cc
@@ -40,6 +40,7 @@
 #include "rgw_rest_usage.h"
 #include "rgw_rest_user.h"
 #include "rgw_rest_bucket.h"
+#include "rgw_rest_metadata.h"
 #include "rgw_swift_auth.h"
 #include "rgw_swift.h"
 #include "rgw_log.h"
@@ -496,7 +497,11 @@ int main(int argc, const char **argv)
     admin_resource->register_resource("usage", new RGWRESTMgr_Usage);
     admin_resource->register_resource("user", new RGWRESTMgr_User);
     admin_resource->register_resource("bucket", new RGWRESTMgr_Bucket);
+  
+    /*Registering resource for /admin/metadata */
+    admin_resource->register_resource("metadata", new RGWRESTMgr_Metadata);
     rest.register_resource(g_conf->rgw_admin_entry, admin_resource);
+    rgw_user_init(store->meta_mgr);
   }
 
   OpsLogSocket *olog = NULL;
diff --git a/src/rgw/rgw_rest_metadata.cc b/src/rgw/rgw_rest_metadata.cc
new file mode 100644
index 00000000000..8844835dacc
--- /dev/null
+++ b/src/rgw/rgw_rest_metadata.cc
@@ -0,0 +1,210 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 eNovance SAS <licensing@enovance.com>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+#include "rgw_rest.h"
+#include "rgw_op.h"
+#include "rgw_rest_s3.h"
+#include "rgw_rest_metadata.h"
+#include "rgw_client_io.h"
+#include "common/errno.h"
+#define dout_subsys ceph_subsys_rgw
+
+int RGWHandler_Metadata::init_from_header(struct req_state *s) {
+  const char *p;
+  const string prefix = "/admin/metadata";
+  string section, req;
+
+  p = s->request_params.c_str();
+  if(p[0] != '\0'){
+    s->args.set(p);
+    s->args.parse();
+  }
+
+  /*Allocate a JSON formatter*/
+  s->format = RGW_FORMAT_JSON;
+  s->formatter = new JSONFormatter(false);
+
+  /*Get the section name*/  
+  if (s->decoded_uri.compare(0, prefix.length(), prefix) != 0) {
+    /*This is a safety check, may never happen*/
+    return -EINVAL;
+  } else {
+    unsigned start_off = prefix.length();
+
+    start_off += (s->decoded_uri.length() > prefix.length())?1:0;
+    s->meta_section_str.assign(s->decoded_uri, 
+                               start_off, 
+                               string::npos);
+  }
+  return 0;
+}
+
+int RGWHandler_Metadata::init(RGWRados *store, 
+                              struct req_state *state, 
+                              RGWClientIO *cio) {
+  int ret = init_from_header(state);
+  if (ret < 0) 
+    return ret;
+
+  return RGWHandler_ObjStore::init(store, state, cio);
+}
+
+int RGWHandler_Metadata::authorize() {
+  int ret = RGW_Auth_S3::authorize(store, s);
+  if (ret < 0) {
+    return ret;
+  }
+  /*TODO : check if the user is admin?*/
+  return 0;
+}
+
+const char *RGWOp_Metadata_Get::name() {
+  bool exists;
+
+  (void)s->args.get("key", &exists);
+  if (exists) {
+    return "get_metadata";
+  } else {
+    return "list_metadata";
+  }
+}
+
+static inline void frame_metadata_key(string& section, string& key, string& out) {
+  out = (section + string(":") + key);
+}
+
+void RGWOp_Metadata_Get::execute() {
+  bool exists;
+  string key = s->args.get("key", &exists);
+  unsigned num_params = s->args.get_num_params();
+  string metadata_key;
+
+  frame_metadata_key(s->meta_section_str, key, metadata_key);
+  if ((num_params > 0) && !exists) {
+    /*Invalid parameter*/
+    http_ret = -EINVAL;
+    return;
+  }
+  if (!exists) {
+    /* List keys */
+    void *handle;
+    int max = 1000;
+
+    http_ret = store->meta_mgr->list_keys_init(metadata_key, &handle);
+    if (http_ret < 0) {
+      dout(5) << "ERROR: can't get key: " << cpp_strerror(http_ret) << dendl;
+      return;
+    }
+
+    bool truncated;
+
+    s->formatter->open_array_section("keys");
+
+    do {
+      list<string> keys;
+      http_ret = store->meta_mgr->list_keys_next(handle, max, keys, &truncated);
+      if (http_ret < 0) {
+        dout(5) << "ERROR: lists_keys_next(): " << cpp_strerror(http_ret) << dendl;
+        return;
+      }
+
+      for (list<string>::iterator iter = keys.begin(); iter != keys.end(); ++iter) {
+        s->formatter->dump_string("key", *iter);
+      }
+
+    } while (truncated);
+
+    s->formatter->close_section();
+
+    store->meta_mgr->list_keys_complete(handle);
+  } else {
+    /* Get keys */
+    http_ret = store->meta_mgr->get(metadata_key, s->formatter);
+    if (http_ret < 0) {
+      dout(5) << "ERROR: can't get key: " << cpp_strerror(http_ret) << dendl;
+      return;
+    }
+  }
+
+  http_ret = 0;
+}
+
+int RGWOp_Metadata_Put::get_data(bufferlist& bl) {
+  size_t cl = 0;
+  if (s->length)
+    cl = atoll(s->length);
+  if (cl) {
+    char *data = (char *)malloc(cl + 1);
+    if (!data) {
+       return -ENOMEM;
+    }
+    int read_len;
+    int r = s->cio->read(data, cl, &read_len);
+    if (cl != (size_t)read_len) {
+      dout(10) << "cio->read incomplete" << dendl;
+    }
+    if (r < 0) {
+      free(data);
+      return r;
+    }
+    bl.append(data, read_len);
+    free(data);
+  }
+
+  return 0;
+}
+
+void RGWOp_Metadata_Put::execute() {
+  bufferlist bl;
+  string key = s->args.get("key");
+  string metadata_key;
+
+  http_ret = get_data(bl);
+  if (http_ret < 0) {
+    return;
+  }
+  
+  frame_metadata_key(s->meta_section_str, key, metadata_key);
+  
+  http_ret = store->meta_mgr->put(metadata_key, bl);
+  if (http_ret < 0) {
+    dout(5) << "ERROR: can't put key: " << cpp_strerror(http_ret) << dendl;
+    return;
+  }
+  http_ret = 0;
+}
+
+void RGWOp_Metadata_Delete::execute() {
+  string key = s->args.get("key");
+  string metadata_key;
+
+  frame_metadata_key(s->meta_section_str, key, metadata_key);
+  http_ret = store->meta_mgr->remove(metadata_key);
+  if (http_ret < 0) {
+    dout(5) << "ERROR: can't remove key: " << cpp_strerror(http_ret) << dendl;
+    return;
+  }
+  http_ret = 0;
+}
+
+RGWOp *RGWHandler_Metadata::op_get() {
+  return new RGWOp_Metadata_Get;
+}
+
+RGWOp *RGWHandler_Metadata::op_put() {
+  return new RGWOp_Metadata_Put;
+}
+
+RGWOp *RGWHandler_Metadata::op_delete() {
+  return new RGWOp_Metadata_Delete;
+}
diff --git a/src/rgw/rgw_rest_metadata.h b/src/rgw/rgw_rest_metadata.h
new file mode 100644
index 00000000000..f83cc9c288f
--- /dev/null
+++ b/src/rgw/rgw_rest_metadata.h
@@ -0,0 +1,82 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 eNovance SAS <licensing@enovance.com>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+#ifndef CEPH_RGW_REST_METADATA_H
+#define CEPH_RGW_REST_METADATA_H
+
+class RGWOp_Metadata_Get : public RGWRESTOp {
+public:
+  RGWOp_Metadata_Get() {}
+  ~RGWOp_Metadata_Get() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("metadata", RGW_CAP_READ);
+  }
+  void execute();
+  virtual const char *name();
+};
+
+class RGWOp_Metadata_Put : public RGWRESTOp {
+  int get_data(bufferlist& bl);
+public:
+  RGWOp_Metadata_Put() {}
+  ~RGWOp_Metadata_Put() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("metadata", RGW_CAP_WRITE);
+  }
+  void execute();
+  virtual const char *name() { return "set_metadata"; }
+};
+
+class RGWOp_Metadata_Delete : public RGWRESTOp {
+public:
+  RGWOp_Metadata_Delete() {}
+  ~RGWOp_Metadata_Delete() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("metadata", RGW_CAP_WRITE);
+  }
+  void execute();
+  virtual const char *name() { return "remove_metadata"; }
+};
+
+class RGWHandler_Metadata : public RGWHandler_ObjStore {
+protected:
+  int init_from_header(struct req_state *state);
+  RGWOp *op_get();
+  RGWOp *op_put();
+  RGWOp *op_delete();
+public:
+  RGWHandler_Metadata() : RGWHandler_ObjStore() {}
+  virtual ~RGWHandler_Metadata() {}
+
+  virtual int validate_bucket_name(const std::string& bucket) { return 0; }
+  virtual int validate_object_name(const std::string& object) { return 0; }
+
+  virtual int init(RGWRados *store, struct req_state *state, RGWClientIO *cio);
+  virtual int authorize();
+};
+
+class RGWRESTMgr_Metadata : public RGWRESTMgr {
+public:
+  RGWRESTMgr_Metadata() {}
+  virtual ~RGWRESTMgr_Metadata() {}
+
+  virtual RGWHandler *get_handler(struct req_state *s){
+    return new RGWHandler_Metadata;
+  }
+};
+
+
+#endif
diff --git a/src/test/test_rgw_admin_meta.cc b/src/test/test_rgw_admin_meta.cc
new file mode 100644
index 00000000000..53011950bd9
--- /dev/null
+++ b/src/test/test_rgw_admin_meta.cc
@@ -0,0 +1,837 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 eNovance SAS <licensing@enovance.com>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+#include <iostream>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <sys/wait.h>
+#include <unistd.h>
+#include <fstream>
+#include <map>
+#include <list>
+extern "C"{
+#include <curl/curl.h>
+}
+#include "common/ceph_crypto.h"
+#include "include/str_list.h"
+#include "common/ceph_json.h"
+#include "common/code_environment.h"
+#include "common/ceph_argparse.h"
+#include "common/Finisher.h"
+#include "global/global_init.h"
+#include "rgw/rgw_common.h"
+#include "rgw/rgw_rados.h"
+#define GTEST
+#ifdef GTEST
+#include <gtest/gtest.h>
+#else
+#define TEST(x, y) void y()
+#define ASSERT_EQ(v, s) if(v != s)cout << "Error at " << __LINE__ << "(" << #v << "!= " << #s << "\n"; \
+                                else cout << "(" << #v << "==" << #s << ") PASSED\n";
+#define EXPECT_EQ(v, s) ASSERT_EQ(v, s)
+#define ASSERT_TRUE(c) if(c)cout << "Error at " << __LINE__ << "(" << #c << ")" << "\n"; \
+                          else cout << "(" << #c << ") PASSED\n";
+#define EXPECT_TRUE(c) ASSERT_TRUE(c) 
+#endif
+using namespace std;
+
+#define CURL_VERBOSE 0
+#define HTTP_RESPONSE_STR "RespCode"
+#define CEPH_CRYPTO_HMACSHA1_DIGESTSIZE 20
+#define RGW_ADMIN_RESP_PATH "/tmp/.test_rgw_admin_resp"
+
+static string uid = "ceph";
+static string display_name = "CEPH";
+static string meta_caps = "metadata";
+
+extern "C" int ceph_armor(char *dst, const char *dst_end, 
+                          const char *src, const char *end);
+static void print_usage(char *exec){
+  cout << "Usage: " << exec << " <Options>\n";
+  cout << "Options:\n"
+          "-g <gw-ip> - The ip address of the gateway\n"
+          "-p <gw-port> - The port number of the gateway\n"
+          "-c <ceph.conf> - Absolute path of ceph config file\n"
+          "-rgw-admin <path/to/radosgw-admin> - radosgw-admin absolute path\n";
+}
+
+namespace admin_meta {
+class test_helper {
+  private:
+    string host;
+    string port;
+    string creds;
+    string rgw_admin_path;
+    string conf_path;
+    CURL *curl_inst;
+    map<string, string> response;
+    list<string> extra_hdrs;
+    string *resp_data;
+    unsigned resp_code;
+  public:
+    test_helper() : resp_data(NULL){
+      curl_global_init(CURL_GLOBAL_ALL);
+    }
+    ~test_helper(){
+      curl_global_cleanup();
+    }
+    int send_request(string method, string uri, 
+                     size_t (*function)(void *,size_t,size_t,void *) = 0,
+                     void *ud = 0, size_t length = 0);
+    int extract_input(int argc, char *argv[]);
+    string& get_response(string hdr){
+      return response[hdr];
+    }
+    void set_extra_header(string hdr){
+      extra_hdrs.push_back(hdr);
+    }
+    void set_response(char *val);
+    void set_response_data(char *data, size_t len){
+      if(resp_data) delete resp_data;
+      resp_data = new string(data, len);
+    }
+    string& get_rgw_admin_path() {
+      return rgw_admin_path;
+    }
+    string& get_ceph_conf_path() {
+      return conf_path;
+    }
+    void set_creds(string& c) {
+      creds = c;
+    }
+    const string *get_response_data(){return resp_data;}
+    unsigned get_resp_code(){return resp_code;}
+};
+
+int test_helper::extract_input(int argc, char *argv[]){
+#define ERR_CHECK_NEXT_PARAM(o) \
+  if((loop + 1) >= argc)return -1; \
+  else o = argv[loop+1];
+
+  for(unsigned loop = 1;loop < (unsigned)argc; loop += 2){
+    if(strcmp(argv[loop], "-g") == 0){
+      ERR_CHECK_NEXT_PARAM(host);
+    }else if(strcmp(argv[loop],"-p") == 0){
+      ERR_CHECK_NEXT_PARAM(port);
+    }else if(strcmp(argv[loop], "-c") == 0){
+      ERR_CHECK_NEXT_PARAM(conf_path);
+    }else if(strcmp(argv[loop], "-rgw-admin") == 0){
+      ERR_CHECK_NEXT_PARAM(rgw_admin_path);
+    }else return -1;
+  }
+  if(host.length() <= 0 ||
+     rgw_admin_path.length() <= 0)
+    return -1;
+  return 0;
+}
+
+void test_helper::set_response(char *r){
+  string sr(r), h, v;
+  unsigned off = sr.find(": ");
+  if(off != string::npos){
+    h.assign(sr, 0, off);
+    v.assign(sr, off + 2, sr.find("\r\n") - (off+2));
+  }else{
+    /*Could be the status code*/
+    if(sr.find("HTTP/") != string::npos){
+      h.assign(HTTP_RESPONSE_STR);
+      off = sr.find(" ");
+      v.assign(sr, off + 1, sr.find("\r\n") - (off + 1));
+      resp_code = atoi((v.substr(0, 3)).c_str());
+    }
+  }
+  response[h] = v;
+}
+
+size_t write_header(void *ptr, size_t size, size_t nmemb, void *ud){
+  test_helper *h = (test_helper *)ud;
+  h->set_response((char *)ptr);
+  return size*nmemb;
+}
+
+size_t write_data(void *ptr, size_t size, size_t nmemb, void *ud){
+  test_helper *h = (test_helper *)ud;
+  h->set_response_data((char *)ptr, size*nmemb);
+  return size*nmemb;
+}
+
+static inline void buf_to_hex(const unsigned char *buf, int len, char *str)
+{
+  int i;
+  str[0] = '\0';
+  for (i = 0; i < len; i++) {
+    sprintf(&str[i*2], "%02x", (int)buf[i]);
+  }
+}
+
+static void calc_hmac_sha1(const char *key, int key_len,
+                    const char *msg, int msg_len, char *dest)
+/* destination should be CEPH_CRYPTO_HMACSHA1_DIGESTSIZE bytes long */
+{
+  ceph::crypto::HMACSHA1 hmac((const unsigned char *)key, key_len);
+  hmac.Update((const unsigned char *)msg, msg_len);
+  hmac.Final((unsigned char *)dest);
+  
+  char hex_str[(CEPH_CRYPTO_HMACSHA1_DIGESTSIZE * 2) + 1];
+  admin_meta::buf_to_hex((unsigned char *)dest, CEPH_CRYPTO_HMACSHA1_DIGESTSIZE, hex_str);
+}
+
+static int get_s3_auth(string method, string creds, string date, string res, string& out){
+  string aid, secret, auth_hdr;
+  string tmp_res;
+  unsigned off = creds.find(":");
+  out = "";
+  if(off != string::npos){
+    aid.assign(creds, 0, off);
+    secret.assign(creds, off + 1, string::npos);
+
+    /*sprintf(auth_hdr, "%s\n\n\n%s\n%s", req_type, date, res);*/
+    char hmac_sha1[CEPH_CRYPTO_HMACSHA1_DIGESTSIZE];
+    char b64[65]; /* 64 is really enough */
+    unsigned off = res.find("?");
+    if(off == string::npos)
+      tmp_res = res;
+    else
+      tmp_res.assign(res, 0, off);
+    auth_hdr.append(method + string("\n\n\n") + date + string("\n") + tmp_res);
+    admin_meta::calc_hmac_sha1(secret.c_str(), secret.length(), 
+                               auth_hdr.c_str(), auth_hdr.length(), hmac_sha1);
+    int ret = ceph_armor(b64, b64 + 64, hmac_sha1,
+                         hmac_sha1 + CEPH_CRYPTO_HMACSHA1_DIGESTSIZE);
+    if (ret < 0) {
+      cout << "ceph_armor failed\n";
+      return -1;
+    }
+    b64[ret] = 0;
+    out.append(aid + string(":") + b64);
+  }else return -1;
+  return 0;
+}
+
+void get_date(string& d){
+  struct timeval tv;
+  char date[64];
+  struct tm tm;
+  char *days[] = {(char *)"Sun", (char *)"Mon", (char *)"Tue",
+                  (char *)"Wed", (char *)"Thu", (char *)"Fri", 
+                  (char *)"Sat"};
+  char *months[] = {(char *)"Jan", (char *)"Feb", (char *)"Mar", 
+                    (char *)"Apr", (char *)"May", (char *)"Jun",
+                    (char *)"Jul",(char *) "Aug", (char *)"Sep", 
+                    (char *)"Oct", (char *)"Nov", (char *)"Dec"};
+  gettimeofday(&tv, NULL);
+  gmtime_r(&tv.tv_sec, &tm);
+  sprintf(date, "%s, %d %s %d %d:%d:%d GMT", 
+          days[tm.tm_wday], 
+          tm.tm_mday, months[tm.tm_mon], 
+          tm.tm_year + 1900,
+          tm.tm_hour, tm.tm_min, 0 /*tm.tm_sec*/);
+  d = date;
+}
+
+int test_helper::send_request(string method, string res, 
+                                   size_t (*read_function)( void *,size_t,size_t,void *),
+                                   void *ud,
+                                   size_t length){
+  string url;
+  string auth, date;
+  url.append(string("http://") + host);
+  if(port.length() > 0)url.append(string(":") + port);
+  url.append(res);
+  curl_inst = curl_easy_init();
+  if(curl_inst){
+    curl_easy_setopt(curl_inst, CURLOPT_URL, url.c_str());
+    curl_easy_setopt(curl_inst, CURLOPT_CUSTOMREQUEST, method.c_str());
+    curl_easy_setopt(curl_inst, CURLOPT_VERBOSE, CURL_VERBOSE);
+    curl_easy_setopt(curl_inst, CURLOPT_HEADERFUNCTION, admin_meta::write_header);
+    curl_easy_setopt(curl_inst, CURLOPT_WRITEHEADER, (void *)this);
+    curl_easy_setopt(curl_inst, CURLOPT_WRITEFUNCTION, admin_meta::write_data);
+    curl_easy_setopt(curl_inst, CURLOPT_WRITEDATA, (void *)this);
+    if(read_function){
+      curl_easy_setopt(curl_inst, CURLOPT_READFUNCTION, read_function);
+      curl_easy_setopt(curl_inst, CURLOPT_READDATA, (void *)ud);
+      curl_easy_setopt(curl_inst, CURLOPT_UPLOAD, 1L);
+      curl_easy_setopt(curl_inst, CURLOPT_INFILESIZE_LARGE, (curl_off_t)length);
+    }
+
+    get_date(date);
+    string http_date;
+    http_date.append(string("Date: ") + date);
+
+    string s3auth;
+    if (admin_meta::get_s3_auth(method, creds, date, res, s3auth) < 0)
+      return -1;
+    auth.append(string("Authorization: AWS ") + s3auth);
+
+    struct curl_slist *slist = NULL;
+    slist = curl_slist_append(slist, auth.c_str());
+    slist = curl_slist_append(slist, http_date.c_str());
+    for(list<string>::iterator it = extra_hdrs.begin();
+        it != extra_hdrs.end(); it++){
+      slist = curl_slist_append(slist, (*it).c_str());
+    }
+    if(read_function)
+      curl_slist_append(slist, "Expect:");
+    curl_easy_setopt(curl_inst, CURLOPT_HTTPHEADER, slist); 
+
+    response.erase(response.begin(), response.end());
+    extra_hdrs.erase(extra_hdrs.begin(), extra_hdrs.end());
+    CURLcode res = curl_easy_perform(curl_inst);
+    if(res != CURLE_OK){
+      cout << "Curl perform failed for " << url << ", res: " << 
+        curl_easy_strerror(res) << "\n";
+      return -1;
+    }
+    curl_slist_free_all(slist);
+  }
+  curl_easy_cleanup(curl_inst);
+  return 0;
+}
+};
+
+admin_meta::test_helper *g_test;
+Finisher *finisher;
+
+int run_rgw_admin(string& cmd, string& resp) {
+  pid_t pid;
+  pid = fork();
+  if (pid == 0) {
+    /* child */
+    list<string> l;
+    get_str_list(cmd, " \t", l);
+    char *argv[l.size()];
+    unsigned loop = 1;
+
+    argv[0] = (char *)"radosgw-admin";
+    for (list<string>::iterator it = l.begin(); 
+         it != l.end(); it++) {
+      argv[loop++] = (char *)(*it).c_str();
+    }
+    argv[loop] = NULL;
+    close(1);
+    stdout = fopen(RGW_ADMIN_RESP_PATH, "w+");
+    if (!stdout) {
+      cout << "Unable to open stdout file" << std::endl;
+    }
+    execv((g_test->get_rgw_admin_path()).c_str(), argv); 
+  } else if (pid > 0) {
+    int status;
+    waitpid(pid, &status, 0);
+    if (WIFEXITED(status)) {
+      if(WEXITSTATUS(status) != 0) {
+        cout << "Child exited with status " << WEXITSTATUS(status) << std::endl;
+        return -1;
+      }
+    }
+    ifstream in;
+    struct stat st;
+
+    if (stat(RGW_ADMIN_RESP_PATH, &st) < 0) {
+      cout << "Error stating the admin response file, errno " << errno << std::endl;
+      return -1;
+    } else {
+      char *data = (char *)malloc(st.st_size + 1);
+      in.open(RGW_ADMIN_RESP_PATH);
+      in.read(data, st.st_size);
+      in.close();
+      data[st.st_size] = 0;
+      resp = data;
+      free(data);
+      unlink(RGW_ADMIN_RESP_PATH);
+      /* cout << "radosgw-admin " << cmd << ": " << resp << std::endl; */
+    }
+  } else 
+    return -1;
+  return 0;
+}
+
+int get_creds(string& json, string& creds) {
+  JSONParser parser;
+  if(!parser.parse(json.c_str(), json.length())) {
+    cout << "Error parsing create user response" << std::endl;
+    return -1;
+  }
+
+  RGWUserInfo info;
+  decode_json_obj(info, &parser);
+  creds = "";
+  for(map<string, RGWAccessKey>::iterator it = info.access_keys.begin();
+      it != info.access_keys.end(); it++) {
+    RGWAccessKey _k = it->second;
+    /*cout << "accesskeys [ " << it->first << " ] = " << 
+      "{ " << _k.id << ", " << _k.key << ", " << _k.subuser << "}" << std::endl;*/
+    creds.append(it->first + string(":") + _k.key);
+  }
+  return 0;
+}
+
+int user_create(string& uid, string& display_name, bool set_creds = true) {
+  stringstream ss;
+  string creds;
+  ss << "-c " << g_test->get_ceph_conf_path() << " user create --uid=" << uid
+    << " --display-name=" << display_name;
+
+  string out;
+  string cmd = ss.str();
+  if(run_rgw_admin(cmd, out) != 0) {
+    cout << "Error creating user" << std::endl;
+    return -1;
+  }
+  get_creds(out, creds);
+  if(set_creds)
+    g_test->set_creds(creds);
+  return 0;
+}
+
+int user_info(string& uid, string& display_name, RGWUserInfo& uinfo) {
+  stringstream ss;
+  string creds;
+  ss << "-c " << g_test->get_ceph_conf_path() << " user info --uid=" << uid
+    << " --display-name=" << display_name;
+
+  string out;
+  string cmd = ss.str();
+  if(run_rgw_admin(cmd, out) != 0) {
+    cout << "Error reading user information" << std::endl;
+    return -1;
+  }
+  JSONParser parser;
+  if(!parser.parse(out.c_str(), out.length())) {
+    cout << "Error parsing create user response" << std::endl;
+    return -1;
+  }
+  decode_json_obj(uinfo, &parser);
+  return 0;
+}
+
+int user_rm(string& uid, string& display_name) {
+  stringstream ss;
+  string creds;
+  ss << "-c " << g_test->get_ceph_conf_path() << " user rm --uid=" << uid
+    << " --display-name=" << display_name;
+
+  string out;
+  string cmd = ss.str();
+  if(run_rgw_admin(cmd, out) != 0) {
+    cout << "Error removing user" << std::endl;
+    return -1;
+  }
+  return 0;
+}
+
+int meta_caps_add(const char *perm) {
+  stringstream ss;
+
+  ss << "-c " << g_test->get_ceph_conf_path() << " caps add --caps=" <<
+     meta_caps << "=" << perm << " --uid=" << uid;
+  string out;
+  string cmd = ss.str();
+  if(run_rgw_admin(cmd, out) != 0) {
+    cout << "Error creating user" << std::endl;
+    return -1;
+  }
+  return 0;
+}
+
+int meta_caps_rm(const char *perm) {
+  stringstream ss;
+
+  ss << "-c " << g_test->get_ceph_conf_path() << " caps rm --caps=" <<
+     meta_caps << "=" << perm << " --uid=" << uid;
+  string out;
+  string cmd = ss.str();
+  if(run_rgw_admin(cmd, out) != 0) {
+    cout << "Error creating user" << std::endl;
+    return -1;
+  }
+  return 0;
+}
+
+int compare_access_keys(RGWAccessKey& k1, RGWAccessKey& k2) {
+  if (k1.id.compare(k2.id) != 0)
+    return -1;
+  if (k1.key.compare(k2.key) != 0) 
+    return -1;
+  if (k1.subuser.compare(k2.subuser) != 0) 
+    return -1;
+
+  return 0;
+}
+
+int compare_user_info(RGWUserInfo& i1, RGWUserInfo& i2) {
+  int rv;
+
+  if ((rv = i1.user_id.compare(i2.user_id)) != 0)
+    return rv;
+  if ((rv = i1.display_name.compare(i2.display_name)) != 0)
+    return rv;
+  if ((rv = i1.user_email.compare(i2.user_email)) != 0)
+    return rv;
+  if (i1.access_keys.size() != i2.access_keys.size())
+    return -1;
+  for (map<string, RGWAccessKey>::iterator it = i1.access_keys.begin();
+       it != i1.access_keys.end(); it++) {
+    RGWAccessKey k1, k2;
+    k1 = it->second;
+    if (i2.access_keys.count(it->first) == 0)
+      return -1;
+    k2 = i2.access_keys[it->first];
+    if (compare_access_keys(k1, k2) != 0)
+      return -1;
+  }
+  if (i1.swift_keys.size() != i2.swift_keys.size())
+    return -1;
+  for (map<string, RGWAccessKey>::iterator it = i1.swift_keys.begin();
+       it != i1.swift_keys.end(); it++) {
+    RGWAccessKey k1, k2;
+    k1 = it->second;
+    if (i2.swift_keys.count(it->first) == 0)
+      return -1;
+    k2 = i2.swift_keys[it->first];
+    if (compare_access_keys(k1, k2) != 0)
+      return -1;
+  }
+  if (i1.subusers.size() != i2.subusers.size()) 
+    return -1;
+  for (map<string, RGWSubUser>::iterator it = i1.subusers.begin();
+       it != i1.subusers.end(); it++) {
+    RGWSubUser k1, k2;
+    k1 = it->second;
+    if (!i2.subusers.count(it->first))
+      return -1;
+    k2 = i2.subusers[it->first];
+    if (k1.name.compare(k2.name) != 0) 
+      return -1;
+    if (k1.perm_mask != k2.perm_mask)
+      return -1;
+  }
+  if (i1.suspended != i2.suspended)
+    return -1;
+  if (i1.max_buckets != i2.max_buckets)
+    return -1;
+  uint32_t p1, p2;
+  p1 = p2 = RGW_CAP_ALL;
+  if (i1.caps.check_cap(meta_caps, p1) != 0)
+    return -1;
+  if (i2.caps.check_cap(meta_caps, p2) != 0)
+    return -1;
+  return 0;
+}
+
+int parse_json_resp(JSONParser &parser) {
+  string *resp;
+  resp = (string *)g_test->get_response_data();
+  if(!resp)
+    return -1;
+  if(!parser.parse(resp->c_str(), resp->length())) {
+    cout << "Error parsing create user response" << std::endl;
+    return -1;
+  }
+  return 0;
+}
+
+size_t meta_read_json(void *ptr, size_t s, size_t n, void *ud){
+  stringstream *ss = (stringstream *)ud;
+  size_t len = ss->str().length();
+  if(s*n < len){
+    cout << "Cannot copy json data, as len is not enough\n";
+    return 0;
+  }
+  memcpy(ptr, (void *)ss->str().c_str(), len);
+  return len;
+}
+
+TEST(TestRGWAdmin, meta_list){
+  JSONParser parser;
+  bool found = false;
+  const char *perm = "*";
+
+  ASSERT_EQ(0, user_create(uid, display_name));
+  ASSERT_EQ(0, meta_caps_add(perm));
+
+  /*Check the sections*/
+  g_test->send_request(string("GET"), string("/admin/metadata/"));
+  EXPECT_EQ(200U, g_test->get_resp_code());
+
+  ASSERT_TRUE(parse_json_resp(parser) == 0);
+  EXPECT_TRUE(parser.is_array());
+  
+  vector<string> l;
+  l = parser.get_array_elements();
+  for(vector<string>::iterator it = l.begin();
+      it != l.end(); it++) {
+    if((*it).compare("\"user\"") == 0) {
+      found = true;
+      break;
+    }
+  }
+  EXPECT_TRUE(found);
+
+  /*Check with a wrong section*/
+  g_test->send_request(string("GET"), string("/admin/metadata/users"));
+  EXPECT_EQ(404U, g_test->get_resp_code());
+
+  /*Check the list of keys*/
+  g_test->send_request(string("GET"), string("/admin/metadata/user"));
+  EXPECT_EQ(200U, g_test->get_resp_code());
+
+  ASSERT_TRUE(parse_json_resp(parser) == 0);
+  EXPECT_TRUE(parser.is_array());
+  
+  l = parser.get_array_elements();
+  EXPECT_EQ(1U, l.size());
+  for(vector<string>::iterator it = l.begin();
+      it != l.end(); it++) {
+    if((*it).compare(string("\"") + uid + string("\"")) == 0) {
+      found = true;
+      break;
+    }
+  }
+  EXPECT_TRUE(found);
+
+  /*Check with second user*/
+  string uid2 = "ceph1", display_name2 = "CEPH1";
+  ASSERT_EQ(0, user_create(uid2, display_name2, false));
+  /*Check the list of keys*/
+  g_test->send_request(string("GET"), string("/admin/metadata/user"));
+  EXPECT_EQ(200U, g_test->get_resp_code());
+
+  ASSERT_TRUE(parse_json_resp(parser) == 0);
+  EXPECT_TRUE(parser.is_array());
+  
+  l = parser.get_array_elements();
+  EXPECT_EQ(2U, l.size());
+  bool found2 = false;
+  for(vector<string>::iterator it = l.begin();
+      it != l.end(); it++) {
+    if((*it).compare(string("\"") + uid + string("\"")) == 0) {
+      found = true;
+    }
+    if((*it).compare(string("\"") + uid2 + string("\"")) == 0) {
+      found2 = true;
+    }
+  }
+  EXPECT_TRUE(found && found2);
+  ASSERT_EQ(0, user_rm(uid2, display_name2));
+
+  /*Remove the metadata caps*/
+  int rv = meta_caps_rm(perm);
+  EXPECT_EQ(0, rv);
+  
+  if(rv == 0) {
+    g_test->send_request(string("GET"), string("/admin/metadata/"));
+    EXPECT_EQ(403U, g_test->get_resp_code());
+
+    g_test->send_request(string("GET"), string("/admin/metadata/user"));
+    EXPECT_EQ(403U, g_test->get_resp_code());
+  }
+  ASSERT_EQ(0, user_rm(uid, display_name));
+}
+
+TEST(TestRGWAdmin, meta_get){
+  JSONParser parser;
+  const char *perm = "*";
+  RGWUserInfo info;
+
+  ASSERT_EQ(0, user_create(uid, display_name));
+  ASSERT_EQ(0, meta_caps_add(perm));
+
+  ASSERT_EQ(0, user_info(uid, display_name, info));
+ 
+  /*Check with a wrong parameter*/
+  g_test->send_request(string("GET"), string("/admin/metadata/user?keys=test"));
+  EXPECT_EQ(400U, g_test->get_resp_code());
+  g_test->send_request(string("GET"), string("/admin/metadata/user?key=test"));
+  EXPECT_EQ(404U, g_test->get_resp_code());
+
+  g_test->send_request(string("GET"), (string("/admin/metadata/user?key=") + uid));
+  EXPECT_EQ(200U, g_test->get_resp_code());
+
+  ASSERT_TRUE(parse_json_resp(parser) == 0);
+  RGWObjVersionTracker objv_tracker;
+  string metadata_key;
+
+  obj_version *objv = &objv_tracker.read_version;
+     
+  JSONDecoder::decode_json("key", metadata_key, &parser);
+  JSONDecoder::decode_json("ver", *objv, &parser);
+  JSONObj *jo = parser.find_obj("data");
+  ASSERT_TRUE(jo);
+  string exp_meta_key = "user:";
+  exp_meta_key.append(uid);
+  EXPECT_TRUE(metadata_key.compare(exp_meta_key) == 0);
+
+  RGWUserInfo obt_info;
+  decode_json_obj(obt_info, jo);
+
+  EXPECT_TRUE(compare_user_info(info, obt_info) == 0);
+
+  /*Make a modification and check if its reflected*/
+  ASSERT_EQ(0, meta_caps_rm(perm));
+  perm = "read";
+  ASSERT_EQ(0, meta_caps_add(perm));
+  
+  JSONParser parser1;
+  g_test->send_request(string("GET"), (string("/admin/metadata/user?key=") + uid));
+  EXPECT_EQ(200U, g_test->get_resp_code());
+
+  ASSERT_TRUE(parse_json_resp(parser1) == 0);
+ 
+  RGWObjVersionTracker objv_tracker1;
+  obj_version *objv1 = &objv_tracker1.read_version;
+
+  JSONDecoder::decode_json("key", metadata_key, &parser1);
+  JSONDecoder::decode_json("ver", *objv1, &parser1);
+  jo = parser1.find_obj("data");
+  ASSERT_TRUE(jo);
+
+  decode_json_obj(obt_info, jo);
+  uint32_t p1, p2;
+  p1 = RGW_CAP_ALL;
+  p2 = RGW_CAP_READ;
+  EXPECT_TRUE (info.caps.check_cap(meta_caps, p1) == 0);
+  EXPECT_TRUE (obt_info.caps.check_cap(meta_caps, p2) == 0);
+  p2 = RGW_CAP_WRITE;
+  EXPECT_TRUE (obt_info.caps.check_cap(meta_caps, p2) != 0);
+
+  /*Version and tag infromation*/
+  EXPECT_TRUE(objv1->ver > objv->ver);
+  EXPECT_EQ(objv1->tag, objv->tag);
+  
+  int rv = meta_caps_rm(perm);
+  EXPECT_EQ(0, rv);
+  
+  if(rv == 0) {
+    g_test->send_request(string("GET"), (string("/admin/metadata/user?key=") + uid));
+    EXPECT_EQ(403U, g_test->get_resp_code());
+  }
+  ASSERT_EQ(0, user_rm(uid, display_name));
+}
+
+TEST(TestRGWAdmin, meta_put){
+  JSONParser parser;
+  const char *perm = "*";
+  RGWUserInfo info;
+
+  ASSERT_EQ(0, user_create(uid, display_name));
+  ASSERT_EQ(0, meta_caps_add(perm));
+  
+  g_test->send_request(string("GET"), (string("/admin/metadata/user?key=") + uid));
+  EXPECT_EQ(200U, g_test->get_resp_code());
+
+  ASSERT_TRUE(parse_json_resp(parser) == 0);
+  RGWObjVersionTracker objv_tracker;
+  string metadata_key;
+
+  obj_version *objv = &objv_tracker.read_version;
+     
+  JSONDecoder::decode_json("key", metadata_key, &parser);
+  JSONDecoder::decode_json("ver", *objv, &parser);
+  JSONObj *jo = parser.find_obj("data");
+  ASSERT_TRUE(jo);
+  string exp_meta_key = "user:";
+  exp_meta_key.append(uid);
+  EXPECT_TRUE(metadata_key.compare(exp_meta_key) == 0);
+
+  RGWUserInfo obt_info;
+  decode_json_obj(obt_info, jo);
+
+  /*Change the cap and PUT */
+  RGWUserCaps caps;
+  string new_cap;
+  Formatter *f = new JSONFormatter();
+
+  new_cap = meta_caps + string("=write");
+  caps.add_from_string(new_cap);
+  obt_info.caps = caps;
+  f->open_object_section("metadata_info");
+  ::encode_json("key", metadata_key, f);
+  ::encode_json("ver", *objv, f);
+  ::encode_json("data", obt_info, f);
+  f->close_section();
+  std::stringstream ss;
+  f->flush(ss);
+
+  g_test->send_request(string("PUT"), (string("/admin/metadata/user?key=") + uid), 
+                       meta_read_json,
+                       (void *)&ss, ss.str().length());
+  EXPECT_EQ(200U, g_test->get_resp_code());
+
+  ASSERT_EQ(0, user_info(uid, display_name, obt_info));
+  uint32_t cp;
+  cp = RGW_CAP_WRITE;
+  EXPECT_TRUE (obt_info.caps.check_cap(meta_caps, cp) == 0);
+  cp = RGW_CAP_READ;
+  EXPECT_TRUE (obt_info.caps.check_cap(meta_caps, cp) != 0);
+  
+  int rv = meta_caps_rm("write");
+  EXPECT_EQ(0, rv);
+  if(rv == 0) {
+    g_test->send_request(string("PUT"), (string("/admin/metadata/user?key=") + uid));
+    EXPECT_EQ(403U, g_test->get_resp_code());
+  }
+  ASSERT_EQ(0, user_rm(uid, display_name));
+}
+
+TEST(TestRGWAdmin, meta_delete){
+  JSONParser parser;
+  const char *perm = "*";
+  RGWUserInfo info;
+
+  ASSERT_EQ(0, user_create(uid, display_name));
+  ASSERT_EQ(0, meta_caps_add(perm));
+
+  g_test->send_request(string("DELETE"), (string("/admin/metadata/user?key=") + uid));
+  EXPECT_EQ(200U, g_test->get_resp_code());
+
+  ASSERT_TRUE(user_info(uid, display_name, info) != 0);
+
+  ASSERT_EQ(0, user_create(uid, display_name));
+  perm = "read";
+  ASSERT_EQ(0, meta_caps_add(perm));
+  
+  g_test->send_request(string("DELETE"), (string("/admin/metadata/user?key=") + uid));
+  EXPECT_EQ(403U, g_test->get_resp_code());
+  ASSERT_EQ(0, user_rm(uid, display_name));
+}
+
+int main(int argc, char *argv[]){
+  vector<const char*> args;
+  argv_to_vec(argc, (const char **)argv, args);
+
+  global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
+  common_init_finish(g_ceph_context);
+  g_test = new admin_meta::test_helper();
+  finisher = new Finisher(g_ceph_context);
+#ifdef GTEST
+  ::testing::InitGoogleTest(&argc, argv);
+#endif
+  finisher->start();
+
+  if(g_test->extract_input(argc, argv) < 0){
+    print_usage(argv[0]);
+    return -1;
+  }
+#ifdef GTEST
+  int r = RUN_ALL_TESTS();
+  if (r >= 0) {
+    cout << "There are no failures in the test case\n";
+  } else {
+    cout << "There are some failures\n";
+  }
+#endif
+  finisher->stop();
+  return 0;
+}