compressor/zlib:make zlib windowBits configurable for compression

Signed-off-by: WangPengfei <wpf_1253@qq.com>

	modified:   src/common/legacy_config_opts.h
	modified:   src/common/options.cc
	modified:   src/compressor/Compressor.h
	modified:   src/compressor/QatAccel.cc
	modified:   src/compressor/QatAccel.h
	modified:   src/compressor/brotli/BrotliCompressor.cc
	modified:   src/compressor/brotli/BrotliCompressor.h
	modified:   src/compressor/lz4/LZ4Compressor.h
	modified:   src/compressor/snappy/SnappyCompressor.h
	modified:   src/compressor/zlib/ZlibCompressor.cc
	modified:   src/compressor/zlib/ZlibCompressor.h
	modified:   src/compressor/zstd/ZstdCompressor.h
	modified:   src/os/bluestore/BlueStore.cc
	modified:   src/os/bluestore/bluestore_types.cc
	modified:   src/os/bluestore/bluestore_types.h
	modified:   src/rgw/rgw_compression.cc
	modified:   src/rgw/rgw_compression.h
	modified:   src/rgw/rgw_compression_types.h
	modified:   src/rgw/rgw_json_enc.cc
	modified:   src/rgw/rgw_op.cc
	modified:   src/rgw/rgw_rados.cc
	modified:   src/test/compressor/compressor_example.h
	modified:   src/test/compressor/test_compression.cc
	modified:   src/test/rgw/test_rgw_compression.cc

10 files changed, 64 insertions, 55 deletions

diff --git a/src/compressor/Compressor.h b/src/compressor/Compressor.h
index 8b26bacf178..6a4eb277668 100644
--- a/src/compressor/Compressor.h
+++ b/src/compressor/Compressor.h
@@ -88,11 +88,11 @@ public:
   CompressionAlgorithm get_type() const {
     return alg;
   }
-  virtual int compress(const ceph::bufferlist &in, ceph::bufferlist &out) = 0;
-  virtual int decompress(const ceph::bufferlist &in, ceph::bufferlist &out) = 0;
+  virtual int compress(const ceph::bufferlist &in, ceph::bufferlist &out, boost::optional<int32_t> &compressor_message) = 0;
+  virtual int decompress(const ceph::bufferlist &in, ceph::bufferlist &out, boost::optional<int32_t> compressor_message) = 0;
   // this is a bit weird but we need non-const iterator to be in
   // alignment with decode methods
-  virtual int decompress(ceph::bufferlist::const_iterator &p, size_t compressed_len, ceph::bufferlist &out) = 0;
+  virtual int decompress(ceph::bufferlist::const_iterator &p, size_t compressed_len, ceph::bufferlist &out, boost::optional<int32_t> compressor_message) = 0;
 
   static CompressorRef create(CephContext *cct, const std::string &type);
   static CompressorRef create(CephContext *cct, int alg);
diff --git a/src/compressor/QatAccel.cc b/src/compressor/QatAccel.cc
index 7836243b8a3..fff9e34e846 100644
--- a/src/compressor/QatAccel.cc
+++ b/src/compressor/QatAccel.cc
@@ -59,7 +59,7 @@ bool QatAccel::init(const std::string &alg) {
   return true;
 }
 
-int QatAccel::compress(const bufferlist &in, bufferlist &out) {
+int QatAccel::compress(const bufferlist &in, bufferlist &out, boost::optional<int32_t> &compressor_message) {
   for (auto &i : in.buffers()) {
     const unsigned char* c_in = (unsigned char*) i.c_str();
     unsigned int len = i.length();
@@ -75,14 +75,15 @@ int QatAccel::compress(const bufferlist &in, bufferlist &out) {
   return 0;
 }
 
-int QatAccel::decompress(const bufferlist &in, bufferlist &out) {
+int QatAccel::decompress(const bufferlist &in, bufferlist &out, boost::optional<int32_t> compressor_message) {
   auto i = in.begin();
-  return decompress(i, in.length(), out);
+  return decompress(i, in.length(), out, compressor_message);
 }
 
 int QatAccel::decompress(bufferlist::const_iterator &p,
 		 size_t compressed_len,
-		 bufferlist &dst) {
+		 bufferlist &dst,
+		 boost::optional<int32_t> compressor_message) {
   unsigned int ratio_idx = 0;
   bool read_more = false;
   bool joint = false;
diff --git a/src/compressor/QatAccel.h b/src/compressor/QatAccel.h
index 295b180eb21..f15e3303701 100644
--- a/src/compressor/QatAccel.h
+++ b/src/compressor/QatAccel.h
@@ -27,9 +27,9 @@ class QatAccel {
 
   bool init(const std::string &alg);
 
-  int compress(const bufferlist &in, bufferlist &out);
-  int decompress(const bufferlist &in, bufferlist &out);
-  int decompress(bufferlist::const_iterator &p, size_t compressed_len, bufferlist &dst);
+  int compress(const bufferlist &in, bufferlist &out, boost::optional<int32_t> &compressor_message);
+  int decompress(const bufferlist &in, bufferlist &out, boost::optional<int32_t> compressor_message);
+  int decompress(bufferlist::const_iterator &p, size_t compressed_len, bufferlist &dst, boost::optional<int32_t> compressor_message);
 };
 
 #endif
diff --git a/src/compressor/brotli/BrotliCompressor.cc b/src/compressor/brotli/BrotliCompressor.cc
index 27685da35bd..ed4abef4bbf 100644
--- a/src/compressor/brotli/BrotliCompressor.cc
+++ b/src/compressor/brotli/BrotliCompressor.cc
@@ -5,7 +5,7 @@
 
 #define MAX_LEN (CEPH_PAGE_SIZE)
 
-int BrotliCompressor::compress(const bufferlist &in, bufferlist &out) 
+int BrotliCompressor::compress(const bufferlist &in, bufferlist &out, boost::optional<int32_t> &compressor_message) 
 {
   BrotliEncoderState* s = BrotliEncoderCreateInstance(nullptr,
                                                       nullptr,
@@ -49,7 +49,8 @@ int BrotliCompressor::compress(const bufferlist &in, bufferlist &out)
 
 int BrotliCompressor::decompress(bufferlist::const_iterator &p,
                                  size_t compressed_size,
-                                 bufferlist &out) 
+                                 bufferlist &out,
+				 boost::optional<int32_t> compressor_message) 
 {
   BrotliDecoderState* s = BrotliDecoderCreateInstance(nullptr,
                                                       nullptr,
@@ -88,8 +89,8 @@ int BrotliCompressor::decompress(bufferlist::const_iterator &p,
   return 0;
 }
 
-int BrotliCompressor::decompress(const bufferlist &in, bufferlist &out) 
+int BrotliCompressor::decompress(const bufferlist &in, bufferlist &out, boost::optional<int32_t> compressor_message) 
 {  
   auto i = std::cbegin(in);
-  return decompress(i, in.length(), out);
+  return decompress(i, in.length(), out, compressor_message);
 }
diff --git a/src/compressor/brotli/BrotliCompressor.h b/src/compressor/brotli/BrotliCompressor.h
index 482fe5e20eb..373300645f8 100644
--- a/src/compressor/brotli/BrotliCompressor.h
+++ b/src/compressor/brotli/BrotliCompressor.h
@@ -22,9 +22,9 @@ class BrotliCompressor : public Compressor
   public:
   BrotliCompressor() : Compressor(COMP_ALG_BROTLI, "brotli") {}
   
-  int compress(const bufferlist &in, bufferlist &out) override;
-  int decompress(const bufferlist &in, bufferlist &out) override;
-  int decompress(bufferlist::const_iterator &p, size_t compressed_len, bufferlist &out) override;
+  int compress(const bufferlist &in, bufferlist &out, boost::optional<int32_t> &compressor_message) override;
+  int decompress(const bufferlist &in, bufferlist &out, boost::optional<int32_t> compressor_message) override;
+  int decompress(bufferlist::const_iterator &p, size_t compressed_len, bufferlist &out, boost::optional<int32_t> compressor_message) override;
 };
 
 #endif //CEPH_BROTLICOMPRESSOR_H
diff --git a/src/compressor/lz4/LZ4Compressor.h b/src/compressor/lz4/LZ4Compressor.h
index d9ab78a30a4..b97d91f2b2e 100644
--- a/src/compressor/lz4/LZ4Compressor.h
+++ b/src/compressor/lz4/LZ4Compressor.h
@@ -35,7 +35,7 @@ class LZ4Compressor : public Compressor {
 #endif
   }
 
-  int compress(const ceph::buffer::list &src, ceph::buffer::list &dst) override {
+  int compress(const ceph::buffer::list &src, ceph::buffer::list &dst, boost::optional<int32_t> &compressor_message) override {
     // older versions of liblz4 introduce bit errors when compressing
     // fragmented buffers.  this was fixed in lz4 commit
     // af127334670a5e7b710bbd6adb71aa7c3ef0cd72, which first
@@ -45,12 +45,12 @@ class LZ4Compressor : public Compressor {
     if (!src.is_contiguous()) {
       ceph::buffer::list new_src = src;
       new_src.rebuild();
-      return compress(new_src, dst);
+      return compress(new_src, dst, compressor_message);
     }
 
 #ifdef HAVE_QATZIP
     if (qat_enabled)
-      return qat_accel.compress(src, dst);
+      return qat_accel.compress(src, dst, compressor_message);
 #endif
     ceph::buffer::ptr outptr = ceph::buffer::create_small_page_aligned(
       LZ4_compressBound(src.length()));
@@ -83,21 +83,22 @@ class LZ4Compressor : public Compressor {
     return 0;
   }
 
-  int decompress(const ceph::buffer::list &src, ceph::buffer::list &dst) override {
+  int decompress(const ceph::buffer::list &src, ceph::buffer::list &dst, boost::optional<int32_t> compressor_message) override {
 #ifdef HAVE_QATZIP
     if (qat_enabled)
-      return qat_accel.decompress(src, dst);
+      return qat_accel.decompress(src, dst, compressor_message);
 #endif
     auto i = std::cbegin(src);
-    return decompress(i, src.length(), dst);
+    return decompress(i, src.length(), dst, compressor_message);
   }
 
   int decompress(ceph::buffer::list::const_iterator &p,
 		 size_t compressed_len,
-		 ceph::buffer::list &dst) override {
+		 ceph::buffer::list &dst,
+		 boost::optional<int32_t> compressor_message) override {
 #ifdef HAVE_QATZIP
     if (qat_enabled)
-      return qat_accel.decompress(p, compressed_len, dst);
+      return qat_accel.decompress(p, compressed_len, dst, compressor_message);
 #endif
     using ceph::decode;
     uint32_t count;
diff --git a/src/compressor/snappy/SnappyCompressor.h b/src/compressor/snappy/SnappyCompressor.h
index 9da7feaa4bc..25393f8dc1f 100644
--- a/src/compressor/snappy/SnappyCompressor.h
+++ b/src/compressor/snappy/SnappyCompressor.h
@@ -66,10 +66,10 @@ class SnappyCompressor : public Compressor {
 #endif
   }
 
-  int compress(const ceph::bufferlist &src, ceph::bufferlist &dst) override {
+  int compress(const ceph::bufferlist &src, ceph::bufferlist &dst, boost::optional<int32_t> &compressor_message) override {
 #ifdef HAVE_QATZIP
     if (qat_enabled)
-      return qat_accel.compress(src, dst);
+      return qat_accel.compress(src, dst, compressor_message);
 #endif
     BufferlistSource source(const_cast<ceph::bufferlist&>(src).begin(), src.length());
     ceph::bufferptr ptr = ceph::buffer::create_small_page_aligned(
@@ -80,21 +80,22 @@ class SnappyCompressor : public Compressor {
     return 0;
   }
 
-  int decompress(const ceph::bufferlist &src, ceph::bufferlist &dst) override {
+  int decompress(const ceph::bufferlist &src, ceph::bufferlist &dst, boost::optional<int32_t> compressor_message) override {
 #ifdef HAVE_QATZIP
     if (qat_enabled)
-      return qat_accel.decompress(src, dst);
+      return qat_accel.decompress(src, dst, compressor_message);
 #endif
     auto i = src.begin();
-    return decompress(i, src.length(), dst);
+    return decompress(i, src.length(), dst, compressor_message);
   }
 
   int decompress(ceph::bufferlist::const_iterator &p,
 		 size_t compressed_len,
-		 ceph::bufferlist &dst) override {
+		 ceph::bufferlist &dst,
+		 boost::optional<int32_t> compressor_message) override {
 #ifdef HAVE_QATZIP
     if (qat_enabled)
-      return qat_accel.decompress(p, compressed_len, dst);
+      return qat_accel.decompress(p, compressed_len, dst, compressor_message);
 #endif
     snappy::uint32 res_len = 0;
     BufferlistSource source_1(p, compressed_len);
diff --git a/src/compressor/zlib/ZlibCompressor.cc b/src/compressor/zlib/ZlibCompressor.cc
index c9fe6f6519b..5edaba67242 100644
--- a/src/compressor/zlib/ZlibCompressor.cc
+++ b/src/compressor/zlib/ZlibCompressor.cc
@@ -52,7 +52,7 @@ _prefix(std::ostream* _dout)
 // compression ratio.
 #define ZLIB_MEMORY_LEVEL 8
 
-int ZlibCompressor::zlib_compress(const bufferlist &in, bufferlist &out)
+int ZlibCompressor::zlib_compress(const bufferlist &in, bufferlist &out, boost::optional<int32_t> &compressor_message)
 {
   int ret;
   unsigned have;
@@ -64,12 +64,13 @@ int ZlibCompressor::zlib_compress(const bufferlist &in, bufferlist &out)
   strm.zalloc = Z_NULL;
   strm.zfree = Z_NULL;
   strm.opaque = Z_NULL;
-  ret = deflateInit2(&strm, cct->_conf->compressor_zlib_level, Z_DEFLATED, ZLIB_DEFAULT_WIN_SIZE, ZLIB_MEMORY_LEVEL, Z_DEFAULT_STRATEGY);
+  ret = deflateInit2(&strm, cct->_conf->compressor_zlib_level, Z_DEFLATED, cct->_conf->compressor_zlib_winsize, ZLIB_MEMORY_LEVEL, Z_DEFAULT_STRATEGY);
   if (ret != Z_OK) {
     dout(1) << "Compression init error: init return "
          << ret << " instead of Z_OK" << dendl;
     return -1;
   }
+  compressor_message = cct->_conf->compressor_zlib_winsize;
 
   for (ceph::bufferlist::buffers_t::const_iterator i = in.buffers().begin();
       i != in.buffers().end();) {
@@ -113,7 +114,7 @@ int ZlibCompressor::zlib_compress(const bufferlist &in, bufferlist &out)
 }
 
 #if __x86_64__ && defined(HAVE_BETTER_YASM_ELF64)
-int ZlibCompressor::isal_compress(const bufferlist &in, bufferlist &out)
+int ZlibCompressor::isal_compress(const bufferlist &in, bufferlist &out, boost::optional<int32_t> &compressor_message)
 {
   int ret;
   unsigned have;
@@ -124,6 +125,7 @@ int ZlibCompressor::isal_compress(const bufferlist &in, bufferlist &out)
   /* allocate deflate state */
   isal_deflate_init(&strm);
   strm.end_of_stream = 0;
+  compressor_message = ZLIB_DEFAULT_WIN_SIZE;
 
   for (ceph::bufferlist::buffers_t::const_iterator i = in.buffers().begin();
       i != in.buffers().end();) {
@@ -166,27 +168,27 @@ int ZlibCompressor::isal_compress(const bufferlist &in, bufferlist &out)
 }
 #endif
 
-int ZlibCompressor::compress(const bufferlist &in, bufferlist &out)
+int ZlibCompressor::compress(const bufferlist &in, bufferlist &out, boost::optional<int32_t> &compressor_message)
 {
 #ifdef HAVE_QATZIP
   if (qat_enabled)
-    return qat_accel.compress(in, out);
+    return qat_accel.compress(in, out, compressor_message);
 #endif
 #if __x86_64__ && defined(HAVE_BETTER_YASM_ELF64)
   if (isal_enabled)
-    return isal_compress(in, out);
+    return isal_compress(in, out, compressor_message);
   else
-    return zlib_compress(in, out);
+    return zlib_compress(in, out, compressor_message);
 #else
-  return zlib_compress(in, out);
+  return zlib_compress(in, out, compressor_message);
 #endif
 }
 
-int ZlibCompressor::decompress(bufferlist::const_iterator &p, size_t compressed_size, bufferlist &out)
+int ZlibCompressor::decompress(bufferlist::const_iterator &p, size_t compressed_size, bufferlist &out, boost::optional<int32_t> compressor_message)
 {
 #ifdef HAVE_QATZIP
   if (qat_enabled)
-    return qat_accel.decompress(p, compressed_size, out);
+    return qat_accel.decompress(p, compressed_size, out, compressor_message);
 #endif
 
   int ret;
@@ -203,7 +205,9 @@ int ZlibCompressor::decompress(bufferlist::const_iterator &p, size_t compressed_
   strm.next_in = Z_NULL;
 
   // choose the variation of compressor
-  ret = inflateInit2(&strm, ZLIB_DEFAULT_WIN_SIZE);
+  if (!compressor_message)
+    compressor_message = ZLIB_DEFAULT_WIN_SIZE;
+  ret = inflateInit2(&strm, *compressor_message);
   if (ret != Z_OK) {
     dout(1) << "Decompression init error: init return "
          << ret << " instead of Z_OK" << dendl;
@@ -240,12 +244,12 @@ int ZlibCompressor::decompress(bufferlist::const_iterator &p, size_t compressed_
   return 0;
 }
 
-int ZlibCompressor::decompress(const bufferlist &in, bufferlist &out)
+int ZlibCompressor::decompress(const bufferlist &in, bufferlist &out, boost::optional<int32_t> compressor_message)
 {
 #ifdef HAVE_QATZIP
   if (qat_enabled)
-    return qat_accel.decompress(in, out);
+    return qat_accel.decompress(in, out, compressor_message);
 #endif
   auto i = std::cbegin(in);
-  return decompress(i, in.length(), out);
+  return decompress(i, in.length(), out, compressor_message);
 }
diff --git a/src/compressor/zlib/ZlibCompressor.h b/src/compressor/zlib/ZlibCompressor.h
index b26d0c2fa98..b8e984110ad 100644
--- a/src/compressor/zlib/ZlibCompressor.h
+++ b/src/compressor/zlib/ZlibCompressor.h
@@ -34,12 +34,12 @@ public:
 #endif
   }
 
-  int compress(const ceph::buffer::list &in, ceph::buffer::list &out) override;
-  int decompress(const ceph::buffer::list &in, ceph::buffer::list &out) override;
-  int decompress(ceph::buffer::list::const_iterator &p, size_t compressed_len, ceph::buffer::list &out) override;
+  int compress(const ceph::buffer::list &in, ceph::buffer::list &out, boost::optional<int32_t> &compressor_message) override;
+  int decompress(const ceph::buffer::list &in, ceph::buffer::list &out, boost::optional<int32_t> compressor_message) override;
+  int decompress(ceph::buffer::list::const_iterator &p, size_t compressed_len, ceph::buffer::list &out, boost::optional<int32_t> compressor_message) override;
 private:
-  int zlib_compress(const ceph::buffer::list &in, ceph::buffer::list &out);
-  int isal_compress(const ceph::buffer::list &in, ceph::buffer::list &out);
+  int zlib_compress(const ceph::buffer::list &in, ceph::buffer::list &out, boost::optional<int32_t> &compressor_message);
+  int isal_compress(const ceph::buffer::list &in, ceph::buffer::list &out, boost::optional<int32_t> &compressor_message);
  };
 
 
diff --git a/src/compressor/zstd/ZstdCompressor.h b/src/compressor/zstd/ZstdCompressor.h
index b59f3314fb4..95b492deb87 100644
--- a/src/compressor/zstd/ZstdCompressor.h
+++ b/src/compressor/zstd/ZstdCompressor.h
@@ -26,7 +26,7 @@ class ZstdCompressor : public Compressor {
  public:
   ZstdCompressor(CephContext *cct) : Compressor(COMP_ALG_ZSTD, "zstd"), cct(cct) {}
 
-  int compress(const ceph::buffer::list &src, ceph::buffer::list &dst) override {
+  int compress(const ceph::buffer::list &src, ceph::buffer::list &dst, boost::optional<int32_t> &compressor_message) override {
     ZSTD_CStream *s = ZSTD_createCStream();
     ZSTD_initCStream_srcSize(s, cct->_conf->compressor_zstd_level, src.length());
     auto p = src.begin();
@@ -61,14 +61,15 @@ class ZstdCompressor : public Compressor {
     return 0;
   }
 
-  int decompress(const ceph::buffer::list &src, ceph::buffer::list &dst) override {
+  int decompress(const ceph::buffer::list &src, ceph::buffer::list &dst, boost::optional<int32_t> compressor_message) override {
     auto i = std::cbegin(src);
-    return decompress(i, src.length(), dst);
+    return decompress(i, src.length(), dst, compressor_message);
   }
 
   int decompress(ceph::buffer::list::const_iterator &p,
 		 size_t compressed_len,
-		 ceph::buffer::list &dst) override {
+		 ceph::buffer::list &dst,
+		 boost::optional<int32_t> compressor_message) override {
     if (compressed_len < 4) {
       return -1;
     }