librbd: crypto alignment support

This commit extents the recently added CryptoObjectDispatch layer to support block ciphers (as opposed to stream ciphers).
In other words, the crypto layer now handles the necessary data alignments before data is passed on to the crypto engine.

Signed-off-by: Or Ozeri <oro@il.ibm.com>

1 files changed, 371 insertions, 24 deletions

diff --git a/src/test/librbd/crypto/test_mock_CryptoObjectDispatch.cc b/src/test/librbd/crypto/test_mock_CryptoObjectDispatch.cc
index 658b442f466..181cc2caf94 100644
--- a/src/test/librbd/crypto/test_mock_CryptoObjectDispatch.cc
+++ b/src/test/librbd/crypto/test_mock_CryptoObjectDispatch.cc
@@ -7,6 +7,7 @@
 #include "test/librbd/mock/crypto/MockCryptoInterface.h"
 #include "librbd/crypto/CryptoObjectDispatch.h"
 #include "librbd/io/ObjectDispatchSpec.h"
+#include "librbd/io/Utils.h"
 
 namespace librbd {
 
@@ -17,6 +18,40 @@ inline ImageCtx *get_image_ctx(MockImageCtx *image_ctx) {
 }
 
 } // namespace util
+
+namespace io {
+namespace util {
+
+namespace {
+
+struct Mock {
+    static Mock* s_instance;
+
+    Mock() {
+      s_instance = this;
+    }
+
+    MOCK_METHOD6(read_parent,
+            void(MockImageCtx*, uint64_t, io::ReadExtents*,
+                 librados::snap_t, const ZTracer::Trace &, Context*));
+};
+
+Mock *Mock::s_instance = nullptr;
+
+} // anonymous namespace
+
+template <> void read_parent(
+        MockImageCtx *image_ctx, uint64_t object_no,
+        io::ReadExtents* extents, librados::snap_t snap_id,
+        const ZTracer::Trace &trace, Context* on_finish) {
+
+  Mock::s_instance->read_parent(image_ctx, object_no, extents, snap_id, trace,
+                                on_finish);
+}
+
+} // namespace util
+} // namespace io
+
 } // namespace librbd
 
 #include "librbd/crypto/CryptoObjectDispatch.cc"
@@ -25,10 +60,14 @@ namespace librbd {
 namespace crypto {
 
 using ::testing::_;
+using ::testing::ElementsAre;
 using ::testing::Invoke;
+using ::testing::Pair;
+using ::testing::WithArg;
 
 struct TestMockCryptoCryptoObjectDispatch : public TestMockFixture {
   typedef CryptoObjectDispatch<librbd::MockImageCtx> MockCryptoObjectDispatch;
+  typedef io::util::Mock MockUtils;
 
   MockCryptoInterface* crypto;
   MockImageCtx* mock_image_ctx;
@@ -43,6 +82,7 @@ struct TestMockCryptoCryptoObjectDispatch : public TestMockFixture {
   io::DispatchResult dispatch_result;
   io::Extents extent_map;
   int object_dispatch_flags = 0;
+  MockUtils mock_utils;
 
   void SetUp() override {
     TestMockFixture::SetUp();
@@ -52,7 +92,7 @@ struct TestMockCryptoCryptoObjectDispatch : public TestMockFixture {
     crypto = new MockCryptoInterface();
     mock_image_ctx = new MockImageCtx(*ictx);
     mock_crypto_object_dispatch = new MockCryptoObjectDispatch(mock_image_ctx, crypto);
-    data.append("X");
+    data.append(std::string(4096, '1'));
   }
 
   void TearDown() override {
@@ -66,25 +106,54 @@ struct TestMockCryptoCryptoObjectDispatch : public TestMockFixture {
     TestMockFixture::TearDown();
   }
 
-  void expect_object_read() {
+  void expect_object_read(io::ReadExtents* extents, uint64_t version = 0) {
     EXPECT_CALL(*mock_image_ctx->io_object_dispatcher, send(_))
-            .WillOnce(Invoke([this](io::ObjectDispatchSpec* spec) {
+            .WillOnce(Invoke([this, extents,
+                              version](io::ObjectDispatchSpec* spec) {
                 auto* read = boost::get<io::ObjectDispatchSpec::ReadRequest>(
                         &spec->request);
                 ASSERT_TRUE(read != nullptr);
 
+                ASSERT_EQ(extents->size(), read->extents->size());
+                for (uint64_t i = 0; i < extents->size(); ++i) {
+                  ASSERT_EQ((*extents)[i].offset, (*read->extents)[i].offset);
+                  ASSERT_EQ((*extents)[i].length, (*read->extents)[i].length);
+                  (*read->extents)[i].bl = (*extents)[i].bl;
+                  (*read->extents)[i].extent_map = (*extents)[i].extent_map;
+                }
+
+                if (read->version != nullptr) {
+                  *(read->version) = version;
+                }
+
                 spec->dispatch_result = io::DISPATCH_RESULT_COMPLETE;
                 dispatcher_ctx = &spec->dispatcher_ctx;
             }));
   }
 
-  void expect_object_write() {
+  void expect_read_parent(MockUtils &mock_utils, uint64_t object_no,
+                          io::ReadExtents* extents, librados::snap_t snap_id,
+                          int r) {
+    EXPECT_CALL(mock_utils,
+                read_parent(_, object_no, extents, snap_id, _, _))
+            .WillOnce(WithArg<5>(CompleteContext(r, static_cast<asio::ContextWQ*>(nullptr))));
+  }
+
+  void expect_object_write(uint64_t object_off, const std::string& data,
+                           int write_flags,
+                           std::optional<uint64_t> assert_version) {
     EXPECT_CALL(*mock_image_ctx->io_object_dispatcher, send(_))
-            .WillOnce(Invoke([this](io::ObjectDispatchSpec* spec) {
+            .WillOnce(Invoke([this, object_off, data, write_flags,
+                              assert_version](io::ObjectDispatchSpec* spec) {
                 auto* write = boost::get<io::ObjectDispatchSpec::WriteRequest>(
                         &spec->request);
                 ASSERT_TRUE(write != nullptr);
 
+                ASSERT_EQ(object_off, write->object_off);
+                ASSERT_TRUE(data == write->data.to_str());
+                ASSERT_EQ(write_flags, write->write_flags);
+                ASSERT_EQ(assert_version, write->assert_version);
+
                 spec->dispatch_result = io::DISPATCH_RESULT_COMPLETE;
                 dispatcher_ctx = &spec->dispatcher_ctx;
             }));
@@ -133,38 +202,122 @@ TEST_F(TestMockCryptoCryptoObjectDispatch, Discard) {
   ASSERT_EQ(0, dispatched_cond.wait());
 }
 
-TEST_F(TestMockCryptoCryptoObjectDispatch, ReadFail) {
-  expect_object_read();
+TEST_F(TestMockCryptoCryptoObjectDispatch, AlignedReadFail) {
   io::ReadExtents extents = {{0, 4096}};
+  expect_object_read(&extents);
   ASSERT_TRUE(mock_crypto_object_dispatch->read(
       0, &extents, mock_image_ctx->get_data_io_context(), 0, 0, {},
       nullptr, &object_dispatch_flags, &dispatch_result,
       &on_finish, on_dispatched));
   ASSERT_EQ(dispatch_result, io::DISPATCH_RESULT_COMPLETE);
-  ASSERT_NE(on_finish, &finished_cond);
+  ASSERT_EQ(on_finish, &finished_cond);
 
   ASSERT_EQ(ETIMEDOUT, dispatched_cond.wait_for(0));
   dispatcher_ctx->complete(-EIO);
   ASSERT_EQ(-EIO, dispatched_cond.wait());
 }
 
-TEST_F(TestMockCryptoCryptoObjectDispatch, Read) {
-  expect_object_read();
-  io::ReadExtents extents = {{0, 4096}, {8192, 4096}};
+TEST_F(TestMockCryptoCryptoObjectDispatch, AlignedRead) {
+  io::ReadExtents extents = {{0, 16384}, {32768, 4096}};
+  extents[0].bl.append(std::string(1024, '1') + std::string(1024, '2') +
+                       std::string(1024, '3') + std::string(1024, '4'));
+  extents[0].extent_map = {{1024, 1024}, {3072, 2048}, {16384 - 1024, 1024}};
+  extents[1].bl.append(std::string(4096, '0'));
+  expect_object_read(&extents);
   ASSERT_TRUE(mock_crypto_object_dispatch->read(
           0, &extents, mock_image_ctx->get_data_io_context(), 0, 0, {},
           nullptr, &object_dispatch_flags, &dispatch_result,
           &on_finish, on_dispatched));
   ASSERT_EQ(dispatch_result, io::DISPATCH_RESULT_COMPLETE);
-  ASSERT_NE(on_finish, &finished_cond);
+  ASSERT_EQ(on_finish, &finished_cond);
   ASSERT_EQ(ETIMEDOUT, dispatched_cond.wait_for(0));
 
-  expect_decrypt(2);
+  expect_decrypt(3);
   dispatcher_ctx->complete(0);
+  ASSERT_EQ(16384 + 4096, dispatched_cond.wait());
+
+  auto expected_bl_data = (
+          std::string(1024, '\0') + std::string(1024, '1') +
+          std::string(1024, '\0') + std::string(1024, '2') +
+          std::string(1024, '3') + std::string(3072, '\0') +
+          std::string(3072, '\0') + std::string(1024, '4'));
+  ASSERT_TRUE(extents[0].bl.to_str() == expected_bl_data);
+  ASSERT_THAT(extents[0].extent_map,
+              ElementsAre(Pair(0, 8192), Pair(16384 - 4096, 4096)));
+}
+
+TEST_F(TestMockCryptoCryptoObjectDispatch, ReadFromParent) {
+  io::ReadExtents extents = {{0, 4096}, {8192, 4096}};
+  expect_object_read(&extents);
+  expect_read_parent(mock_utils, 0, &extents, CEPH_NOSNAP, 8192);
+  ASSERT_TRUE(mock_crypto_object_dispatch->read(
+          0, &extents, mock_image_ctx->get_data_io_context(), 0, 0, {},
+          nullptr, &object_dispatch_flags, &dispatch_result,
+          &on_finish, on_dispatched));
+  ASSERT_EQ(dispatch_result, io::DISPATCH_RESULT_COMPLETE);
+  ASSERT_EQ(on_finish, &finished_cond);
+  ASSERT_EQ(ETIMEDOUT, dispatched_cond.wait_for(0));
+
+  // no decrypt
+  dispatcher_ctx->complete(-ENOENT);
   ASSERT_EQ(8192, dispatched_cond.wait());
 }
 
-TEST_F(TestMockCryptoCryptoObjectDispatch, Write) {
+TEST_F(TestMockCryptoCryptoObjectDispatch, ReadFromParentDisabled) {
+  io::ReadExtents extents = {{0, 4096}, {8192, 4096}};
+  expect_object_read(&extents);
+  ASSERT_TRUE(mock_crypto_object_dispatch->read(
+          0, &extents, mock_image_ctx->get_data_io_context(), 0,
+          io::READ_FLAG_DISABLE_READ_FROM_PARENT, {},
+          nullptr, &object_dispatch_flags, &dispatch_result,
+          &on_finish, on_dispatched));
+  ASSERT_EQ(dispatch_result, io::DISPATCH_RESULT_COMPLETE);
+  ASSERT_EQ(on_finish, &finished_cond);
+  ASSERT_EQ(ETIMEDOUT, dispatched_cond.wait_for(0));
+
+  // no decrypt
+  dispatcher_ctx->complete(-ENOENT);
+  ASSERT_EQ(-ENOENT, dispatched_cond.wait());
+}
+
+TEST_F(TestMockCryptoCryptoObjectDispatch, UnalignedRead) {
+  io::ReadExtents extents = {{0, 1}, {8191, 1}, {8193, 1},
+                             {16384 + 1, 4096 * 5 - 2}};
+  io::ReadExtents aligned_extents = {{0, 4096}, {4096, 4096}, {8192, 4096},
+                                     {16384, 4096 * 5}};
+  aligned_extents[0].bl.append(std::string("1") + std::string(4096, '0'));
+  aligned_extents[1].bl.append(std::string(4095, '0') + std::string("2"));
+  aligned_extents[2].bl.append(std::string("03") + std::string(4094, '0'));
+  aligned_extents[3].bl.append(std::string("0") + std::string(4095, '4') +
+                               std::string(4096, '5') +
+                               std::string(4095, '6') + std::string("0"));
+  aligned_extents[3].extent_map = {{16384, 4096}, {16384 + 2 * 4096, 4096},
+                                   {16384 + 4 * 4096, 4096}};
+
+  expect_object_read(&aligned_extents);
+  ASSERT_TRUE(mock_crypto_object_dispatch->read(
+          0, &extents, mock_image_ctx->get_data_io_context(), 0, 0, {},
+          nullptr, &object_dispatch_flags, &dispatch_result,
+          &on_finish, on_dispatched));
+  ASSERT_EQ(dispatch_result, io::DISPATCH_RESULT_COMPLETE);
+  ASSERT_EQ(on_finish, &finished_cond);
+  ASSERT_EQ(ETIMEDOUT, dispatched_cond.wait_for(0));
+
+  dispatcher_ctx->complete(0);
+  ASSERT_EQ(3 + 4096 * 5 - 2, dispatched_cond.wait());
+  ASSERT_TRUE(extents[0].bl.to_str() == std::string("1"));
+  ASSERT_TRUE(extents[1].bl.to_str() == std::string("2"));
+  ASSERT_TRUE(extents[2].bl.to_str() == std::string("3"));
+
+  auto expected_bl_data = (std::string(4095, '4') + std::string(4096, '5') +
+                           std::string(4095, '6'));
+  ASSERT_TRUE(extents[3].bl.to_str() == expected_bl_data);
+  ASSERT_THAT(extents[3].extent_map,
+              ElementsAre(Pair(16384 + 1, 4095), Pair(16384 + 2 * 4096, 4096),
+                          Pair(16384 + 4 * 4096, 4095)));
+}
+
+TEST_F(TestMockCryptoCryptoObjectDispatch, AlignedWrite) {
   expect_encrypt();
   ASSERT_TRUE(mock_crypto_object_dispatch->write(
         0, 0, std::move(data), mock_image_ctx->get_data_io_context(), 0, 0,
@@ -176,26 +329,220 @@ TEST_F(TestMockCryptoCryptoObjectDispatch, Write) {
   ASSERT_EQ(0, finished_cond.wait());
 }
 
+TEST_F(TestMockCryptoCryptoObjectDispatch, UnalignedWrite) {
+  ceph::bufferlist write_data;
+  uint64_t version = 1234;
+  write_data.append(std::string(8192, '1'));
+  io::ReadExtents extents = {{0, 4096}, {8192, 4096}};
+  extents[0].bl.append(std::string(4096, '2'));
+  extents[1].bl.append(std::string(4096, '3'));
+  expect_object_read(&extents, version);
+  ASSERT_TRUE(mock_crypto_object_dispatch->write(
+          0, 1, std::move(write_data), mock_image_ctx->get_data_io_context(),
+          0, 0, std::nullopt, {}, nullptr, nullptr, &dispatch_result,
+          &on_finish, on_dispatched));
+  ASSERT_EQ(dispatch_result, io::DISPATCH_RESULT_COMPLETE);
+  ASSERT_EQ(on_finish, &finished_cond);
+
+  auto expected_data =
+          std::string("2") + std::string(8192, '1') + std::string(4095, '3');
+  expect_object_write(0, expected_data, 0, std::make_optional(version));
+  dispatcher_ctx->complete(0); // complete read
+  ASSERT_EQ(ETIMEDOUT, dispatched_cond.wait_for(0));
+  dispatcher_ctx->complete(0); // complete write
+  ASSERT_EQ(0, dispatched_cond.wait());
+}
+
+TEST_F(TestMockCryptoCryptoObjectDispatch, UnalignedWriteWithNoObject) {
+  ceph::bufferlist write_data;
+  write_data.append(std::string(8192, '1'));
+  io::ReadExtents extents = {{0, 4096}, {8192, 4096}};
+  expect_object_read(&extents);
+  ASSERT_TRUE(mock_crypto_object_dispatch->write(
+          0, 1, std::move(write_data), mock_image_ctx->get_data_io_context(),
+          0, 0, std::nullopt, {}, nullptr, nullptr, &dispatch_result,
+          &on_finish, on_dispatched));
+  ASSERT_EQ(dispatch_result, io::DISPATCH_RESULT_COMPLETE);
+  ASSERT_EQ(on_finish, &finished_cond);
+
+  auto expected_data = (std::string(1, '\0') + std::string(8192, '1') +
+                        std::string(4095, '\0'));
+  expect_object_write(0, expected_data, io::OBJECT_WRITE_FLAG_CREATE_EXCLUSIVE,
+                      std::nullopt);
+  dispatcher_ctx->complete(-ENOENT); // complete read
+  ASSERT_EQ(ETIMEDOUT, dispatched_cond.wait_for(0));
+  dispatcher_ctx->complete(0); // complete write
+  ASSERT_EQ(0, dispatched_cond.wait());
+}
+
+TEST_F(TestMockCryptoCryptoObjectDispatch, UnalignedWriteFailCreate) {
+  ceph::bufferlist write_data;
+  write_data.append(std::string(8192, '1'));
+  io::ReadExtents extents = {{0, 4096}, {8192, 4096}};
+  expect_object_read(&extents);
+  ASSERT_TRUE(mock_crypto_object_dispatch->write(
+          0, 1, std::move(write_data), mock_image_ctx->get_data_io_context(),
+          0, 0, std::nullopt, {}, nullptr, nullptr, &dispatch_result,
+          &on_finish, on_dispatched));
+  ASSERT_EQ(dispatch_result, io::DISPATCH_RESULT_COMPLETE);
+  ASSERT_EQ(on_finish, &finished_cond);
+
+  auto expected_data = (std::string(1, '\0') + std::string(8192, '1') +
+                        std::string(4095, '\0'));
+  expect_object_write(0, expected_data, io::OBJECT_WRITE_FLAG_CREATE_EXCLUSIVE,
+          std::nullopt);
+  dispatcher_ctx->complete(-ENOENT); // complete read
+  ASSERT_EQ(ETIMEDOUT, dispatched_cond.wait_for(0));
+
+  extents[0].bl.append(std::string(4096, '2'));
+  extents[1].bl.append(std::string(4096, '3'));
+  uint64_t version = 1234;
+  expect_object_read(&extents, version);
+  dispatcher_ctx->complete(-EEXIST); // complete write, request will restart
+  ASSERT_EQ(ETIMEDOUT, dispatched_cond.wait_for(0));
+
+  auto expected_data2 =
+        std::string("2") + std::string(8192, '1') + std::string(4095, '3');
+  expect_object_write(0, expected_data2, 0, std::make_optional(version));
+  dispatcher_ctx->complete(0); // complete read
+  ASSERT_EQ(ETIMEDOUT, dispatched_cond.wait_for(0));
+  dispatcher_ctx->complete(0); // complete write
+  ASSERT_EQ(0, dispatched_cond.wait());
+}
+
+TEST_F(TestMockCryptoCryptoObjectDispatch, UnalignedWriteFailVersionCheck) {
+  ceph::bufferlist write_data;
+  uint64_t version = 1234;
+  write_data.append(std::string(8192, '1'));
+  io::ReadExtents extents = {{0, 4096}, {8192, 4096}};
+  extents[0].bl.append(std::string(4096, '2'));
+  extents[1].bl.append(std::string(4096, '3'));
+  expect_object_read(&extents, version);
+  ASSERT_TRUE(mock_crypto_object_dispatch->write(
+          0, 1, std::move(write_data), mock_image_ctx->get_data_io_context(),
+          0, 0, std::nullopt, {}, nullptr, nullptr, &dispatch_result,
+          &on_finish, on_dispatched));
+  ASSERT_EQ(dispatch_result, io::DISPATCH_RESULT_COMPLETE);
+  ASSERT_EQ(on_finish, &finished_cond);
+
+  auto expected_data =
+          std::string("2") + std::string(8192, '1') + std::string(4095, '3');
+  expect_object_write(0, expected_data, 0, std::make_optional(version));
+  dispatcher_ctx->complete(0); // complete read
+  ASSERT_EQ(ETIMEDOUT, dispatched_cond.wait_for(0));
+
+  version = 1235;
+  expect_object_read(&extents, version);
+  extents[0].bl.append(std::string(4096, '2'));
+  extents[1].bl.append(std::string(4096, '3'));
+  dispatcher_ctx->complete(-ERANGE); // complete write, request will restart
+  ASSERT_EQ(ETIMEDOUT, dispatched_cond.wait_for(0));
+
+  expect_object_write(0, expected_data, 0, std::make_optional(version));
+  dispatcher_ctx->complete(0); // complete read
+  ASSERT_EQ(ETIMEDOUT, dispatched_cond.wait_for(0));
+  dispatcher_ctx->complete(0); // complete write
+  ASSERT_EQ(0, dispatched_cond.wait());
+}
+
+TEST_F(TestMockCryptoCryptoObjectDispatch, UnalignedWriteWithAssertVersion) {
+  ceph::bufferlist write_data;
+  uint64_t version = 1234;
+  uint64_t assert_version = 1233;
+  write_data.append(std::string(8192, '1'));
+  io::ReadExtents extents = {{0, 4096}, {8192, 4096}};
+  extents[0].bl.append(std::string(4096, '2'));
+  extents[1].bl.append(std::string(4096, '3'));
+  expect_object_read(&extents, version);
+  ASSERT_TRUE(mock_crypto_object_dispatch->write(
+          0, 1, std::move(write_data), mock_image_ctx->get_data_io_context(),
+          0, 0, std::make_optional(assert_version), {}, nullptr, nullptr,
+          &dispatch_result, &on_finish, on_dispatched));
+  ASSERT_EQ(dispatch_result, io::DISPATCH_RESULT_COMPLETE);
+  ASSERT_EQ(on_finish, &finished_cond);
+
+  dispatcher_ctx->complete(0); // complete read
+  ASSERT_EQ(-ERANGE, dispatched_cond.wait());
+}
+
+TEST_F(TestMockCryptoCryptoObjectDispatch, UnalignedWriteWithExclusiveCreate) {
+  ceph::bufferlist write_data;
+  write_data.append(std::string(8192, '1'));
+  io::ReadExtents extents = {{0, 4096}, {8192, 4096}};
+  extents[0].bl.append(std::string(4096, '2'));
+  extents[1].bl.append(std::string(4096, '3'));
+  expect_object_read(&extents);
+  ASSERT_TRUE(mock_crypto_object_dispatch->write(
+          0, 1, std::move(write_data), mock_image_ctx->get_data_io_context(),
+          0, io::OBJECT_WRITE_FLAG_CREATE_EXCLUSIVE, std::nullopt, {}, nullptr,
+          nullptr, &dispatch_result, &on_finish, on_dispatched));
+  ASSERT_EQ(dispatch_result, io::DISPATCH_RESULT_COMPLETE);
+  ASSERT_EQ(on_finish, &finished_cond);
+
+  dispatcher_ctx->complete(0); // complete read
+  ASSERT_EQ(-EEXIST, dispatched_cond.wait());
+}
+
 TEST_F(TestMockCryptoCryptoObjectDispatch, CompareAndWrite) {
-  ceph::bufferlist cmp_data(data);
+  ceph::bufferlist write_data;
+  uint64_t version = 1234;
+  write_data.append(std::string(8192, '1'));
+  ceph::bufferlist cmp_data;
+  cmp_data.append(std::string(4096, '2'));
+  io::ReadExtents extents = {{0, 4096}, {8192, 4096}, {0, 8192}};
+  extents[0].bl.append(std::string(4096, '2'));
+  extents[1].bl.append(std::string(4096, '3'));
+  extents[2].bl.append(std::string(8192, '2'));
+  expect_object_read(&extents, version);
 
-  expect_encrypt(2);
   ASSERT_TRUE(mock_crypto_object_dispatch->compare_and_write(
-          0, 0, std::move(cmp_data), std::move(data),
+          0, 1, std::move(cmp_data), std::move(write_data),
           mock_image_ctx->get_data_io_context(), 0, {}, nullptr, nullptr,
           nullptr, &dispatch_result, &on_finish, on_dispatched));
-  ASSERT_EQ(dispatch_result, io::DISPATCH_RESULT_CONTINUE);
-  ASSERT_EQ(on_finish, &finished_cond); // not modified
-  on_finish->complete(0);
-  ASSERT_EQ(0, finished_cond.wait());
+  ASSERT_EQ(dispatch_result, io::DISPATCH_RESULT_COMPLETE);
+  ASSERT_EQ(on_finish, &finished_cond);
+
+  auto expected_data =
+          std::string("2") + std::string(8192, '1') + std::string(4095, '3');
+  expect_object_write(0, expected_data, 0, std::make_optional(version));
+  dispatcher_ctx->complete(0); // complete read
+  ASSERT_EQ(ETIMEDOUT, dispatched_cond.wait_for(0));
+  dispatcher_ctx->complete(0); // complete write
+  ASSERT_EQ(0, dispatched_cond.wait());
+}
+
+TEST_F(TestMockCryptoCryptoObjectDispatch, CompareAndWriteFail) {
+  ceph::bufferlist write_data;
+  uint64_t version = 1234;
+  write_data.append(std::string(8192, '1'));
+  ceph::bufferlist cmp_data;
+  cmp_data.append(std::string(4094, '2') + std::string(2, '4'));
+  io::ReadExtents extents = {{0, 4096}, {8192, 4096}, {0, 8192}};
+  extents[0].bl.append(std::string(4096, '2'));
+  extents[1].bl.append(std::string(4096, '3'));
+  extents[2].bl.append(std::string(8192, '2'));
+  expect_object_read(&extents, version);
+
+  uint64_t mismatch_offset;
+  ASSERT_TRUE(mock_crypto_object_dispatch->compare_and_write(
+          0, 1, std::move(cmp_data), std::move(write_data),
+          mock_image_ctx->get_data_io_context(), 0, {}, &mismatch_offset,
+          nullptr, nullptr, &dispatch_result, &on_finish, on_dispatched));
+  ASSERT_EQ(dispatch_result, io::DISPATCH_RESULT_COMPLETE);
+  ASSERT_EQ(on_finish, &finished_cond);
+
+  dispatcher_ctx->complete(0); // complete read
+  ASSERT_EQ(-EILSEQ, dispatched_cond.wait());
+  ASSERT_EQ(mismatch_offset, 4094);
 }
 
 TEST_F(TestMockCryptoCryptoObjectDispatch, WriteSame) {
   io::LightweightBufferExtents buffer_extents;
-
-  expect_object_write();
+  ceph::bufferlist write_data;
+  write_data.append(std::string("12"));
+  expect_object_write(0, std::string("12121") , 0, std::nullopt);
   ASSERT_TRUE(mock_crypto_object_dispatch->write_same(
-          0, 0, 4096, std::move(buffer_extents), std::move(data),
+          0, 0, 5, {{0, 5}}, std::move(write_data),
           mock_image_ctx->get_data_io_context(), 0, {}, nullptr, nullptr,
           &dispatch_result, &on_finish, on_dispatched));
   ASSERT_EQ(dispatch_result, io::DISPATCH_RESULT_COMPLETE);