[#3655] Better handle maintenance cancel

When the maintenance is canceled the partners communicate their new
states. This is essential when the partners receive DHCP traffic while
transitioning to avoid rapid return to the maintenance state.

8 files changed, 318 insertions, 19 deletions

diff --git a/src/hooks/dhcp/high_availability/command_creator.cc b/src/hooks/dhcp/high_availability/command_creator.cc
index 933a922814..d966441d2b 100644
--- a/src/hooks/dhcp/high_availability/command_creator.cc
+++ b/src/hooks/dhcp/high_availability/command_creator.cc
@@ -7,6 +7,7 @@
 #include <config.h>
 
 #include <command_creator.h>
+#include <ha_service_states.h>
 #include <cc/command_interpreter.h>
 #include <exceptions/exceptions.h>
 #include <boost/pointer_cast.hpp>
@@ -259,10 +260,12 @@ CommandCreator::createLease6GetPage(const Lease6Ptr& last_lease6,
 ConstElementPtr
 CommandCreator::createMaintenanceNotify(const std::string& server_name,
                                         const bool cancel,
+                                        const int state,
                                         const HAServerType& server_type) {
     auto args = Element::createMap();
     args->set("server-name", Element::create(server_name));
     args->set("cancel", Element::create(cancel));
+    args->set("state", Element::create(stateToString(state)));
     auto command = config::createCommand("ha-maintenance-notify", args);
     insertService(command, server_type);
     return (command);
diff --git a/src/hooks/dhcp/high_availability/command_creator.h b/src/hooks/dhcp/high_availability/command_creator.h
index f06065de7b..34c5065e6f 100644
--- a/src/hooks/dhcp/high_availability/command_creator.h
+++ b/src/hooks/dhcp/high_availability/command_creator.h
@@ -176,11 +176,13 @@ public:
     /// for associating the command with the relationship.
     /// @param cancel boolean value indicating if the maintenance
     /// is being cancelled (true) or requested (false).
+    /// @param state server state.
     /// @param server_type type of the DHCP server, i.e. v4 or v6.
     /// @return Pointer to the JSON representation of the command.
     static data::ConstElementPtr
     createMaintenanceNotify(const std::string& server_name,
                             const bool cancel,
+                            const int state,
                             const HAServerType& server_type);
 
     /// @brief Creates ha-sync-complete-notify command.
diff --git a/src/hooks/dhcp/high_availability/ha_impl.cc b/src/hooks/dhcp/high_availability/ha_impl.cc
index 495f93b260..ab542e0eed 100644
--- a/src/hooks/dhcp/high_availability/ha_impl.cc
+++ b/src/hooks/dhcp/high_availability/ha_impl.cc
@@ -789,9 +789,15 @@ HAImpl::maintenanceNotifyHandler(hooks::CalloutHandle& callout_handle) {
             isc_throw(BadValue, "'cancel' must be a boolean in the 'ha-maintenance-notify' command");
         }
 
+        ConstElementPtr state = args->get("state");
+        if (state && state->getType() != Element::string) {
+            isc_throw(BadValue, "'state' must be a string in the 'ha-maintenance-notify' command");
+        }
+
         service = getHAServiceByServerName("ha-maintenance-notify", args);
 
-        ConstElementPtr response = service->processMaintenanceNotify(cancel_op->boolValue());
+        ConstElementPtr response = service->processMaintenanceNotify(cancel_op->boolValue(),
+                                                                     state ? state->stringValue() : "unavailable");
         callout_handle.setArgument("response", response);
 
     } catch (const std::exception& ex) {
diff --git a/src/hooks/dhcp/high_availability/ha_service.cc b/src/hooks/dhcp/high_availability/ha_service.cc
index a3436ffab3..d93b556394 100644
--- a/src/hooks/dhcp/high_availability/ha_service.cc
+++ b/src/hooks/dhcp/high_availability/ha_service.cc
@@ -1076,7 +1076,7 @@ HAService::inScopeInternal(QueryPtrType& query) {
     query->addClass(dhcp::ClientClass(scope_class));
     // The following is the part of the server failure detection algorithm.
     // If the query should be processed by the partner we need to check if
-    // the partner responds. If the number of unansweered queries exceeds a
+    // the partner responds. If the number of unanswered queries exceeds a
     // configured threshold, we will consider the partner to be offline.
     if (!in_scope && communication_state_->isCommunicationInterrupted()) {
         communication_state_->analyzeMessage(query);
@@ -2743,7 +2743,7 @@ HAService::processContinue() {
 }
 
 ConstElementPtr
-HAService::processMaintenanceNotify(const bool cancel) {
+HAService::processMaintenanceNotify(const bool cancel, const std::string& state) {
     if (cancel) {
         if (getCurrState() != HA_IN_MAINTENANCE_ST) {
             return (createAnswer(CONTROL_RESULT_ERROR, "Unable to cancel the"
@@ -2751,10 +2751,28 @@ HAService::processMaintenanceNotify(const bool cancel) {
                                  " in-maintenance state."));
         }
 
+        try {
+            communication_state_->setPartnerState(state);
+
+        } catch (...) {
+            // Hopefully the received state is correct. If it isn't, let's set the
+            // partner state to unavailable and count on the state machine to resolve.
+            communication_state_->setPartnerUnavailable();
+        }
         postNextEvent(HA_MAINTENANCE_CANCEL_EVT);
-        verboseTransition(getPrevState());
+        // In rare cases the previous state may be the server's current state. Transitioning
+        // to it would cause a deadlock and the server will remain stuck in maintenance.
+        // In these cases let's simply transition to the waiting state and the state machine
+        // should solve it.
+        verboseTransition(getPrevState() == HA_IN_MAINTENANCE_ST ? HA_WAITING_ST : getPrevState());
         runModel(NOP_EVT);
-        return (createAnswer(CONTROL_RESULT_SUCCESS, "Server maintenance canceled."));
+
+        // Communicate the new state to the partner.
+        ElementPtr arguments = Element::createMap();
+        std::string state_label = getState(getCurrState())->getLabel();
+        arguments->set("state", Element::create(state_label));
+
+        return (createAnswer(CONTROL_RESULT_SUCCESS, "Server maintenance canceled.", arguments));
     }
 
     switch (getCurrState()) {
@@ -2801,7 +2819,7 @@ HAService::processMaintenanceStart() {
          HostHttpHeader(remote_config->getUrl().getStrippedHostname()));
     remote_config->addBasicAuthHttpHeader(request);
     request->setBodyAsJson(CommandCreator::createMaintenanceNotify(config_->getThisServerName(),
-                                                                   false, server_type_));
+                                                                   false, getCurrState(), server_type_));
     request->finalize();
 
     // Response object should also be created because the HTTP client needs
@@ -2932,7 +2950,7 @@ HAService::processMaintenanceCancel() {
          HostHttpHeader(remote_config->getUrl().getStrippedHostname()));
     remote_config->addBasicAuthHttpHeader(request);
     request->setBodyAsJson(CommandCreator::createMaintenanceNotify(config_->getThisServerName(),
-                                                                   true, server_type_));
+                                                                   true, getCurrState(), server_type_));
     request->finalize();
 
     // Response object should also be created because the HTTP client needs
@@ -2968,8 +2986,26 @@ HAService::processMaintenanceCancel() {
                  // Handle third group of errors.
                  try {
                      int rcode = 0;
-                     static_cast<void>(verifyAsyncResponse(response, rcode));
-
+                     ConstElementPtr args = verifyAsyncResponse(response, rcode);
+
+                     // Partner's state has changed after the notification. However, we don't know
+                     // its new state. We'll check if the partner returned its state. If it didn't,
+                     // we set the unavailable state as a default.
+                     communication_state_->setPartnerUnavailable();
+
+                     // Newer Kea versions return the state of the notified server.
+                     // Older versions don't, so the arguments may not be present.
+                     if (args && args->getType() == Element::map) {
+                         // Arguments may include partner's state.
+                         ConstElementPtr state = args->get("state");
+                         if (state) {
+                             if (state->getType() != Element::string) {
+                                 isc_throw(CtrlChannelError, "server state not returned in response"
+                                           " to a ha-heartbeat command or it is not a string");
+                             }
+                             communication_state_->setPartnerState(state->stringValue());
+                         }
+                     }
                  } catch (const std::exception& ex) {
                      error_message = ex.what();
                      LOG_ERROR(ha_logger, HA_MAINTENANCE_NOTIFY_CANCEL_FAILED)
@@ -3009,9 +3045,10 @@ HAService::processMaintenanceCancel() {
     }
 
     // Successfully reverted partner's state. Let's also revert our state to the
-    // previous one.
+    // previous one. Avoid returning to the partner-in-maintenance if it was
+    // the previous state.
     postNextEvent(HA_MAINTENANCE_CANCEL_EVT);
-    verboseTransition(getPrevState());
+    verboseTransition(getPrevState() == HA_PARTNER_IN_MAINTENANCE_ST ? HA_WAITING_ST : getPrevState());
     runModel(NOP_EVT);
 
     return (createAnswer(CONTROL_RESULT_SUCCESS,
diff --git a/src/hooks/dhcp/high_availability/ha_service.h b/src/hooks/dhcp/high_availability/ha_service.h
index 0b6bc3a086..885254ea23 100644
--- a/src/hooks/dhcp/high_availability/ha_service.h
+++ b/src/hooks/dhcp/high_availability/ha_service.h
@@ -1048,9 +1048,12 @@ public:
     /// @param cancel boolean value indicating if the maintenance is being
     /// canceled with this operation. If it is set to false the maintenance
     /// is being started.
+    /// @param state partner's state as string. It should be set to "unavailable"
+    /// if the state was not explicitly provided by the partner.
     ///
     /// @return Pointer to the response to the ha-maintenance-notify.
-    data::ConstElementPtr processMaintenanceNotify(const bool cancel);
+    data::ConstElementPtr processMaintenanceNotify(const bool cancel,
+                                                   const std::string& state);
 
     /// @brief Processes ha-maintenance-start command and returns a response.
     ///
diff --git a/src/hooks/dhcp/high_availability/tests/command_creator_unittest.cc b/src/hooks/dhcp/high_availability/tests/command_creator_unittest.cc
index 0b73181ce1..f6a76b02c3 100644
--- a/src/hooks/dhcp/high_availability/tests/command_creator_unittest.cc
+++ b/src/hooks/dhcp/high_availability/tests/command_creator_unittest.cc
@@ -7,6 +7,7 @@
 #include <config.h>
 
 #include <ha_server_type.h>
+#include <ha_service_states.h>
 #include <lease_update_backlog.h>
 #include <command_creator.h>
 #include <asiolink/io_address.h>
@@ -562,7 +563,8 @@ TEST(CommandCreatorTest, createLease6GetPageZeroLimit) {
 // This test verifies that the ha-maintenance-notify command is correct
 // while being sent to the DHCPv4 server.
 TEST(CommandCreatorTest, createMaintenanceNotify4) {
-    ConstElementPtr command = CommandCreator::createMaintenanceNotify("server1", true, HAServerType::DHCPv4);
+    ConstElementPtr command = CommandCreator::createMaintenanceNotify("server1", true, HA_READY_ST,
+                                                                      HAServerType::DHCPv4);
     ConstElementPtr arguments;
     ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "ha-maintenance-notify", "dhcp4",
                                               arguments));
@@ -571,12 +573,18 @@ TEST(CommandCreatorTest, createMaintenanceNotify4) {
     ASSERT_TRUE(cancel);
     ASSERT_EQ(Element::boolean, cancel->getType());
     EXPECT_TRUE(cancel->boolValue());
+
+    auto state = arguments->get("state");
+    ASSERT_TRUE(state);
+    ASSERT_EQ(Element::string, state->getType());
+    EXPECT_EQ("ready", state->stringValue());
 }
 
 // This test verifies that the ha-maintenance-notify command is correct
 // while being sent to the DHCPv6 server.
 TEST(CommandCreatorTest, createMaintenanceNotify6) {
-    ConstElementPtr command = CommandCreator::createMaintenanceNotify("server1", false, HAServerType::DHCPv6);
+    ConstElementPtr command = CommandCreator::createMaintenanceNotify("server1", false, HA_SYNCING_ST,
+                                                                      HAServerType::DHCPv6);
     ConstElementPtr arguments;
     ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "ha-maintenance-notify", "dhcp6",
                                               arguments));
@@ -585,6 +593,11 @@ TEST(CommandCreatorTest, createMaintenanceNotify6) {
     ASSERT_TRUE(cancel);
     ASSERT_EQ(Element::boolean, cancel->getType());
     EXPECT_FALSE(cancel->boolValue());
+
+    auto state = arguments->get("state");
+    ASSERT_TRUE(state);
+    ASSERT_EQ(Element::string, state->getType());
+    EXPECT_EQ("syncing", state->stringValue());
 }
 
 // This test verifies that the ha-sync-complete-notify command sent to a
diff --git a/src/hooks/dhcp/high_availability/tests/ha_impl_unittest.cc b/src/hooks/dhcp/high_availability/tests/ha_impl_unittest.cc
index f99bc6e993..b5622c5b7a 100644
--- a/src/hooks/dhcp/high_availability/tests/ha_impl_unittest.cc
+++ b/src/hooks/dhcp/high_availability/tests/ha_impl_unittest.cc
@@ -2176,6 +2176,60 @@ TEST_F(HAImplTest, maintenanceNotifyBadServerName) {
     checkAnswer(response, CONTROL_RESULT_ERROR, "server5 matches no configured 'server-name'");
 }
 
+// Test ha-maintenance-notify command handler with partner state.
+TEST_F(HAImplTest, maintenanceNotifyPartnerState) {
+    ha_impl_.reset(new HAImpl());
+    ha_impl_->setIOService(io_service_);
+    ASSERT_NO_THROW(ha_impl_->configure(createValidJsonConfiguration()));
+
+    // Starting the service is required prior to running any callouts.
+    NetworkStatePtr network_state(new NetworkState());
+    ASSERT_NO_THROW(ha_impl_->startServices(network_state,
+                                            HAServerType::DHCPv4));
+
+    // Start the maintenance.
+    ConstElementPtr command = Element::fromJSON(
+        "{"
+        "    \"command\": \"ha-maintenance-notify\","
+        "    \"arguments\": {"
+        "        \"cancel\": false"
+        "    }"
+         "}"
+    );
+
+    CalloutHandlePtr callout_handle = HooksManager::createCalloutHandle();
+    callout_handle->setArgument("command", command);
+
+    ASSERT_NO_THROW(ha_impl_->maintenanceNotifyHandler(*callout_handle));
+
+    ConstElementPtr response;
+    callout_handle->getArgument("response", response);
+    ASSERT_TRUE(response);
+
+    checkAnswer(response, CONTROL_RESULT_SUCCESS, "Server is in-maintenance state.");
+
+    // Cancel the maintenance.
+    command = Element::fromJSON(
+        "{"
+        "    \"command\": \"ha-maintenance-notify\","
+        "    \"arguments\": {"
+        "        \"cancel\": true,"
+        "        \"server-name\": \"server1\","
+        "        \"state\": \"waiting\""
+        "    }"
+         "}"
+    );
+
+    callout_handle->setArgument("command", command);
+
+    ASSERT_NO_THROW(ha_impl_->maintenanceNotifyHandler(*callout_handle));
+
+    callout_handle->getArgument("response", response);
+    ASSERT_TRUE(response);
+
+    checkAnswer(response, CONTROL_RESULT_SUCCESS, "Server maintenance canceled.");
+}
+
 // Test ha-reset command handler with a specified server name.
 TEST_F(HAImplTest, haReset) {
     ha_impl_.reset(new HAImpl());
diff --git a/src/hooks/dhcp/high_availability/tests/ha_service_unittest.cc b/src/hooks/dhcp/high_availability/tests/ha_service_unittest.cc
index 75d6b4d252..6e00f55610 100644
--- a/src/hooks/dhcp/high_availability/tests/ha_service_unittest.cc
+++ b/src/hooks/dhcp/high_availability/tests/ha_service_unittest.cc
@@ -5294,7 +5294,7 @@ TEST_F(HAServiceTest, processMaintenanceNotify) {
         // Process ha-maintenance-notify command that should transition the
         // state machine to the in-maintenance state.
         ConstElementPtr rsp;
-        ASSERT_NO_THROW(rsp = service.processMaintenanceNotify(false));
+        ASSERT_NO_THROW(rsp = service.processMaintenanceNotify(false, "unavailable"));
 
         // The server should have responded.
         ASSERT_TRUE(rsp);
@@ -5304,7 +5304,8 @@ TEST_F(HAServiceTest, processMaintenanceNotify) {
         EXPECT_EQ(HA_IN_MAINTENANCE_ST, service.getCurrState());
 
         // Try to cancel the maintenance.
-        ASSERT_NO_THROW(rsp = service.processMaintenanceNotify(true));
+        ASSERT_NO_THROW(rsp = service.processMaintenanceNotify(true, "unavailable"));
+        EXPECT_EQ(HA_UNAVAILABLE_ST, service.communication_state_->getPartnerState());
 
         // The server should have responded.
         ASSERT_TRUE(rsp);
@@ -5322,12 +5323,13 @@ TEST_F(HAServiceTest, processMaintenanceNotify) {
     };
 
     // Make sure that the transition from the other states is not allowed.
+    service.communication_state_->setPartnerState("waiting");
     for (auto const& state : invalid_states) {
         EXPECT_NO_THROW(service.transition(state, HAService::NOP_EVT));
         EXPECT_NO_THROW(service.runModel(HAService::NOP_EVT));
 
         ConstElementPtr rsp;
-        ASSERT_NO_THROW(rsp = service.processMaintenanceNotify(false));
+        ASSERT_NO_THROW(rsp = service.processMaintenanceNotify(false, "unavailable"));
 
         ASSERT_TRUE(rsp);
         checkAnswer(rsp, TestHAService::HA_CONTROL_RESULT_MAINTENANCE_NOT_ALLOWED,
@@ -5336,6 +5338,71 @@ TEST_F(HAServiceTest, processMaintenanceNotify) {
     }
 }
 
+// This test verifies that the ha-maintenance-notify command is processed
+// and the partner state is set according to the partner's notion of state.
+TEST_F(HAServiceTest, processMaintenanceNotifyCancelSetPartnerState) {
+    HAConfigPtr config_storage = createValidConfiguration();
+
+    TestHAService service(1, io_service_, network_state_, config_storage);
+
+    EXPECT_NO_THROW(service.transition(HA_WAITING_ST, HAService::NOP_EVT));
+
+    // Process ha-maintenance-notify command that should transition the
+    // state machine to the in-maintenance state.
+    ConstElementPtr rsp;
+    ASSERT_NO_THROW(rsp = service.processMaintenanceNotify(false, ""));
+
+    // The server should have responded.
+    ASSERT_TRUE(rsp);
+    checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "Server is in-maintenance state.");
+
+    // The state machine should have been transitioned to the in-maintenance state.
+    EXPECT_EQ(HA_IN_MAINTENANCE_ST, service.getCurrState());
+
+    // Try to cancel the maintenance with specifying the state.
+    ASSERT_NO_THROW(rsp = service.processMaintenanceNotify(true, "ready"));
+    EXPECT_EQ(HA_READY_ST, service.communication_state_->getPartnerState());
+
+    // The state machine should have been transitioned to the state it was in
+    // prior to transitioning to the in-maintenance state.
+    EXPECT_EQ(HA_WAITING_ST, service.getCurrState());
+}
+
+// This test verifies that the ha-maintenance-notify command is processed
+// and the server transitions to the waiting state when its prvious state
+// before the maintenance was in-maintenance.
+TEST_F(HAServiceTest, processMaintenanceNotifyCancelSetAvoidDeadlock) {
+    HAConfigPtr config_storage = createValidConfiguration();
+
+    TestHAService service(1, io_service_, network_state_, config_storage);
+
+    // Set the in-maintenance state. Normally the server returns to a previous
+    // state when the maintenance is canceled. In this case, it doesn't make much
+    // sense because it will cause a deadlock. In this exceptional case the server
+    // should rather transition to a waiting state instead.
+    EXPECT_NO_THROW(service.transition(HA_IN_MAINTENANCE_ST, HAService::NOP_EVT));
+
+    // Process ha-maintenance-notify command that should re-transition the
+    // state machine to the in-maintenance state.
+    ConstElementPtr rsp;
+    ASSERT_NO_THROW(rsp = service.processMaintenanceNotify(false, ""));
+
+    // The server should have responded.
+    ASSERT_TRUE(rsp);
+    checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "Server is in-maintenance state.");
+
+    // The state machine should have been transitioned to the in-maintenance state.
+    EXPECT_EQ(HA_IN_MAINTENANCE_ST, service.getCurrState());
+
+    // Try to cancel the maintenance with specifying the state.
+    ASSERT_NO_THROW(rsp = service.processMaintenanceNotify(true, "ready"));
+    EXPECT_EQ(HA_READY_ST, service.communication_state_->getPartnerState());
+
+    // The state machine should have been transitioned to the state it was in
+    // prior to transitioning to the in-maintenance state.
+    EXPECT_EQ(HA_WAITING_ST, service.getCurrState());
+}
+
 // This test verifies the case when the server receiving the ha-maintenance-start
 // command successfully transitions to the partner-in-maintenance state and its
 // partner transitions to the in-maintenance state.
@@ -5629,6 +5696,10 @@ TEST_F(HAServiceTest, processMaintenanceCancelSuccess) {
     HAConfigPtr config_storage = createValidConfiguration();
     setBasicAuth(config_storage);
 
+    auto response_arguments = Element::createMap();
+    response_arguments->set("state", Element::create("hot-standby"));
+    factory2_->getResponseCreator()->setArguments("ha-maintenance-notify", response_arguments);
+
     // Start the servers.
     ASSERT_NO_THROW({
         listener_->start();
@@ -5657,12 +5728,119 @@ TEST_F(HAServiceTest, processMaintenanceCancelSuccess) {
     }
 
     // The partner of our server is online and should have responded with
-    // the success status. Therefore, this server should have transitioned
-    // to the partner-in-maintenance state.
+    // the success status, and include its new state.
+    ASSERT_TRUE(rsp);
+    checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "Server maintenance successfully canceled.");
+
+    EXPECT_EQ(HA_WAITING_ST, service.getCurrState());
+    EXPECT_EQ(HA_HOT_STANDBY_ST, service.communication_state_->getPartnerState());
+}
+
+// This test verifies the case when the server receiving the ha-maintenance-cancel
+// command does not receive a new state from its partner.
+TEST_F(HAServiceTest, processMaintenanceCancelSuccessPartnerNewStateUnknown) {
+    // Create HA configuration for 3 servers. This server is
+    // server 1.
+    HAConfigPtr config_storage = createValidConfiguration();
+    setBasicAuth(config_storage);
+
+    auto response_arguments = Element::createMap();
+    factory2_->getResponseCreator()->setArguments("ha-maintenance-notify", response_arguments);
+
+    // Start the servers.
+    ASSERT_NO_THROW({
+        listener_->start();
+        listener2_->start();
+    });
+
+    TestHAService service(1, io_service_, network_state_, config_storage);
+
+    // Initial transition to the non-default state. The server should return to
+    // it when the maintenance is canceled.
+    ASSERT_NO_THROW(service.verboseTransition(HA_READY_ST));
+    service.runModel(TestHAService::NOP_EVT);
+
+    ASSERT_NO_THROW(service.verboseTransition(HA_PARTNER_IN_MAINTENANCE_ST));
+
+    // The tested function is synchronous, so we need to run server side IO service
+    // in background to not block the main thread.
+    auto thread = runIOServiceInThread();
+
+    // Process ha-maintenance-cancel command.
+    ConstElementPtr rsp;
+    ASSERT_NO_THROW(rsp = service.processMaintenanceCancel());
+
+    // Stop the IO service. This should cause the thread to terminate.
+    io_service_->stop();
+    thread->join();
+    io_service_->restart();
+    try {
+        io_service_->poll();
+    } catch (...) {
+    }
+
+    // The partner of our server is online and should have responded with
+    // the success status, but not include its new state.
+    ASSERT_TRUE(rsp);
+    checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "Server maintenance successfully canceled.");
+
+    EXPECT_EQ(HA_READY_ST, service.getCurrState());
+    EXPECT_EQ(HA_UNAVAILABLE_ST, service.communication_state_->getPartnerState());
+}
+
+// This test verifies the case when the server receiving the ha-maintenance-cancel
+// had been in the partner-in-maintenance state before the maintenance was started.
+// In this case the server should transition to the waiting state when the maintenance
+// is canceled.
+TEST_F(HAServiceTest, processMaintenanceCancelSuccessAvoidDeadlock) {
+    // Create HA configuration for 3 servers. This server is
+    // server 1.
+    HAConfigPtr config_storage = createValidConfiguration();
+    setBasicAuth(config_storage);
+
+    auto response_arguments = Element::createMap();
+    factory2_->getResponseCreator()->setArguments("ha-maintenance-notify", response_arguments);
+
+    // Start the servers.
+    ASSERT_NO_THROW({
+        listener_->start();
+        listener2_->start();
+    });
+
+    TestHAService service(1, io_service_, network_state_, config_storage);
+
+    // Initial transition to the partner-in-maintenance state. It will be remembered
+    // as the previous state.
+    ASSERT_NO_THROW(service.verboseTransition(HA_PARTNER_IN_MAINTENANCE_ST));
+    service.runModel(TestHAService::NOP_EVT);
+
+    // Transition again and then cancel transition.
+    ASSERT_NO_THROW(service.verboseTransition(HA_PARTNER_IN_MAINTENANCE_ST));
+    // The tested function is synchronous, so we need to run server side IO service
+    // in background to not block the main thread.
+    auto thread = runIOServiceInThread();
+
+    // Process ha-maintenance-cancel command.
+    ConstElementPtr rsp;
+    ASSERT_NO_THROW(rsp = service.processMaintenanceCancel());
+
+    // Stop the IO service. This should cause the thread to terminate.
+    io_service_->stop();
+    thread->join();
+    io_service_->restart();
+    try {
+        io_service_->poll();
+    } catch (...) {
+    }
+
+    // The partner of our server is online and should have responded with
+    // the success status, but not include its new state.
     ASSERT_TRUE(rsp);
     checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "Server maintenance successfully canceled.");
 
+    // The server should not get stuck in the partner-in-maintenance state.
     EXPECT_EQ(HA_WAITING_ST, service.getCurrState());
+    EXPECT_EQ(HA_UNAVAILABLE_ST, service.communication_state_->getPartnerState());
 }
 
 // This test verifies the case when the server receiving the ha-maintenance-cancel
@@ -5678,6 +5856,9 @@ TEST_F(HAServiceTest, processMaintenanceCancelSuccessAuthorized) {
     factory2_->getResponseCreator()->addBasicAuth("foo", "bar");
     factory3_->getResponseCreator()->addBasicAuth("test", "1234");
 
+    auto response_arguments = Element::createMap();
+    factory2_->getResponseCreator()->setArguments("ha-maintenance-notify", response_arguments);
+
     // Create HA configuration for 3 servers. This server is
     // server 1.
     HAConfigPtr config_storage = createValidConfiguration();