diff options
8 files changed, 318 insertions, 19 deletions
diff --git a/src/hooks/dhcp/high_availability/command_creator.cc b/src/hooks/dhcp/high_availability/command_creator.cc index 933a922814..d966441d2b 100644 --- a/src/hooks/dhcp/high_availability/command_creator.cc +++ b/src/hooks/dhcp/high_availability/command_creator.cc @@ -7,6 +7,7 @@ #include <config.h> #include <command_creator.h> +#include <ha_service_states.h> #include <cc/command_interpreter.h> #include <exceptions/exceptions.h> #include <boost/pointer_cast.hpp> @@ -259,10 +260,12 @@ CommandCreator::createLease6GetPage(const Lease6Ptr& last_lease6, ConstElementPtr CommandCreator::createMaintenanceNotify(const std::string& server_name, const bool cancel, + const int state, const HAServerType& server_type) { auto args = Element::createMap(); args->set("server-name", Element::create(server_name)); args->set("cancel", Element::create(cancel)); + args->set("state", Element::create(stateToString(state))); auto command = config::createCommand("ha-maintenance-notify", args); insertService(command, server_type); return (command); diff --git a/src/hooks/dhcp/high_availability/command_creator.h b/src/hooks/dhcp/high_availability/command_creator.h index f06065de7b..34c5065e6f 100644 --- a/src/hooks/dhcp/high_availability/command_creator.h +++ b/src/hooks/dhcp/high_availability/command_creator.h @@ -176,11 +176,13 @@ public: /// for associating the command with the relationship. /// @param cancel boolean value indicating if the maintenance /// is being cancelled (true) or requested (false). + /// @param state server state. /// @param server_type type of the DHCP server, i.e. v4 or v6. /// @return Pointer to the JSON representation of the command. static data::ConstElementPtr createMaintenanceNotify(const std::string& server_name, const bool cancel, + const int state, const HAServerType& server_type); /// @brief Creates ha-sync-complete-notify command. diff --git a/src/hooks/dhcp/high_availability/ha_impl.cc b/src/hooks/dhcp/high_availability/ha_impl.cc index 495f93b260..ab542e0eed 100644 --- a/src/hooks/dhcp/high_availability/ha_impl.cc +++ b/src/hooks/dhcp/high_availability/ha_impl.cc @@ -789,9 +789,15 @@ HAImpl::maintenanceNotifyHandler(hooks::CalloutHandle& callout_handle) { isc_throw(BadValue, "'cancel' must be a boolean in the 'ha-maintenance-notify' command"); } + ConstElementPtr state = args->get("state"); + if (state && state->getType() != Element::string) { + isc_throw(BadValue, "'state' must be a string in the 'ha-maintenance-notify' command"); + } + service = getHAServiceByServerName("ha-maintenance-notify", args); - ConstElementPtr response = service->processMaintenanceNotify(cancel_op->boolValue()); + ConstElementPtr response = service->processMaintenanceNotify(cancel_op->boolValue(), + state ? state->stringValue() : "unavailable"); callout_handle.setArgument("response", response); } catch (const std::exception& ex) { diff --git a/src/hooks/dhcp/high_availability/ha_service.cc b/src/hooks/dhcp/high_availability/ha_service.cc index a3436ffab3..d93b556394 100644 --- a/src/hooks/dhcp/high_availability/ha_service.cc +++ b/src/hooks/dhcp/high_availability/ha_service.cc @@ -1076,7 +1076,7 @@ HAService::inScopeInternal(QueryPtrType& query) { query->addClass(dhcp::ClientClass(scope_class)); // The following is the part of the server failure detection algorithm. // If the query should be processed by the partner we need to check if - // the partner responds. If the number of unansweered queries exceeds a + // the partner responds. If the number of unanswered queries exceeds a // configured threshold, we will consider the partner to be offline. if (!in_scope && communication_state_->isCommunicationInterrupted()) { communication_state_->analyzeMessage(query); @@ -2743,7 +2743,7 @@ HAService::processContinue() { } ConstElementPtr -HAService::processMaintenanceNotify(const bool cancel) { +HAService::processMaintenanceNotify(const bool cancel, const std::string& state) { if (cancel) { if (getCurrState() != HA_IN_MAINTENANCE_ST) { return (createAnswer(CONTROL_RESULT_ERROR, "Unable to cancel the" @@ -2751,10 +2751,28 @@ HAService::processMaintenanceNotify(const bool cancel) { " in-maintenance state.")); } + try { + communication_state_->setPartnerState(state); + + } catch (...) { + // Hopefully the received state is correct. If it isn't, let's set the + // partner state to unavailable and count on the state machine to resolve. + communication_state_->setPartnerUnavailable(); + } postNextEvent(HA_MAINTENANCE_CANCEL_EVT); - verboseTransition(getPrevState()); + // In rare cases the previous state may be the server's current state. Transitioning + // to it would cause a deadlock and the server will remain stuck in maintenance. + // In these cases let's simply transition to the waiting state and the state machine + // should solve it. + verboseTransition(getPrevState() == HA_IN_MAINTENANCE_ST ? HA_WAITING_ST : getPrevState()); runModel(NOP_EVT); - return (createAnswer(CONTROL_RESULT_SUCCESS, "Server maintenance canceled.")); + + // Communicate the new state to the partner. + ElementPtr arguments = Element::createMap(); + std::string state_label = getState(getCurrState())->getLabel(); + arguments->set("state", Element::create(state_label)); + + return (createAnswer(CONTROL_RESULT_SUCCESS, "Server maintenance canceled.", arguments)); } switch (getCurrState()) { @@ -2801,7 +2819,7 @@ HAService::processMaintenanceStart() { HostHttpHeader(remote_config->getUrl().getStrippedHostname())); remote_config->addBasicAuthHttpHeader(request); request->setBodyAsJson(CommandCreator::createMaintenanceNotify(config_->getThisServerName(), - false, server_type_)); + false, getCurrState(), server_type_)); request->finalize(); // Response object should also be created because the HTTP client needs @@ -2932,7 +2950,7 @@ HAService::processMaintenanceCancel() { HostHttpHeader(remote_config->getUrl().getStrippedHostname())); remote_config->addBasicAuthHttpHeader(request); request->setBodyAsJson(CommandCreator::createMaintenanceNotify(config_->getThisServerName(), - true, server_type_)); + true, getCurrState(), server_type_)); request->finalize(); // Response object should also be created because the HTTP client needs @@ -2968,8 +2986,26 @@ HAService::processMaintenanceCancel() { // Handle third group of errors. try { int rcode = 0; - static_cast<void>(verifyAsyncResponse(response, rcode)); - + ConstElementPtr args = verifyAsyncResponse(response, rcode); + + // Partner's state has changed after the notification. However, we don't know + // its new state. We'll check if the partner returned its state. If it didn't, + // we set the unavailable state as a default. + communication_state_->setPartnerUnavailable(); + + // Newer Kea versions return the state of the notified server. + // Older versions don't, so the arguments may not be present. + if (args && args->getType() == Element::map) { + // Arguments may include partner's state. + ConstElementPtr state = args->get("state"); + if (state) { + if (state->getType() != Element::string) { + isc_throw(CtrlChannelError, "server state not returned in response" + " to a ha-heartbeat command or it is not a string"); + } + communication_state_->setPartnerState(state->stringValue()); + } + } } catch (const std::exception& ex) { error_message = ex.what(); LOG_ERROR(ha_logger, HA_MAINTENANCE_NOTIFY_CANCEL_FAILED) @@ -3009,9 +3045,10 @@ HAService::processMaintenanceCancel() { } // Successfully reverted partner's state. Let's also revert our state to the - // previous one. + // previous one. Avoid returning to the partner-in-maintenance if it was + // the previous state. postNextEvent(HA_MAINTENANCE_CANCEL_EVT); - verboseTransition(getPrevState()); + verboseTransition(getPrevState() == HA_PARTNER_IN_MAINTENANCE_ST ? HA_WAITING_ST : getPrevState()); runModel(NOP_EVT); return (createAnswer(CONTROL_RESULT_SUCCESS, diff --git a/src/hooks/dhcp/high_availability/ha_service.h b/src/hooks/dhcp/high_availability/ha_service.h index 0b6bc3a086..885254ea23 100644 --- a/src/hooks/dhcp/high_availability/ha_service.h +++ b/src/hooks/dhcp/high_availability/ha_service.h @@ -1048,9 +1048,12 @@ public: /// @param cancel boolean value indicating if the maintenance is being /// canceled with this operation. If it is set to false the maintenance /// is being started. + /// @param state partner's state as string. It should be set to "unavailable" + /// if the state was not explicitly provided by the partner. /// /// @return Pointer to the response to the ha-maintenance-notify. - data::ConstElementPtr processMaintenanceNotify(const bool cancel); + data::ConstElementPtr processMaintenanceNotify(const bool cancel, + const std::string& state); /// @brief Processes ha-maintenance-start command and returns a response. /// diff --git a/src/hooks/dhcp/high_availability/tests/command_creator_unittest.cc b/src/hooks/dhcp/high_availability/tests/command_creator_unittest.cc index 0b73181ce1..f6a76b02c3 100644 --- a/src/hooks/dhcp/high_availability/tests/command_creator_unittest.cc +++ b/src/hooks/dhcp/high_availability/tests/command_creator_unittest.cc @@ -7,6 +7,7 @@ #include <config.h> #include <ha_server_type.h> +#include <ha_service_states.h> #include <lease_update_backlog.h> #include <command_creator.h> #include <asiolink/io_address.h> @@ -562,7 +563,8 @@ TEST(CommandCreatorTest, createLease6GetPageZeroLimit) { // This test verifies that the ha-maintenance-notify command is correct // while being sent to the DHCPv4 server. TEST(CommandCreatorTest, createMaintenanceNotify4) { - ConstElementPtr command = CommandCreator::createMaintenanceNotify("server1", true, HAServerType::DHCPv4); + ConstElementPtr command = CommandCreator::createMaintenanceNotify("server1", true, HA_READY_ST, + HAServerType::DHCPv4); ConstElementPtr arguments; ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "ha-maintenance-notify", "dhcp4", arguments)); @@ -571,12 +573,18 @@ TEST(CommandCreatorTest, createMaintenanceNotify4) { ASSERT_TRUE(cancel); ASSERT_EQ(Element::boolean, cancel->getType()); EXPECT_TRUE(cancel->boolValue()); + + auto state = arguments->get("state"); + ASSERT_TRUE(state); + ASSERT_EQ(Element::string, state->getType()); + EXPECT_EQ("ready", state->stringValue()); } // This test verifies that the ha-maintenance-notify command is correct // while being sent to the DHCPv6 server. TEST(CommandCreatorTest, createMaintenanceNotify6) { - ConstElementPtr command = CommandCreator::createMaintenanceNotify("server1", false, HAServerType::DHCPv6); + ConstElementPtr command = CommandCreator::createMaintenanceNotify("server1", false, HA_SYNCING_ST, + HAServerType::DHCPv6); ConstElementPtr arguments; ASSERT_NO_FATAL_FAILURE(testCommandBasics(command, "ha-maintenance-notify", "dhcp6", arguments)); @@ -585,6 +593,11 @@ TEST(CommandCreatorTest, createMaintenanceNotify6) { ASSERT_TRUE(cancel); ASSERT_EQ(Element::boolean, cancel->getType()); EXPECT_FALSE(cancel->boolValue()); + + auto state = arguments->get("state"); + ASSERT_TRUE(state); + ASSERT_EQ(Element::string, state->getType()); + EXPECT_EQ("syncing", state->stringValue()); } // This test verifies that the ha-sync-complete-notify command sent to a diff --git a/src/hooks/dhcp/high_availability/tests/ha_impl_unittest.cc b/src/hooks/dhcp/high_availability/tests/ha_impl_unittest.cc index f99bc6e993..b5622c5b7a 100644 --- a/src/hooks/dhcp/high_availability/tests/ha_impl_unittest.cc +++ b/src/hooks/dhcp/high_availability/tests/ha_impl_unittest.cc @@ -2176,6 +2176,60 @@ TEST_F(HAImplTest, maintenanceNotifyBadServerName) { checkAnswer(response, CONTROL_RESULT_ERROR, "server5 matches no configured 'server-name'"); } +// Test ha-maintenance-notify command handler with partner state. +TEST_F(HAImplTest, maintenanceNotifyPartnerState) { + ha_impl_.reset(new HAImpl()); + ha_impl_->setIOService(io_service_); + ASSERT_NO_THROW(ha_impl_->configure(createValidJsonConfiguration())); + + // Starting the service is required prior to running any callouts. + NetworkStatePtr network_state(new NetworkState()); + ASSERT_NO_THROW(ha_impl_->startServices(network_state, + HAServerType::DHCPv4)); + + // Start the maintenance. + ConstElementPtr command = Element::fromJSON( + "{" + " \"command\": \"ha-maintenance-notify\"," + " \"arguments\": {" + " \"cancel\": false" + " }" + "}" + ); + + CalloutHandlePtr callout_handle = HooksManager::createCalloutHandle(); + callout_handle->setArgument("command", command); + + ASSERT_NO_THROW(ha_impl_->maintenanceNotifyHandler(*callout_handle)); + + ConstElementPtr response; + callout_handle->getArgument("response", response); + ASSERT_TRUE(response); + + checkAnswer(response, CONTROL_RESULT_SUCCESS, "Server is in-maintenance state."); + + // Cancel the maintenance. + command = Element::fromJSON( + "{" + " \"command\": \"ha-maintenance-notify\"," + " \"arguments\": {" + " \"cancel\": true," + " \"server-name\": \"server1\"," + " \"state\": \"waiting\"" + " }" + "}" + ); + + callout_handle->setArgument("command", command); + + ASSERT_NO_THROW(ha_impl_->maintenanceNotifyHandler(*callout_handle)); + + callout_handle->getArgument("response", response); + ASSERT_TRUE(response); + + checkAnswer(response, CONTROL_RESULT_SUCCESS, "Server maintenance canceled."); +} + // Test ha-reset command handler with a specified server name. TEST_F(HAImplTest, haReset) { ha_impl_.reset(new HAImpl()); diff --git a/src/hooks/dhcp/high_availability/tests/ha_service_unittest.cc b/src/hooks/dhcp/high_availability/tests/ha_service_unittest.cc index 75d6b4d252..6e00f55610 100644 --- a/src/hooks/dhcp/high_availability/tests/ha_service_unittest.cc +++ b/src/hooks/dhcp/high_availability/tests/ha_service_unittest.cc @@ -5294,7 +5294,7 @@ TEST_F(HAServiceTest, processMaintenanceNotify) { // Process ha-maintenance-notify command that should transition the // state machine to the in-maintenance state. ConstElementPtr rsp; - ASSERT_NO_THROW(rsp = service.processMaintenanceNotify(false)); + ASSERT_NO_THROW(rsp = service.processMaintenanceNotify(false, "unavailable")); // The server should have responded. ASSERT_TRUE(rsp); @@ -5304,7 +5304,8 @@ TEST_F(HAServiceTest, processMaintenanceNotify) { EXPECT_EQ(HA_IN_MAINTENANCE_ST, service.getCurrState()); // Try to cancel the maintenance. - ASSERT_NO_THROW(rsp = service.processMaintenanceNotify(true)); + ASSERT_NO_THROW(rsp = service.processMaintenanceNotify(true, "unavailable")); + EXPECT_EQ(HA_UNAVAILABLE_ST, service.communication_state_->getPartnerState()); // The server should have responded. ASSERT_TRUE(rsp); @@ -5322,12 +5323,13 @@ TEST_F(HAServiceTest, processMaintenanceNotify) { }; // Make sure that the transition from the other states is not allowed. + service.communication_state_->setPartnerState("waiting"); for (auto const& state : invalid_states) { EXPECT_NO_THROW(service.transition(state, HAService::NOP_EVT)); EXPECT_NO_THROW(service.runModel(HAService::NOP_EVT)); ConstElementPtr rsp; - ASSERT_NO_THROW(rsp = service.processMaintenanceNotify(false)); + ASSERT_NO_THROW(rsp = service.processMaintenanceNotify(false, "unavailable")); ASSERT_TRUE(rsp); checkAnswer(rsp, TestHAService::HA_CONTROL_RESULT_MAINTENANCE_NOT_ALLOWED, @@ -5336,6 +5338,71 @@ TEST_F(HAServiceTest, processMaintenanceNotify) { } } +// This test verifies that the ha-maintenance-notify command is processed +// and the partner state is set according to the partner's notion of state. +TEST_F(HAServiceTest, processMaintenanceNotifyCancelSetPartnerState) { + HAConfigPtr config_storage = createValidConfiguration(); + + TestHAService service(1, io_service_, network_state_, config_storage); + + EXPECT_NO_THROW(service.transition(HA_WAITING_ST, HAService::NOP_EVT)); + + // Process ha-maintenance-notify command that should transition the + // state machine to the in-maintenance state. + ConstElementPtr rsp; + ASSERT_NO_THROW(rsp = service.processMaintenanceNotify(false, "")); + + // The server should have responded. + ASSERT_TRUE(rsp); + checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "Server is in-maintenance state."); + + // The state machine should have been transitioned to the in-maintenance state. + EXPECT_EQ(HA_IN_MAINTENANCE_ST, service.getCurrState()); + + // Try to cancel the maintenance with specifying the state. + ASSERT_NO_THROW(rsp = service.processMaintenanceNotify(true, "ready")); + EXPECT_EQ(HA_READY_ST, service.communication_state_->getPartnerState()); + + // The state machine should have been transitioned to the state it was in + // prior to transitioning to the in-maintenance state. + EXPECT_EQ(HA_WAITING_ST, service.getCurrState()); +} + +// This test verifies that the ha-maintenance-notify command is processed +// and the server transitions to the waiting state when its prvious state +// before the maintenance was in-maintenance. +TEST_F(HAServiceTest, processMaintenanceNotifyCancelSetAvoidDeadlock) { + HAConfigPtr config_storage = createValidConfiguration(); + + TestHAService service(1, io_service_, network_state_, config_storage); + + // Set the in-maintenance state. Normally the server returns to a previous + // state when the maintenance is canceled. In this case, it doesn't make much + // sense because it will cause a deadlock. In this exceptional case the server + // should rather transition to a waiting state instead. + EXPECT_NO_THROW(service.transition(HA_IN_MAINTENANCE_ST, HAService::NOP_EVT)); + + // Process ha-maintenance-notify command that should re-transition the + // state machine to the in-maintenance state. + ConstElementPtr rsp; + ASSERT_NO_THROW(rsp = service.processMaintenanceNotify(false, "")); + + // The server should have responded. + ASSERT_TRUE(rsp); + checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "Server is in-maintenance state."); + + // The state machine should have been transitioned to the in-maintenance state. + EXPECT_EQ(HA_IN_MAINTENANCE_ST, service.getCurrState()); + + // Try to cancel the maintenance with specifying the state. + ASSERT_NO_THROW(rsp = service.processMaintenanceNotify(true, "ready")); + EXPECT_EQ(HA_READY_ST, service.communication_state_->getPartnerState()); + + // The state machine should have been transitioned to the state it was in + // prior to transitioning to the in-maintenance state. + EXPECT_EQ(HA_WAITING_ST, service.getCurrState()); +} + // This test verifies the case when the server receiving the ha-maintenance-start // command successfully transitions to the partner-in-maintenance state and its // partner transitions to the in-maintenance state. @@ -5629,6 +5696,10 @@ TEST_F(HAServiceTest, processMaintenanceCancelSuccess) { HAConfigPtr config_storage = createValidConfiguration(); setBasicAuth(config_storage); + auto response_arguments = Element::createMap(); + response_arguments->set("state", Element::create("hot-standby")); + factory2_->getResponseCreator()->setArguments("ha-maintenance-notify", response_arguments); + // Start the servers. ASSERT_NO_THROW({ listener_->start(); @@ -5657,12 +5728,119 @@ TEST_F(HAServiceTest, processMaintenanceCancelSuccess) { } // The partner of our server is online and should have responded with - // the success status. Therefore, this server should have transitioned - // to the partner-in-maintenance state. + // the success status, and include its new state. + ASSERT_TRUE(rsp); + checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "Server maintenance successfully canceled."); + + EXPECT_EQ(HA_WAITING_ST, service.getCurrState()); + EXPECT_EQ(HA_HOT_STANDBY_ST, service.communication_state_->getPartnerState()); +} + +// This test verifies the case when the server receiving the ha-maintenance-cancel +// command does not receive a new state from its partner. +TEST_F(HAServiceTest, processMaintenanceCancelSuccessPartnerNewStateUnknown) { + // Create HA configuration for 3 servers. This server is + // server 1. + HAConfigPtr config_storage = createValidConfiguration(); + setBasicAuth(config_storage); + + auto response_arguments = Element::createMap(); + factory2_->getResponseCreator()->setArguments("ha-maintenance-notify", response_arguments); + + // Start the servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + }); + + TestHAService service(1, io_service_, network_state_, config_storage); + + // Initial transition to the non-default state. The server should return to + // it when the maintenance is canceled. + ASSERT_NO_THROW(service.verboseTransition(HA_READY_ST)); + service.runModel(TestHAService::NOP_EVT); + + ASSERT_NO_THROW(service.verboseTransition(HA_PARTNER_IN_MAINTENANCE_ST)); + + // The tested function is synchronous, so we need to run server side IO service + // in background to not block the main thread. + auto thread = runIOServiceInThread(); + + // Process ha-maintenance-cancel command. + ConstElementPtr rsp; + ASSERT_NO_THROW(rsp = service.processMaintenanceCancel()); + + // Stop the IO service. This should cause the thread to terminate. + io_service_->stop(); + thread->join(); + io_service_->restart(); + try { + io_service_->poll(); + } catch (...) { + } + + // The partner of our server is online and should have responded with + // the success status, but not include its new state. + ASSERT_TRUE(rsp); + checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "Server maintenance successfully canceled."); + + EXPECT_EQ(HA_READY_ST, service.getCurrState()); + EXPECT_EQ(HA_UNAVAILABLE_ST, service.communication_state_->getPartnerState()); +} + +// This test verifies the case when the server receiving the ha-maintenance-cancel +// had been in the partner-in-maintenance state before the maintenance was started. +// In this case the server should transition to the waiting state when the maintenance +// is canceled. +TEST_F(HAServiceTest, processMaintenanceCancelSuccessAvoidDeadlock) { + // Create HA configuration for 3 servers. This server is + // server 1. + HAConfigPtr config_storage = createValidConfiguration(); + setBasicAuth(config_storage); + + auto response_arguments = Element::createMap(); + factory2_->getResponseCreator()->setArguments("ha-maintenance-notify", response_arguments); + + // Start the servers. + ASSERT_NO_THROW({ + listener_->start(); + listener2_->start(); + }); + + TestHAService service(1, io_service_, network_state_, config_storage); + + // Initial transition to the partner-in-maintenance state. It will be remembered + // as the previous state. + ASSERT_NO_THROW(service.verboseTransition(HA_PARTNER_IN_MAINTENANCE_ST)); + service.runModel(TestHAService::NOP_EVT); + + // Transition again and then cancel transition. + ASSERT_NO_THROW(service.verboseTransition(HA_PARTNER_IN_MAINTENANCE_ST)); + // The tested function is synchronous, so we need to run server side IO service + // in background to not block the main thread. + auto thread = runIOServiceInThread(); + + // Process ha-maintenance-cancel command. + ConstElementPtr rsp; + ASSERT_NO_THROW(rsp = service.processMaintenanceCancel()); + + // Stop the IO service. This should cause the thread to terminate. + io_service_->stop(); + thread->join(); + io_service_->restart(); + try { + io_service_->poll(); + } catch (...) { + } + + // The partner of our server is online and should have responded with + // the success status, but not include its new state. ASSERT_TRUE(rsp); checkAnswer(rsp, CONTROL_RESULT_SUCCESS, "Server maintenance successfully canceled."); + // The server should not get stuck in the partner-in-maintenance state. EXPECT_EQ(HA_WAITING_ST, service.getCurrState()); + EXPECT_EQ(HA_UNAVAILABLE_ST, service.communication_state_->getPartnerState()); } // This test verifies the case when the server receiving the ha-maintenance-cancel @@ -5678,6 +5856,9 @@ TEST_F(HAServiceTest, processMaintenanceCancelSuccessAuthorized) { factory2_->getResponseCreator()->addBasicAuth("foo", "bar"); factory3_->getResponseCreator()->addBasicAuth("test", "1234"); + auto response_arguments = Element::createMap(); + factory2_->getResponseCreator()->setArguments("ha-maintenance-notify", response_arguments); + // Create HA configuration for 3 servers. This server is // server 1. HAConfigPtr config_storage = createValidConfiguration(); |