Merge pull request #60511 from rhcs-dashboard/health-test-fix

qa/dashboard: fix tasks.mgr.dashboard.test_health.HealthTest

Reviewed-by: Ernesto Puerta <epuertat@redhat.com>

3 files changed, 20 insertions, 7 deletions

diff --git a/qa/tasks/mgr/dashboard/helper.py b/qa/tasks/mgr/dashboard/helper.py
index e6a7c35a23d..55355048a36 100644
--- a/qa/tasks/mgr/dashboard/helper.py
+++ b/qa/tasks/mgr/dashboard/helper.py
@@ -220,13 +220,11 @@ class DashboardTestCase(MgrTestCase):
 
             # To avoid any issues with e.g. unlink bugs, we destroy and recreate
             # the filesystem rather than just doing a rm -rf of files
-            cls.mds_cluster.mds_stop()
-            cls.mds_cluster.mds_fail()
             cls.mds_cluster.delete_all_filesystems()
+            cls.mds_cluster.mds_restart()  # to reset any run-time configs, etc.
             cls.fs = None  # is now invalid!
 
             cls.fs = cls.mds_cluster.newfs(create=True)
-            cls.fs.mds_restart()
 
             # In case some test messed with auth caps, reset them
             # pylint: disable=not-an-iterable
diff --git a/qa/tasks/mgr/dashboard/test_mgr_module.py b/qa/tasks/mgr/dashboard/test_mgr_module.py
index d6a368905b6..1dbdef23d34 100644
--- a/qa/tasks/mgr/dashboard/test_mgr_module.py
+++ b/qa/tasks/mgr/dashboard/test_mgr_module.py
@@ -4,6 +4,7 @@ from __future__ import absolute_import
 import logging
 
 import requests
+from urllib3.exceptions import MaxRetryError
 
 from .helper import (DashboardTestCase, JLeaf, JList, JObj,
                      module_options_object_schema, module_options_schema,
@@ -24,10 +25,11 @@ class MgrModuleTestCase(DashboardTestCase):
         def _check_connection():
             try:
                 # Try reaching an API endpoint successfully.
+                logger.info('Trying to reach the REST API endpoint')
                 self._get('/api/mgr/module')
                 if self._resp.status_code == 200:
                     return True
-            except requests.ConnectionError:
+            except (MaxRetryError, requests.ConnectionError):
                 pass
             return False
 
diff --git a/qa/tasks/mgr/mgr_test_case.py b/qa/tasks/mgr/mgr_test_case.py
index 9032e0e2658..4a5506391f2 100644
--- a/qa/tasks/mgr/mgr_test_case.py
+++ b/qa/tasks/mgr/mgr_test_case.py
@@ -1,5 +1,6 @@
 import json
 import logging
+import socket
 
 from unittest import SkipTest
 
@@ -229,15 +230,22 @@ class MgrTestCase(CephTestCase):
         """
         # Start handing out ports well above Ceph's range.
         assign_port = min_port
+        ip_addr = cls.mgr_cluster.get_mgr_map()['active_addr'].split(':')[0]
 
         for mgr_id in cls.mgr_cluster.mgr_ids:
             cls.mgr_cluster.mgr_stop(mgr_id)
             cls.mgr_cluster.mgr_fail(mgr_id)
 
+
         for mgr_id in cls.mgr_cluster.mgr_ids:
-            log.debug("Using port {0} for {1} on mgr.{2}".format(
-                assign_port, module_name, mgr_id
-            ))
+            # Find a port that isn't in use
+            while True:
+                if not cls.is_port_in_use(ip_addr, assign_port):
+                    break
+                log.debug(f"Port {assign_port} in use, trying next")
+                assign_port += 1
+
+            log.debug(f"Using port {assign_port} for {module_name} on mgr.{mgr_id}")
             cls.mgr_cluster.set_module_localized_conf(module_name, mgr_id,
                                                       config_name,
                                                       str(assign_port),
@@ -255,3 +263,8 @@ class MgrTestCase(CephTestCase):
                     mgr_map['active_name'], mgr_map['active_gid']))
             return done
         cls.wait_until_true(is_available, timeout=30)
+
+    @classmethod
+    def is_port_in_use(cls, ip_addr: str, port: int) -> bool:
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+            return s.connect_ex((ip_addr, port)) == 0