diff options
| -rw-r--r-- | qa/suites/nvmeof/basic/workloads/nvmeof_mtls.yaml | 36 | ||||
| -rw-r--r-- | qa/tasks/nvmeof.py | 37 | ||||
| -rwxr-xr-x | qa/workunits/nvmeof/mtls_test.sh | 76 |
3 files changed, 148 insertions, 1 deletions
diff --git a/qa/suites/nvmeof/basic/workloads/nvmeof_mtls.yaml b/qa/suites/nvmeof/basic/workloads/nvmeof_mtls.yaml new file mode 100644 index 00000000000..8eb4f6dc63c --- /dev/null +++ b/qa/suites/nvmeof/basic/workloads/nvmeof_mtls.yaml @@ -0,0 +1,36 @@ +tasks: +- nvmeof: + installer: host.a + gw_image: quay.io/ceph/nvmeof:latest # "default" is the image cephadm defaults to; change to test specific nvmeof images, example "latest" + rbd: + pool_name: mypool + image_name_prefix: myimage + gateway_config: + subsystems_count: 3 + namespaces_count: 20 + cli_image: quay.io/ceph/nvmeof-cli:latest + create_mtls_secrets: true + +- cephadm.wait_for_service: + service: nvmeof.mypool.mygroup0 + +- workunit: + no_coverage_and_limits: true + timeout: 30m + clients: + client.0: + - nvmeof/setup_subsystem.sh + - nvmeof/basic_tests.sh + - nvmeof/fio_test.sh --rbd_iostat + env: + RBD_POOL: mypool + RBD_IMAGE_PREFIX: myimage + IOSTAT_INTERVAL: '10' + RUNTIME: '60' + +- workunit: + no_coverage_and_limits: true + timeout: 30m + clients: + client.0: + - nvmeof/mtls_test.sh diff --git a/qa/tasks/nvmeof.py b/qa/tasks/nvmeof.py index b89f123c97e..42e357294d9 100644 --- a/qa/tasks/nvmeof.py +++ b/qa/tasks/nvmeof.py @@ -32,6 +32,7 @@ class Nvmeof(Task): gateway_config: namespaces_count: 10 cli_version: latest + create_mtls_secrets: False """ @@ -69,6 +70,7 @@ class Nvmeof(Task): self.serial = gateway_config.get('serial', 'SPDK00000000000001') self.port = gateway_config.get('port', '4420') self.srport = gateway_config.get('srport', '5500') + self.create_mtls_secrets = gateway_config.get('create_mtls_secrets', False) def deploy_nvmeof(self): """ @@ -147,7 +149,38 @@ class Nvmeof(Task): started=True, ) log.info("[nvmeof]: executed deploy_nvmeof successfully!") - + + def write_mtls_config(self, gateway_ips): + log.info("[nvmeof]: writing mtls config...") + allowed_ips = "" + for ip in gateway_ips: + allowed_ips += ("IP:" + ip + ",") + self.remote.run( + args=[ + "sudo", "openssl", "req", "-x509", "-newkey", "rsa:4096", "-nodes", "-keyout", "/etc/ceph/server.key", + "-out", "/etc/ceph/server.crt", "-days", "3650", "-subj", "/CN=my.server", "-addext", f"subjectAltName={allowed_ips[:-1]}" + ] + ) + self.remote.run( + args=[ + "sudo", "openssl", "req", "-x509", "-newkey", "rsa:4096", "-nodes", "-keyout", "/etc/ceph/client.key", + "-out", "/etc/ceph/client.crt", "-days", "3650", "-subj", "/CN=client1" + ] + ) + secrets_files = {"/etc/ceph/server.key": None, + "/etc/ceph/server.crt": None, + "/etc/ceph/client.key": None, + "/etc/ceph/client.crt": None, + } + for file in secrets_files.keys(): + secrets_files[file] = self.remote.read_file(path=file, sudo=True) + + for remote in self.ctx.cluster.remotes.keys(): + for remote_file in secrets_files.keys(): + data = secrets_files[remote_file] + remote.sudo_write_file(path=remote_file, data=data, mode='0644') + log.info("[nvmeof]: written mtls config!") + def set_gateway_cfg(self): log.info('[nvmeof]: running set_gateway_cfg...') ip_address = self.remote.ip_address @@ -174,6 +207,8 @@ class Nvmeof(Task): data=conf_data, sudo=True ) + if self.create_mtls_secrets: + self.write_mtls_config(gateway_ips) log.info("[nvmeof]: executed set_gateway_cfg successfully!") diff --git a/qa/workunits/nvmeof/mtls_test.sh b/qa/workunits/nvmeof/mtls_test.sh new file mode 100755 index 00000000000..e13ca530e8d --- /dev/null +++ b/qa/workunits/nvmeof/mtls_test.sh @@ -0,0 +1,76 @@ +#!/bin/bash + +set -ex +source /etc/ceph/nvmeof.env + +# install yq +wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /tmp/yq && chmod +x /tmp/yq + +subjectAltName=$(echo "$NVMEOF_GATEWAY_IP_ADDRESSES" | sed 's/,/,IP:/g') + +# create mtls spec files +ceph orch ls nvmeof --export > /tmp/gw-conf-original.yaml +sudo /tmp/yq ".spec.enable_auth=true | \ + .spec.root_ca_cert=\"mountcert\" | \ + .spec.client_cert = load_str(\"/etc/ceph/client.crt\") | \ + .spec.client_key = load_str(\"/etc/ceph/client.key\") | \ + .spec.server_cert = load_str(\"/etc/ceph/server.crt\") | \ + .spec.server_key = load_str(\"/etc/ceph/server.key\")" /tmp/gw-conf-original.yaml > /tmp/gw-conf-with-mtls.yaml +cp /tmp/gw-conf-original.yaml /tmp/gw-conf-without-mtls.yaml +sudo /tmp/yq '.spec.enable_auth=false' -i /tmp/gw-conf-without-mtls.yaml + +wait_for_service() { + MAX_RETRIES=30 + for ((RETRY_COUNT=1; RETRY_COUNT<=MAX_RETRIES; RETRY_COUNT++)); do + + if ceph orch ls --refresh | grep -q "nvmeof"; then + echo "Found nvmeof in the output!" + break + fi + if [ $RETRY_COUNT -eq $MAX_RETRIES ]; then + echo "Reached maximum retries ($MAX_RETRIES). Exiting." + break + fi + sleep 5 + done + ceph orch ps + ceph orch ls --refresh +} + +# deploy mtls +cat /tmp/gw-conf-with-mtls.yaml +ceph orch apply -i /tmp/gw-conf-with-mtls.yaml +ceph orch redeploy nvmeof.mypool.mygroup0 +sleep 100 +wait_for_service + + +# test +IFS=',' read -ra gateway_ips <<< "$NVMEOF_GATEWAY_IP_ADDRESSES" +for i in "${!gateway_ips[@]}" +do + ip="${gateway_ips[i]}" + sudo podman run -v /etc/ceph/server.crt:/server.crt:z -v /etc/ceph/client.crt:/client.crt:z \ + -v /etc/ceph/client.key:/client.key:z \ + -it $NVMEOF_CLI_IMAGE --server-address $ip --server-port $NVMEOF_SRPORT \ + --client-key /client.key --client-cert /client.crt --server-cert /server.crt --format json subsystem list +done + + +# remove mtls +cat /tmp/gw-conf-without-mtls.yaml +ceph orch apply -i /tmp/gw-conf-without-mtls.yaml +ceph orch redeploy nvmeof.mypool.mygroup0 +sleep 100 +wait_for_service + + +# test +IFS=',' read -ra gateway_ips <<< "$NVMEOF_GATEWAY_IP_ADDRESSES" +for i in "${!gateway_ips[@]}" +do + ip="${gateway_ips[i]}" + sudo podman run -it $NVMEOF_CLI_IMAGE --server-address $ip --server-port $NVMEOF_SRPORT \ + --format json subsystem list +done + |