diff options
author | Aashish Sharma <aasharma@redhat.com> | 2023-03-28 10:04:25 +0200 |
---|---|---|
committer | Aashish Sharma <aasharma@li-e74156cc-2f67-11b2-a85c-e98659a63c5c.ibm.com> | 2023-07-04 13:20:32 +0200 |
commit | 5415a2611de7c50cbf429e92b253ed7bbd2ab7ea (patch) | |
tree | c7d5dce136ea823724540afe24e212c5b06a1f27 /src/pybind/mgr/dashboard/services | |
parent | Merge pull request #52194 from zdover23/wip-doc-2023-06-26-radosgw-s3select-o... (diff) | |
download | ceph-5415a2611de7c50cbf429e92b253ed7bbd2ab7ea.tar.xz ceph-5415a2611de7c50cbf429e92b253ed7bbd2ab7ea.zip |
mgr/dashboard: Allow the user to import and export multi-site configuration
Fixes: https://tracker.ceph.com/issues/61776
Signed-off-by: Aashish Sharma <aasharma@redhat.com>
Diffstat (limited to 'src/pybind/mgr/dashboard/services')
-rw-r--r-- | src/pybind/mgr/dashboard/services/ceph_service.py | 29 | ||||
-rw-r--r-- | src/pybind/mgr/dashboard/services/rgw_client.py | 975 |
2 files changed, 483 insertions, 521 deletions
diff --git a/src/pybind/mgr/dashboard/services/ceph_service.py b/src/pybind/mgr/dashboard/services/ceph_service.py index f0e21c59898..135f88ca2c9 100644 --- a/src/pybind/mgr/dashboard/services/ceph_service.py +++ b/src/pybind/mgr/dashboard/services/ceph_service.py @@ -294,6 +294,35 @@ class CephService(object): return {} @classmethod + def set_multisite_config(cls, realm_name, zonegroup_name, zone_name, daemon_name): + full_daemon_name = 'rgw.' + daemon_name + + KMS_CONFIG = [ + ['rgw_realm', realm_name], + ['rgw_zonegroup', zonegroup_name], + ['rgw_zone', zone_name] + ] + + for (key, value) in KMS_CONFIG: + if value == 'null': + continue + CephService.send_command('mon', 'config set', + who=name_to_config_section(full_daemon_name), + name=key, value=value) + return {} + + @classmethod + def get_realm_tokens(cls): + tokens_info = mgr.remote('rgw', 'get_realm_tokens') + return tokens_info + + @classmethod + def import_realm_token(cls, realm_token, zone_name): + tokens_info = mgr.remote('rgw', 'import_realm_token', zone_name=zone_name, + realm_token=realm_token, start_radosgw=True) + return tokens_info + + @classmethod def get_pool_pg_status(cls, pool_name): # type: (str) -> dict pool = cls.get_pool_by_attribute('pool_name', pool_name) diff --git a/src/pybind/mgr/dashboard/services/rgw_client.py b/src/pybind/mgr/dashboard/services/rgw_client.py index 648afb13339..707612acf83 100644 --- a/src/pybind/mgr/dashboard/services/rgw_client.py +++ b/src/pybind/mgr/dashboard/services/rgw_client.py @@ -8,7 +8,6 @@ import json import logging import os import re -import subprocess import xml.etree.ElementTree as ET # noqa: N814 from subprocess import SubprocessError @@ -594,6 +593,355 @@ class RgwClient(RestClient): return realm_info['name'] return None + @RestClient.api_get('/{bucket_name}?versioning') + def get_bucket_versioning(self, bucket_name, request=None): + """ + Get bucket versioning. + :param str bucket_name: the name of the bucket. + :return: versioning info + :rtype: Dict + """ + # pylint: disable=unused-argument + result = request() + if 'Status' not in result: + result['Status'] = 'Suspended' + if 'MfaDelete' not in result: + result['MfaDelete'] = 'Disabled' + return result + + @RestClient.api_put('/{bucket_name}?versioning') + def set_bucket_versioning(self, bucket_name, versioning_state, mfa_delete, + mfa_token_serial, mfa_token_pin, request=None): + """ + Set bucket versioning. + :param str bucket_name: the name of the bucket. + :param str versioning_state: + https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketPUTVersioningStatus.html + :param str mfa_delete: MFA Delete state. + :param str mfa_token_serial: + https://docs.ceph.com/docs/master/radosgw/mfa/ + :param str mfa_token_pin: value of a TOTP token at a certain time (auth code) + :return: None + """ + # pylint: disable=unused-argument + versioning_configuration = ET.Element('VersioningConfiguration') + status_element = ET.SubElement(versioning_configuration, 'Status') + status_element.text = versioning_state + + headers = {} + if mfa_delete and mfa_token_serial and mfa_token_pin: + headers['x-amz-mfa'] = '{} {}'.format(mfa_token_serial, mfa_token_pin) + mfa_delete_element = ET.SubElement(versioning_configuration, 'MfaDelete') + mfa_delete_element.text = mfa_delete + + data = ET.tostring(versioning_configuration, encoding='unicode') + + try: + request(data=data, headers=headers) + except RequestException as error: + msg = str(error) + if mfa_delete and mfa_token_serial and mfa_token_pin \ + and 'AccessDenied' in error.content.decode(): + msg = 'Bad MFA credentials: {}'.format(msg) + raise DashboardException(msg=msg, + http_status_code=error.status_code, + component='rgw') + + @RestClient.api_get('/{bucket_name}?encryption') + def get_bucket_encryption(self, bucket_name, request=None): + # pylint: disable=unused-argument + try: + result = request() # type: ignore + result['Status'] = 'Enabled' + return result + except RequestException as e: + if e.content: + content = json_str_to_object(e.content) + if content.get( + 'Code') == 'ServerSideEncryptionConfigurationNotFoundError': + return { + 'Status': 'Disabled', + } + raise e + + @RestClient.api_delete('/{bucket_name}?encryption') + def delete_bucket_encryption(self, bucket_name, request=None): + # pylint: disable=unused-argument + result = request() # type: ignore + return result + + @RestClient.api_put('/{bucket_name}?encryption') + def set_bucket_encryption(self, bucket_name, key_id, + sse_algorithm, request: Optional[object] = None): + # pylint: disable=unused-argument + encryption_configuration = ET.Element('ServerSideEncryptionConfiguration') + rule_element = ET.SubElement(encryption_configuration, 'Rule') + default_encryption_element = ET.SubElement(rule_element, + 'ApplyServerSideEncryptionByDefault') + sse_algo_element = ET.SubElement(default_encryption_element, + 'SSEAlgorithm') + sse_algo_element.text = sse_algorithm + if sse_algorithm == 'aws:kms': + kms_master_key_element = ET.SubElement(default_encryption_element, + 'KMSMasterKeyID') + kms_master_key_element.text = key_id + data = ET.tostring(encryption_configuration, encoding='unicode') + try: + _ = request(data=data) # type: ignore + except RequestException as e: + raise DashboardException(msg=str(e), component='rgw') + + @RestClient.api_get('/{bucket_name}?object-lock') + def get_bucket_locking(self, bucket_name, request=None): + # type: (str, Optional[object]) -> dict + """ + Gets the locking configuration for a bucket. The locking + configuration will be applied by default to every new object + placed in the specified bucket. + :param bucket_name: The name of the bucket. + :type bucket_name: str + :return: The locking configuration. + :rtype: Dict + """ + # pylint: disable=unused-argument + + # Try to get the Object Lock configuration. If there is none, + # then return default values. + try: + result = request() # type: ignore + return { + 'lock_enabled': dict_get(result, 'ObjectLockEnabled') == 'Enabled', + 'lock_mode': dict_get(result, 'Rule.DefaultRetention.Mode'), + 'lock_retention_period_days': dict_get(result, 'Rule.DefaultRetention.Days', 0), + 'lock_retention_period_years': dict_get(result, 'Rule.DefaultRetention.Years', 0) + } + except RequestException as e: + if e.content: + content = json_str_to_object(e.content) + if content.get( + 'Code') == 'ObjectLockConfigurationNotFoundError': + return { + 'lock_enabled': False, + 'lock_mode': 'compliance', + 'lock_retention_period_days': None, + 'lock_retention_period_years': None + } + raise e + + @RestClient.api_put('/{bucket_name}?object-lock') + def set_bucket_locking(self, + bucket_name: str, + mode: str, + retention_period_days: Optional[Union[int, str]] = None, + retention_period_years: Optional[Union[int, str]] = None, + request: Optional[object] = None) -> None: + """ + Places the locking configuration on the specified bucket. The + locking configuration will be applied by default to every new + object placed in the specified bucket. + :param bucket_name: The name of the bucket. + :type bucket_name: str + :param mode: The lock mode, e.g. `COMPLIANCE` or `GOVERNANCE`. + :type mode: str + :param retention_period_days: + :type retention_period_days: int + :param retention_period_years: + :type retention_period_years: int + :rtype: None + """ + # pylint: disable=unused-argument + + retention_period_days, retention_period_years = self.perform_validations( + retention_period_days, retention_period_years, mode) + + # Generate the XML data like this: + # <ObjectLockConfiguration> + # <ObjectLockEnabled>string</ObjectLockEnabled> + # <Rule> + # <DefaultRetention> + # <Days>integer</Days> + # <Mode>string</Mode> + # <Years>integer</Years> + # </DefaultRetention> + # </Rule> + # </ObjectLockConfiguration> + locking_configuration = ET.Element('ObjectLockConfiguration') + enabled_element = ET.SubElement(locking_configuration, + 'ObjectLockEnabled') + enabled_element.text = 'Enabled' # Locking can't be disabled. + rule_element = ET.SubElement(locking_configuration, 'Rule') + default_retention_element = ET.SubElement(rule_element, + 'DefaultRetention') + mode_element = ET.SubElement(default_retention_element, 'Mode') + mode_element.text = mode.upper() + if retention_period_days: + days_element = ET.SubElement(default_retention_element, 'Days') + days_element.text = str(retention_period_days) + if retention_period_years: + years_element = ET.SubElement(default_retention_element, 'Years') + years_element.text = str(retention_period_years) + + data = ET.tostring(locking_configuration, encoding='unicode') + + try: + _ = request(data=data) # type: ignore + except RequestException as e: + raise DashboardException(msg=str(e), component='rgw') + + def list_roles(self) -> List[Dict[str, Any]]: + rgw_list_roles_command = ['role', 'list'] + code, roles, err = mgr.send_rgwadmin_command(rgw_list_roles_command) + if code < 0: + logger.warning('Error listing roles with code %d: %s', code, err) + return [] + + return roles + + def create_role(self, role_name: str, role_path: str, role_assume_policy_doc: str) -> None: + try: + json.loads(role_assume_policy_doc) + except: # noqa: E722 + raise DashboardException('Assume role policy document is not a valid json') + + # valid values: + # pylint: disable=C0301 + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-role.html#cfn-iam-role-path # noqa: E501 + if len(role_name) > 64: + raise DashboardException( + f'Role name "{role_name}" is invalid. Should be 64 characters or less') + + role_name_regex = '[0-9a-zA-Z_+=,.@-]+' + if not re.fullmatch(role_name_regex, role_name): + raise DashboardException( + f'Role name "{role_name}" is invalid. Valid characters are "{role_name_regex}"') + + if not os.path.isabs(role_path): + raise DashboardException( + f'Role path "{role_path}" is invalid. It should be an absolute path') + if role_path[-1] != '/': + raise DashboardException( + f'Role path "{role_path}" is invalid. It should start and end with a slash') + path_regex = '(\u002F)|(\u002F[\u0021-\u007E]+\u002F)' + if not re.fullmatch(path_regex, role_path): + raise DashboardException( + (f'Role path "{role_path}" is invalid.' + f'Role path should follow the pattern "{path_regex}"')) + + rgw_create_role_command = ['role', 'create', '--role-name', role_name, '--path', role_path] + if role_assume_policy_doc: + rgw_create_role_command += ['--assume-role-policy-doc', f"{role_assume_policy_doc}"] + + code, _roles, _err = mgr.send_rgwadmin_command(rgw_create_role_command, + stdout_as_json=False) + if code != 0: + # pylint: disable=C0301 + link = 'https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-role.html#cfn-iam-role-path' # noqa: E501 + msg = (f'Error creating role with code {code}: ' + 'Looks like the document has a wrong format.' + f' For more information about the format look at {link}') + raise DashboardException(msg=msg, component='rgw') + + def perform_validations(self, retention_period_days, retention_period_years, mode): + try: + retention_period_days = int(retention_period_days) if retention_period_days else 0 + retention_period_years = int(retention_period_years) if retention_period_years else 0 + if retention_period_days < 0 or retention_period_years < 0: + raise ValueError + except (TypeError, ValueError): + msg = "Retention period must be a positive integer." + raise DashboardException(msg=msg, component='rgw') + if retention_period_days and retention_period_years: + # https://docs.aws.amazon.com/AmazonS3/latest/API/archive-RESTBucketPUTObjectLockConfiguration.html + msg = "Retention period requires either Days or Years. "\ + "You can't specify both at the same time." + raise DashboardException(msg=msg, component='rgw') + if not retention_period_days and not retention_period_years: + msg = "Retention period requires either Days or Years. "\ + "You must specify at least one." + raise DashboardException(msg=msg, component='rgw') + if not isinstance(mode, str) or mode.upper() not in ['COMPLIANCE', 'GOVERNANCE']: + msg = "Retention mode must be either COMPLIANCE or GOVERNANCE." + raise DashboardException(msg=msg, component='rgw') + return retention_period_days, retention_period_years + + +class RgwMultisite: + def migrate_to_multisite(self, realm_name: str, zonegroup_name: str, zone_name: str, + zonegroup_endpoints: str, zone_endpoints: str, access_key: str, + secret_key: str): + rgw_realm_create_cmd = ['realm', 'create', '--rgw-realm', realm_name, '--default'] + try: + exit_code, _, err = mgr.send_rgwadmin_command(rgw_realm_create_cmd, False) + if exit_code > 0: + raise DashboardException(e=err, msg='Unable to create realm', + http_status_code=500, component='rgw') + except SubprocessError as error: + raise DashboardException(error, http_status_code=500, component='rgw') + + rgw_zonegroup_edit_cmd = ['zonegroup', 'rename', '--rgw-zonegroup', 'default', + '--zonegroup-new-name', zonegroup_name] + try: + exit_code, _, err = mgr.send_rgwadmin_command(rgw_zonegroup_edit_cmd, False) + if exit_code > 0: + raise DashboardException(e=err, msg='Unable to rename zonegroup to {}'.format(zonegroup_name), # noqa E501 #pylint: disable=line-too-long + http_status_code=500, component='rgw') + except SubprocessError as error: + raise DashboardException(error, http_status_code=500, component='rgw') + + rgw_zone_edit_cmd = ['zone', 'rename', '--rgw-zone', + 'default', '--zone-new-name', zone_name, + '--rgw-zonegroup', zonegroup_name] + try: + exit_code, _, err = mgr.send_rgwadmin_command(rgw_zone_edit_cmd, False) + if exit_code > 0: + raise DashboardException(e=err, msg='Unable to rename zone to {}'.format(zone_name), # noqa E501 #pylint: disable=line-too-long + http_status_code=500, component='rgw') + except SubprocessError as error: + raise DashboardException(error, http_status_code=500, component='rgw') + + rgw_zonegroup_modify_cmd = ['zonegroup', 'modify', + '--rgw-realm', realm_name, + '--rgw-zonegroup', zonegroup_name] + if zonegroup_endpoints: + rgw_zonegroup_modify_cmd.append('--endpoints') + rgw_zonegroup_modify_cmd.append(zonegroup_endpoints) + rgw_zonegroup_modify_cmd.append('--master') + rgw_zonegroup_modify_cmd.append('--default') + try: + exit_code, _, err = mgr.send_rgwadmin_command(rgw_zonegroup_modify_cmd) + if exit_code > 0: + raise DashboardException(e=err, msg='Unable to modify zonegroup {}'.format(zonegroup_name), # noqa E501 #pylint: disable=line-too-long + http_status_code=500, component='rgw') + except SubprocessError as error: + raise DashboardException(error, http_status_code=500, component='rgw') + + rgw_zone_modify_cmd = ['zone', 'modify', '--rgw-realm', realm_name, + '--rgw-zonegroup', zonegroup_name, + '--rgw-zone', zone_name] + if zone_endpoints: + rgw_zone_modify_cmd.append('--endpoints') + rgw_zone_modify_cmd.append(zone_endpoints) + rgw_zone_modify_cmd.append('--master') + rgw_zone_modify_cmd.append('--default') + try: + exit_code, _, err = mgr.send_rgwadmin_command(rgw_zone_modify_cmd) + if exit_code > 0: + raise DashboardException(e=err, msg='Unable to modify zone', + http_status_code=500, component='rgw') + except SubprocessError as error: + raise DashboardException(error, http_status_code=500, component='rgw') + + if access_key and secret_key: + rgw_zone_modify_cmd = ['zone', 'modify', '--rgw-zone', zone_name, + '--access-key', access_key, '--secret', secret_key] + try: + exit_code, _, err = mgr.send_rgwadmin_command(rgw_zone_modify_cmd) + if exit_code > 0: + raise DashboardException(e=err, msg='Unable to modify zone', + http_status_code=500, component='rgw') + except SubprocessError as error: + raise DashboardException(error, http_status_code=500, component='rgw') + def create_realm(self, realm_name: str, default: bool): rgw_realm_create_cmd = ['realm', 'create'] cmd_create_realm_options = ['--rgw-realm', realm_name] @@ -652,26 +1000,6 @@ class RgwClient(RestClient): all_realms_info['default_realm'] = '' # type: ignore return all_realms_info - def delete_realm(self, realm_name: str): - rgw_delete_realm_cmd = ['realm', 'rm', '--rgw-realm', realm_name] - try: - exit_code, _, _ = mgr.send_rgwadmin_command(rgw_delete_realm_cmd) - if exit_code > 0: - raise DashboardException(msg='Unable to delete realm', - http_status_code=500, component='rgw') - except SubprocessError as error: - raise DashboardException(error, http_status_code=500, component='rgw') - - def update_period(self): - rgw_update_period_cmd = ['period', 'update', '--commit'] - try: - exit_code, _, err = mgr.send_rgwadmin_command(rgw_update_period_cmd) - if exit_code > 0: - raise DashboardException(e=err, msg='Unable to update period', - http_status_code=500, component='rgw') - except SubprocessError as error: - raise DashboardException(error, http_status_code=500, component='rgw') - def edit_realm(self, realm_name: str, new_realm_name: str, default: str = ''): rgw_realm_edit_cmd = [] if new_realm_name != realm_name: @@ -694,8 +1022,18 @@ class RgwClient(RestClient): except SubprocessError as error: raise DashboardException(error, http_status_code=500, component='rgw') + def delete_realm(self, realm_name: str): + rgw_delete_realm_cmd = ['realm', 'rm', '--rgw-realm', realm_name] + try: + exit_code, _, _ = mgr.send_rgwadmin_command(rgw_delete_realm_cmd) + if exit_code > 0: + raise DashboardException(msg='Unable to delete realm', + http_status_code=500, component='rgw') + except SubprocessError as error: + raise DashboardException(error, http_status_code=500, component='rgw') + def create_zonegroup(self, realm_name: str, zonegroup_name: str, - default: bool, master: bool, endpoints: List[str]): + default: bool, master: bool, endpoints: str): rgw_zonegroup_create_cmd = ['zonegroup', 'create'] cmd_create_zonegroup_options = ['--rgw-zonegroup', zonegroup_name] if realm_name != 'null': @@ -705,13 +1043,9 @@ class RgwClient(RestClient): cmd_create_zonegroup_options.append('--default') if master != 'false': cmd_create_zonegroup_options.append('--master') - if endpoints != 'null': # type: ignore - if isinstance(endpoints, list) and len(endpoints) > 1: - endpoint = ','.join(endpoints) - else: - endpoint = endpoints # type: ignore + if endpoints: cmd_create_zonegroup_options.append('--endpoints') - cmd_create_zonegroup_options.append(endpoint) + cmd_create_zonegroup_options.append(endpoints) rgw_zonegroup_create_cmd += cmd_create_zonegroup_options try: exit_code, out, err = mgr.send_rgwadmin_command(rgw_zonegroup_create_cmd) @@ -722,19 +1056,79 @@ class RgwClient(RestClient): raise DashboardException(error, http_status_code=500, component='rgw') return out + def list_zonegroups(self): + rgw_zonegroup_list = {} + rgw_zonegroup_list_cmd = ['zonegroup', 'list'] + try: + exit_code, out, _ = mgr.send_rgwadmin_command(rgw_zonegroup_list_cmd) + if exit_code > 0: + raise DashboardException(msg='Unable to fetch zonegroup list', + http_status_code=500, component='rgw') + rgw_zonegroup_list = out + except SubprocessError as error: + raise DashboardException(error, http_status_code=500, component='rgw') + return rgw_zonegroup_list + + def get_zonegroup(self, zonegroup_name: str): + zonegroup_info = {} + if zonegroup_name != 'default': + rgw_zonegroup_info_cmd = ['zonegroup', 'get', '--rgw-zonegroup', zonegroup_name] + else: + rgw_zonegroup_info_cmd = ['zonegroup', 'get', '--rgw-zonegroup', + zonegroup_name, '--rgw-realm', 'default'] + try: + exit_code, out, _ = mgr.send_rgwadmin_command(rgw_zonegroup_info_cmd) + if exit_code > 0: + raise DashboardException('Unable to get zonegroup info', + http_status_code=500, component='rgw') + zonegroup_info = out + except SubprocessError as error: + raise DashboardException(error, http_status_code=500, component='rgw') + return zonegroup_info + + def get_all_zonegroups_info(self): + all_zonegroups_info = {} + zonegroups_info = [] + rgw_zonegroup_list = self.list_zonegroups() + if 'zonegroups' in rgw_zonegroup_list: + if rgw_zonegroup_list['zonegroups'] != []: + for rgw_zonegroup in rgw_zonegroup_list['zonegroups']: + zonegroup_info = self.get_zonegroup(rgw_zonegroup) + zonegroups_info.append(zonegroup_info) + all_zonegroups_info['zonegroups'] = zonegroups_info # type: ignore + else: + all_zonegroups_info['zonegroups'] = [] # type: ignore + if 'default_info' in rgw_zonegroup_list and rgw_zonegroup_list['default_info'] != '': + all_zonegroups_info['default_zonegroup'] = rgw_zonegroup_list['default_info'] + else: + all_zonegroups_info['default_zonegroup'] = '' # type: ignore + return all_zonegroups_info + + def delete_zonegroup(self, zonegroup_name: str, delete_pools: str, pools: List[str]): + if delete_pools == 'true': + zonegroup_info = self.get_zonegroup(zonegroup_name) + rgw_delete_zonegroup_cmd = ['zonegroup', 'delete', '--rgw-zonegroup', zonegroup_name] + try: + exit_code, _, _ = mgr.send_rgwadmin_command(rgw_delete_zonegroup_cmd) + if exit_code > 0: + raise DashboardException(msg='Unable to delete zonegroup', + http_status_code=500, component='rgw') + except SubprocessError as error: + raise DashboardException(error, http_status_code=500, component='rgw') + self.update_period() + if delete_pools == 'true': + for zone in zonegroup_info['zones']: + self.delete_zone(zone['name'], 'true', pools) + def modify_zonegroup(self, realm_name: str, zonegroup_name: str, default: str, master: str, - endpoints: List[str]): - if realm_name: - rgw_zonegroup_modify_cmd = ['zonegroup', 'modify', - '--rgw-realm', realm_name, - '--rgw-zonegroup', zonegroup_name] + endpoints: str): + + rgw_zonegroup_modify_cmd = ['zonegroup', 'modify', + '--rgw-realm', realm_name, + '--rgw-zonegroup', zonegroup_name] if endpoints: - if len(endpoints) > 1: - endpoint = ','.join(str(e) for e in endpoints) - else: - endpoint = endpoints[0] rgw_zonegroup_modify_cmd.append('--endpoints') - rgw_zonegroup_modify_cmd.append(endpoint) + rgw_zonegroup_modify_cmd.append(endpoints) if master and str_to_bool(master): rgw_zonegroup_modify_cmd.append('--master') if default and str_to_bool(default): @@ -852,7 +1246,7 @@ class RgwClient(RestClient): # pylint: disable=W0102 def edit_zonegroup(self, realm_name: str, zonegroup_name: str, new_zonegroup_name: str, - default: str = '', master: str = '', endpoints: List[str] = [], + default: str = '', master: str = '', endpoints: str = '', add_zones: List[str] = [], remove_zones: List[str] = [], placement_targets: List[Dict[str, str]] = []): rgw_zonegroup_edit_cmd = [] @@ -883,73 +1277,18 @@ class RgwClient(RestClient): else: self.add_placement_targets(new_zonegroup_name, placement_targets) - def list_zonegroups(self): - rgw_zonegroup_list = {} - rgw_zonegroup_list_cmd = ['zonegroup', 'list'] - try: - exit_code, out, _ = mgr.send_rgwadmin_command(rgw_zonegroup_list_cmd) - if exit_code > 0: - raise DashboardException(msg='Unable to fetch zonegroup list', - http_status_code=500, component='rgw') - rgw_zonegroup_list = out - except SubprocessError as error: - raise DashboardException(error, http_status_code=500, component='rgw') - return rgw_zonegroup_list - - def get_zonegroup(self, zonegroup_name: str): - zonegroup_info = {} - rgw_zonegroup_info_cmd = ['zonegroup', 'get', '--rgw-zonegroup', zonegroup_name] - try: - exit_code, out, _ = mgr.send_rgwadmin_command(rgw_zonegroup_info_cmd) - if exit_code > 0: - raise DashboardException('Unable to get zonegroup info', - http_status_code=500, component='rgw') - zonegroup_info = out - except SubprocessError as error: - raise DashboardException(error, http_status_code=500, component='rgw') - return zonegroup_info - - def get_all_zonegroups_info(self): - all_zonegroups_info = {} - zonegroups_info = [] - rgw_zonegroup_list = self.list_zonegroups() - if 'zonegroups' in rgw_zonegroup_list: - if rgw_zonegroup_list['zonegroups'] != []: - for rgw_zonegroup in rgw_zonegroup_list['zonegroups']: - zonegroup_info = self.get_zonegroup(rgw_zonegroup) - zonegroups_info.append(zonegroup_info) - all_zonegroups_info['zonegroups'] = zonegroups_info # type: ignore - else: - all_zonegroups_info['zonegroups'] = [] # type: ignore - if 'default_info' in rgw_zonegroup_list and rgw_zonegroup_list['default_info'] != '': - all_zonegroups_info['default_zonegroup'] = rgw_zonegroup_list['default_info'] - else: - all_zonegroups_info['default_zonegroup'] = '' # type: ignore - return all_zonegroups_info - - def delete_zonegroup(self, zonegroup_name: str, delete_pools: str, pools: List[str]): - if delete_pools == 'true': - zonegroup_info = self.get_zonegroup(zonegroup_name) - rgw_delete_zonegroup_cmd = ['zonegroup', 'delete', '--rgw-zonegroup', zonegroup_name] + def update_period(self): + rgw_update_period_cmd = ['period', 'update', '--commit'] try: - exit_code, _, _ = mgr.send_rgwadmin_command(rgw_delete_zonegroup_cmd) + exit_code, _, err = mgr.send_rgwadmin_command(rgw_update_period_cmd) if exit_code > 0: - raise DashboardException(msg='Unable to delete zonegroup', + raise DashboardException(e=err, msg='Unable to update period', http_status_code=500, component='rgw') except SubprocessError as error: raise DashboardException(error, http_status_code=500, component='rgw') - self.update_period() - if delete_pools == 'true': - for zone in zonegroup_info['zones']: - self.delete_zone(zone['name'], 'true', pools) - def create_zone(self, zone_name, zonegroup_name, default, master, endpoints, user, - createSystemUser, master_zone_of_master_zonegroup): - if user != 'null': - access_key, secret_key = self.get_rgw_user_keys(user, master_zone_of_master_zonegroup) - else: - access_key = None # type: ignore - secret_key = None # type: ignore + def create_zone(self, zone_name, zonegroup_name, default, master, endpoints, access_key, + secret_key): rgw_zone_create_cmd = ['zone', 'create'] cmd_create_zone_options = ['--rgw-zone', zone_name] if zonegroup_name != 'null': @@ -978,36 +1317,8 @@ class RgwClient(RestClient): raise DashboardException(error, http_status_code=500, component='rgw') self.update_period() - - if createSystemUser == 'true': - self.create_system_user(user, zone_name) - access_key, secret_key = self.get_rgw_user_keys(user, zone_name) - rgw_zone_modify_cmd = ['zone', 'modify', '--rgw-zone', zone_name, - '--access-key', access_key, '--secret', secret_key] - try: - exit_code, _, err = mgr.send_rgwadmin_command(rgw_zone_modify_cmd) - if exit_code > 0: - raise DashboardException(e=err, msg='Unable to modify zone', - http_status_code=500, component='rgw') - except SubprocessError as error: - raise DashboardException(error, http_status_code=500, component='rgw') - self.update_period() - return out - def get_rgw_user_keys(self, user, zone_name): - access_key = '' - secret_key = '' - rgw_user_info_cmd = ['user', 'info', '--uid', user, '--rgw-zone', zone_name] - try: - _, out, _ = mgr.send_rgwadmin_command(rgw_user_info_cmd) - if out: - access_key, secret_key = self.parse_secrets(user, out) - except SubprocessError as error: - logger.exception(error) - - return access_key, secret_key - def parse_secrets(self, user, data): for key in data.get('keys', []): if key.get('user') == user: @@ -1017,21 +1328,12 @@ class RgwClient(RestClient): return '', '' def modify_zone(self, zone_name: str, zonegroup_name: str, default: str, master: str, - endpoints: List[str], user: str, master_zone_of_master_zonegroup): - if user: - access_key, secret_key = self.get_rgw_user_keys(user, master_zone_of_master_zonegroup) - else: - access_key = None - secret_key = None + endpoints: str, access_key: str, secret_key: str): rgw_zone_modify_cmd = ['zone', 'modify', '--rgw-zonegroup', zonegroup_name, '--rgw-zone', zone_name] if endpoints: - if len(endpoints) > 1: - endpoint = ','.join(str(e) for e in endpoints) - else: - endpoint = endpoints[0] rgw_zone_modify_cmd.append('--endpoints') - rgw_zone_modify_cmd.append(endpoint) + rgw_zone_modify_cmd.append(endpoints) if default and str_to_bool(default): rgw_zone_modify_cmd.append('--default') if master and str_to_bool(master): @@ -1083,10 +1385,10 @@ class RgwClient(RestClient): self.update_period() def edit_zone(self, zone_name: str, new_zone_name: str, zonegroup_name: str, default: str = '', - master: str = '', endpoints: List[str] = [], user: str = '', + master: str = '', endpoints: str = '', access_key: str = '', secret_key: str = '', placement_target: str = '', data_pool: str = '', index_pool: str = '', data_extra_pool: str = '', storage_class: str = '', data_pool_class: str = '', - compression: str = '', master_zone_of_master_zonegroup=None): + compression: str = ''): if new_zone_name != zone_name: rgw_zone_rename_cmd = ['zone', 'rename', '--rgw-zone', zone_name, '--zone-new-name', new_zone_name] @@ -1098,8 +1400,8 @@ class RgwClient(RestClient): except SubprocessError as error: raise DashboardException(error, http_status_code=500, component='rgw') self.update_period() - self.modify_zone(new_zone_name, zonegroup_name, default, master, endpoints, user, - master_zone_of_master_zonegroup) + self.modify_zone(new_zone_name, zonegroup_name, default, master, endpoints, access_key, + secret_key) self.add_placement_targets_zone(new_zone_name, placement_target, data_pool, index_pool, data_extra_pool) self.add_storage_class_zone(new_zone_name, placement_target, storage_class, @@ -1179,29 +1481,6 @@ class RgwClient(RestClient): if mgr.rados.pool_exists(pool): mgr.rados.delete_pool(pool) - def get_multisite_status(self): - is_multisite_configured = True - rgw_realm_list = self.list_realms() - rgw_zonegroup_list = self.list_zonegroups() - rgw_zone_list = self.list_zones() - if len(rgw_realm_list['realms']) < 1 and len(rgw_zonegroup_list['zonegroups']) < 1 \ - and len(rgw_zone_list['zones']) < 1: - is_multisite_configured = False - return is_multisite_configured - - def get_multisite_sync_status(self): - sync_status = '' - rgw_sync_status_cmd = ['sync', 'status'] - try: - exit_code, out, _ = mgr.send_rgwadmin_command(rgw_sync_status_cmd, False) - if exit_code > 0: - raise DashboardException('Unable to get sync status', - http_status_code=500, component='rgw') - sync_status = out - except subprocess.TimeoutExpired: - sync_status = 'Timeout Expired' - return sync_status - def create_system_user(self, userName: str, zoneName: str): rgw_user_create_cmd = ['user', 'create', '--uid', userName, '--display-name', userName, '--rgw-zone', zoneName, '--system'] @@ -1240,358 +1519,12 @@ class RgwClient(RestClient): raise DashboardException(error, http_status_code=500, component='rgw') return all_users_info - def migrate_to_multisite(self, realm_name: str, zonegroup_name: str, zone_name: str, - zonegroup_endpoints: List[str], zone_endpoints: List[str], user: str): - rgw_realm_create_cmd = ['realm', 'create', '--rgw-realm', realm_name, '--default'] - try: - exit_code, _, err = mgr.send_rgwadmin_command(rgw_realm_create_cmd, False) - if exit_code > 0: - raise DashboardException(e=err, msg='Unable to create realm', - http_status_code=500, component='rgw') - except SubprocessError as error: - raise DashboardException(error, http_status_code=500, component='rgw') - - rgw_zonegroup_edit_cmd = ['zonegroup', 'rename', '--rgw-zonegroup', 'default', - '--zonegroup-new-name', zonegroup_name] - try: - exit_code, _, err = mgr.send_rgwadmin_command(rgw_zonegroup_edit_cmd, False) - if exit_code > 0: - raise DashboardException(e=err, msg='Unable to rename zonegroup to {}'.format(zonegroup_name), # noqa E501 #pylint: disable=line-too-long - http_status_code=500, component='rgw') - except SubprocessError as error: - raise DashboardException(error, http_status_code=500, component='rgw') - - rgw_zone_edit_cmd = ['zone', 'rename', '--rgw-zone', - 'default', '--zone-new-name', zone_name, - '--rgw-zonegroup', zonegroup_name] - try: - exit_code, _, err = mgr.send_rgwadmin_command(rgw_zone_edit_cmd, False) - if exit_code > 0: - raise DashboardException(e=err, msg='Unable to rename zone to {}'.format(zone_name), # noqa E501 #pylint: disable=line-too-long - http_status_code=500, component='rgw') - except SubprocessError as error: - raise DashboardException(error, http_status_code=500, component='rgw') - - rgw_zonegroup_modify_cmd = ['zonegroup', 'modify', - '--rgw-realm', realm_name, - '--rgw-zonegroup', zonegroup_name] - if zonegroup_endpoints: - if len(zonegroup_endpoints) > 1: - endpoint = ','.join(str(e) for e in zonegroup_endpoints) - else: - endpoint = zonegroup_endpoints[0] - rgw_zonegroup_modify_cmd.append('--endpoints') - rgw_zonegroup_modify_cmd.append(endpoint) - rgw_zonegroup_modify_cmd.append('--master') - rgw_zonegroup_modify_cmd.append('--default') - try: - exit_code, _, err = mgr.send_rgwadmin_command(rgw_zonegroup_modify_cmd) - if exit_code > 0: - raise DashboardException(e=err, msg='Unable to modify zonegroup {}'.format(zonegroup_name), # noqa E501 #pylint: disable=line-too-long - http_status_code=500, component='rgw') - except SubprocessError as error: - raise DashboardException(error, http_status_code=500, component='rgw') - - rgw_zone_modify_cmd = ['zone', 'modify', '--rgw-realm', realm_name, - '--rgw-zonegroup', zonegroup_name, - '--rgw-zone', zone_name] - if zone_endpoints: - if len(zone_endpoints) > 1: - endpoint = ','.join(str(e) for e in zone_endpoints) - else: - endpoint = zone_endpoints[0] - rgw_zone_modify_cmd.append('--endpoints') - rgw_zone_modify_cmd.append(endpoint) - rgw_zone_modify_cmd.append('--master') - rgw_zone_modify_cmd.append('--default') - try: - exit_code, _, err = mgr.send_rgwadmin_command(rgw_zone_modify_cmd) - if exit_code > 0: - raise DashboardException(e=err, msg='Unable to modify zone', - http_status_code=500, component='rgw') - except SubprocessError as error: - raise DashboardException(error, http_status_code=500, component='rgw') - - if user: - access_key, secret_key = self.get_rgw_user_keys(user, zone_name) - rgw_zone_modify_cmd = ['zone', 'modify', '--rgw-zone', zone_name, - '--access-key', access_key, '--secret', secret_key] - try: - exit_code, _, err = mgr.send_rgwadmin_command(rgw_zone_modify_cmd) - if exit_code > 0: - raise DashboardException(e=err, msg='Unable to modify zone', - http_status_code=500, component='rgw') - except SubprocessError as error: - raise DashboardException(error, http_status_code=500, component='rgw') - self.update_period() - - @RestClient.api_get('/{bucket_name}?versioning') - def get_bucket_versioning(self, bucket_name, request=None): - """ - Get bucket versioning. - :param str bucket_name: the name of the bucket. - :return: versioning info - :rtype: Dict - """ - # pylint: disable=unused-argument - result = request() - if 'Status' not in result: - result['Status'] = 'Suspended' - if 'MfaDelete' not in result: - result['MfaDelete'] = 'Disabled' - return result - - @RestClient.api_put('/{bucket_name}?versioning') - def set_bucket_versioning(self, bucket_name, versioning_state, mfa_delete, - mfa_token_serial, mfa_token_pin, request=None): - """ - Set bucket versioning. - :param str bucket_name: the name of the bucket. - :param str versioning_state: - https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketPUTVersioningStatus.html - :param str mfa_delete: MFA Delete state. - :param str mfa_token_serial: - https://docs.ceph.com/docs/master/radosgw/mfa/ - :param str mfa_token_pin: value of a TOTP token at a certain time (auth code) - :return: None - """ - # pylint: disable=unused-argument - versioning_configuration = ET.Element('VersioningConfiguration') - status_element = ET.SubElement(versioning_configuration, 'Status') - status_element.text = versioning_state - - headers = {} - if mfa_delete and mfa_token_serial and mfa_token_pin: - headers['x-amz-mfa'] = '{} {}'.format(mfa_token_serial, mfa_token_pin) - mfa_delete_element = ET.SubElement(versioning_configuration, 'MfaDelete') - mfa_delete_element.text = mfa_delete - - data = ET.tostring(versioning_configuration, encoding='unicode') - - try: - request(data=data, headers=headers) - except RequestException as error: - msg = str(error) - if mfa_delete and mfa_token_serial and mfa_token_pin \ - and 'AccessDenied' in error.content.decode(): - msg = 'Bad MFA credentials: {}'.format(msg) - raise DashboardException(msg=msg, - http_status_code=error.status_code, - component='rgw') - - @RestClient.api_get('/{bucket_name}?encryption') - def get_bucket_encryption(self, bucket_name, request=None): - # pylint: disable=unused-argument - try: - result = request() # type: ignore - result['Status'] = 'Enabled' - return result - except RequestException as e: - if e.content: - content = json_str_to_object(e.content) - if content.get( - 'Code') == 'ServerSideEncryptionConfigurationNotFoundError': - return { - 'Status': 'Disabled', - } - raise e - - @RestClient.api_delete('/{bucket_name}?encryption') - def delete_bucket_encryption(self, bucket_name, request=None): - # pylint: disable=unused-argument - result = request() # type: ignore - return result - - @RestClient.api_put('/{bucket_name}?encryption') - def set_bucket_encryption(self, bucket_name, key_id, - sse_algorithm, request: Optional[object] = None): - # pylint: disable=unused-argument - encryption_configuration = ET.Element('ServerSideEncryptionConfiguration') - rule_element = ET.SubElement(encryption_configuration, 'Rule') - default_encryption_element = ET.SubElement(rule_element, - 'ApplyServerSideEncryptionByDefault') - sse_algo_element = ET.SubElement(default_encryption_element, - 'SSEAlgorithm') - sse_algo_element.text = sse_algorithm - if sse_algorithm == 'aws:kms': - kms_master_key_element = ET.SubElement(default_encryption_element, - 'KMSMasterKeyID') - kms_master_key_element.text = key_id - data = ET.tostring(encryption_configuration, encoding='unicode') - try: - _ = request(data=data) # type: ignore - except RequestException as e: - raise DashboardException(msg=str(e), component='rgw') - - @RestClient.api_get('/{bucket_name}?object-lock') - def get_bucket_locking(self, bucket_name, request=None): - # type: (str, Optional[object]) -> dict - """ - Gets the locking configuration for a bucket. The locking - configuration will be applied by default to every new object - placed in the specified bucket. - :param bucket_name: The name of the bucket. - :type bucket_name: str - :return: The locking configuration. - :rtype: Dict - """ - # pylint: disable=unused-argument - - # Try to get the Object Lock configuration. If there is none, - # then return default values. - try: - result = request() # type: ignore - return { - 'lock_enabled': dict_get(result, 'ObjectLockEnabled') == 'Enabled', - 'lock_mode': dict_get(result, 'Rule.DefaultRetention.Mode'), - 'lock_retention_period_days': dict_get(result, 'Rule.DefaultRetention.Days', 0), - 'lock_retention_period_years': dict_get(result, 'Rule.DefaultRetention.Years', 0) - } - except RequestException as e: - if e.content: - content = json_str_to_object(e.content) - if content.get( - 'Code') == 'ObjectLockConfigurationNotFoundError': - return { - 'lock_enabled': False, - 'lock_mode': 'compliance', - 'lock_retention_period_days': None, - 'lock_retention_period_years': None - } - raise e - - @RestClient.api_put('/{bucket_name}?object-lock') - def set_bucket_locking(self, - bucket_name: str, - mode: str, - retention_period_days: Optional[Union[int, str]] = None, - retention_period_years: Optional[Union[int, str]] = None, - request: Optional[object] = None) -> None: - """ - Places the locking configuration on the specified bucket. The - locking configuration will be applied by default to every new - object placed in the specified bucket. - :param bucket_name: The name of the bucket. - :type bucket_name: str - :param mode: The lock mode, e.g. `COMPLIANCE` or `GOVERNANCE`. - :type mode: str - :param retention_period_days: - :type retention_period_days: int - :param retention_period_years: - :type retention_period_years: int - :rtype: None - """ - # pylint: disable=unused-argument - - retention_period_days, retention_period_years = self.perform_validations( - retention_period_days, retention_period_years, mode) - - # Generate the XML data like this: - # <ObjectLockConfiguration> - # <ObjectLockEnabled>string</ObjectLockEnabled> - # <Rule> - # <DefaultRetention> - # <Days>integer</Days> - # <Mode>string</Mode> - # <Years>integer</Years> - # </DefaultRetention> - # </Rule> - # </ObjectLockConfiguration> - locking_configuration = ET.Element('ObjectLockConfiguration') - enabled_element = ET.SubElement(locking_configuration, - 'ObjectLockEnabled') - enabled_element.text = 'Enabled' # Locking can't be disabled. - rule_element = ET.SubElement(locking_configuration, 'Rule') - default_retention_element = ET.SubElement(rule_element, - 'DefaultRetention') - mode_element = ET.SubElement(default_retention_element, 'Mode') - mode_element.text = mode.upper() - if retention_period_days: - days_element = ET.SubElement(default_retention_element, 'Days') - days_element.text = str(retention_period_days) - if retention_period_years: - years_element = ET.SubElement(default_retention_element, 'Years') - years_element.text = str(retention_period_years) - - data = ET.tostring(locking_configuration, encoding='unicode') - - try: - _ = request(data=data) # type: ignore - except RequestException as e: - raise DashboardException(msg=str(e), component='rgw') - - def list_roles(self) -> List[Dict[str, Any]]: - rgw_list_roles_command = ['role', 'list'] - code, roles, err = mgr.send_rgwadmin_command(rgw_list_roles_command) - if code < 0: - logger.warning('Error listing roles with code %d: %s', code, err) - return [] - - return roles - - def create_role(self, role_name: str, role_path: str, role_assume_policy_doc: str) -> None: - try: - json.loads(role_assume_policy_doc) - except: # noqa: E722 - raise DashboardException('Assume role policy document is not a valid json') - - # valid values: - # pylint: disable=C0301 - # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-role.html#cfn-iam-role-path # noqa: E501 - if len(role_name) > 64: - raise DashboardException( - f'Role name "{role_name}" is invalid. Should be 64 characters or less') - - role_name_regex = '[0-9a-zA-Z_+=,.@-]+' - if not re.fullmatch(role_name_regex, role_name): - raise DashboardException( - f'Role name "{role_name}" is invalid. Valid characters are "{role_name_regex}"') - - if not os.path.isabs(role_path): - raise DashboardException( - f'Role path "{role_path}" is invalid. It should be an absolute path') - if role_path[-1] != '/': - raise DashboardException( - f'Role path "{role_path}" is invalid. It should start and end with a slash') - path_regex = '(\u002F)|(\u002F[\u0021-\u007E]+\u002F)' - if not re.fullmatch(path_regex, role_path): - raise DashboardException( - (f'Role path "{role_path}" is invalid.' - f'Role path should follow the pattern "{path_regex}"')) - - rgw_create_role_command = ['role', 'create', '--role-name', role_name, '--path', role_path] - if role_assume_policy_doc: - rgw_create_role_command += ['--assume-role-policy-doc', f"{role_assume_policy_doc}"] - - code, _roles, _err = mgr.send_rgwadmin_command(rgw_create_role_command, - stdout_as_json=False) - if code != 0: - # pylint: disable=C0301 - link = 'https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-role.html#cfn-iam-role-path' # noqa: E501 - msg = (f'Error creating role with code {code}: ' - 'Looks like the document has a wrong format.' - f' For more information about the format look at {link}') - raise DashboardException(msg=msg, component='rgw') - - def perform_validations(self, retention_period_days, retention_period_years, mode): - try: - retention_period_days = int(retention_period_days) if retention_period_days else 0 - retention_period_years = int(retention_period_years) if retention_period_years else 0 - if retention_period_days < 0 or retention_period_years < 0: - raise ValueError - except (TypeError, ValueError): - msg = "Retention period must be a positive integer." - raise DashboardException(msg=msg, component='rgw') - if retention_period_days and retention_period_years: - # https://docs.aws.amazon.com/AmazonS3/latest/API/archive-RESTBucketPUTObjectLockConfiguration.html - msg = "Retention period requires either Days or Years. "\ - "You can't specify both at the same time." - raise DashboardException(msg=msg, component='rgw') - if not retention_period_days and not retention_period_years: - msg = "Retention period requires either Days or Years. "\ - "You must specify at least one." - raise DashboardException(msg=msg, component='rgw') - if not isinstance(mode, str) or mode.upper() not in ['COMPLIANCE', 'GOVERNANCE']: - msg = "Retention mode must be either COMPLIANCE or GOVERNANCE." - raise DashboardException(msg=msg, component='rgw') - return retention_period_days, retention_period_years + def get_multisite_status(self): + is_multisite_configured = True + rgw_realm_list = self.list_realms() + rgw_zonegroup_list = self.list_zonegroups() + rgw_zone_list = self.list_zones() + if len(rgw_realm_list['realms']) < 1 and len(rgw_zonegroup_list['zonegroups']) < 1 \ + and len(rgw_zone_list['zones']) < 1: + is_multisite_configured = False + return is_multisite_configured |