summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLila Yasin <lyasin@redhat.com>2024-10-23 17:30:00 +0200
committerGitHub <noreply@github.com>2024-10-23 17:30:00 +0200
commite21dd0a0933fd544d1fe2513c54fe41203775eb5 (patch)
tree2c4b723d4d97a7926a8a90c091a4602a55968635
parentbump django 4.2.16 to be in line with DAB (#15596) (diff)
downloadawx-e21dd0a0933fd544d1fe2513c54fe41203775eb5.tar.xz
awx-e21dd0a0933fd544d1fe2513c54fe41203775eb5.zip
Make cloud providers dynamic (#15537)
* Add dynamic pull for cloud inventory plugins and update corresponding tests Co-authored-by: Sviatoslav Sydorenko (Святослав Сидоренко) <wk.cvs.github@sydorenko.org.ua> * Create third dictionary to preserve current functionality and add 'file' there * Migrations for corresponding change --------- Co-authored-by: Sviatoslav Sydorenko (Святослав Сидоренко) <wk.cvs.github@sydorenko.org.ua>
-rw-r--r--awx/api/serializers.py11
-rw-r--r--awx/api/views/__init__.py5
-rw-r--r--awx/main/constants.py20
-rw-r--r--awx/main/migrations/0198_alter_inventorysource_source_and_more.py23
-rw-r--r--awx/main/models/__init__.py2
-rw-r--r--awx/main/models/base.py3
-rw-r--r--awx/main/models/inventory.py35
-rw-r--r--awx/main/tests/functional/models/test_inventory.py6
-rw-r--r--awx/main/tests/functional/test_inventory_source_injectors.py8
-rw-r--r--awx/main/utils/plugins.py59
10 files changed, 108 insertions, 64 deletions
diff --git a/awx/api/serializers.py b/awx/api/serializers.py
index acc13acbc9..2bb405cda3 100644
--- a/awx/api/serializers.py
+++ b/awx/api/serializers.py
@@ -102,7 +102,6 @@ from awx.main.models import (
WorkflowJobTemplate,
WorkflowJobTemplateNode,
StdoutMaxBytesExceeded,
- CLOUD_INVENTORY_SOURCES,
)
from awx.main.models.base import VERBOSITY_CHOICES, NEW_JOB_TYPE_CHOICES
from awx.main.models.rbac import role_summary_fields_generator, give_creator_permissions, get_role_codenames, to_permissions, get_role_from_object_role
@@ -119,7 +118,9 @@ from awx.main.utils import (
truncate_stdout,
get_licenser,
)
+
from awx.main.utils.filters import SmartFilter
+from awx.main.utils.plugins import load_combined_inventory_source_options
from awx.main.utils.named_url_graph import reset_counters
from awx.main.scheduler.task_manager_models import TaskManagerModels
from awx.main.redact import UriCleaner, REPLACE_STR
@@ -2300,6 +2301,7 @@ class GroupVariableDataSerializer(BaseVariableDataSerializer):
class InventorySourceOptionsSerializer(BaseSerializer):
credential = DeprecatedCredentialField(help_text=_('Cloud credential to use for inventory updates.'))
+ source = serializers.ChoiceField(choices=[])
class Meta:
fields = (
@@ -2321,6 +2323,11 @@ class InventorySourceOptionsSerializer(BaseSerializer):
)
read_only_fields = ('*', 'custom_virtualenv')
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ if 'source' in self.fields:
+ self.fields['source'].choices = load_combined_inventory_source_options()
+
def get_related(self, obj):
res = super(InventorySourceOptionsSerializer, self).get_related(obj)
if obj.credential: # TODO: remove when 'credential' field is removed
@@ -5500,7 +5507,7 @@ class ScheduleSerializer(LaunchConfigurationBaseSerializer, SchedulePreviewSeria
return summary_fields
def validate_unified_job_template(self, value):
- if type(value) == InventorySource and value.source not in CLOUD_INVENTORY_SOURCES:
+ if type(value) == InventorySource and value.source not in load_combined_inventory_source_options():
raise serializers.ValidationError(_('Inventory Source must be a cloud resource.'))
elif type(value) == Project and value.scm_type == '':
raise serializers.ValidationError(_('Manual Project cannot have a schedule set.'))
diff --git a/awx/api/views/__init__.py b/awx/api/views/__init__.py
index bbe79bd2a4..6619047863 100644
--- a/awx/api/views/__init__.py
+++ b/awx/api/views/__init__.py
@@ -100,6 +100,7 @@ from awx.main.utils import (
)
from awx.main.utils.encryption import encrypt_value
from awx.main.utils.filters import SmartFilter
+from awx.main.utils.plugins import compute_cloud_inventory_sources
from awx.main.redact import UriCleaner
from awx.api.permissions import (
JobTemplateCallbackPermission,
@@ -2196,9 +2197,9 @@ class InventorySourceNotificationTemplatesAnyList(SubListCreateAttachDetachAPIVi
def post(self, request, *args, **kwargs):
parent = self.get_parent_object()
- if parent.source not in models.CLOUD_INVENTORY_SOURCES:
+ if parent.source not in compute_cloud_inventory_sources():
return Response(
- dict(msg=_("Notification Templates can only be assigned when source is one of {}.").format(models.CLOUD_INVENTORY_SOURCES, parent.source)),
+ dict(msg=_("Notification Templates can only be assigned when source is one of {}.").format(compute_cloud_inventory_sources(), parent.source)),
status=status.HTTP_400_BAD_REQUEST,
)
return super(InventorySourceNotificationTemplatesAnyList, self).post(request, *args, **kwargs)
diff --git a/awx/main/constants.py b/awx/main/constants.py
index 7a93481f6f..59f23f7c53 100644
--- a/awx/main/constants.py
+++ b/awx/main/constants.py
@@ -6,7 +6,6 @@ import re
from django.utils.translation import gettext_lazy as _
__all__ = [
- 'CLOUD_PROVIDERS',
'PRIVILEGE_ESCALATION_METHODS',
'ANSI_SGR_PATTERN',
'CAN_CANCEL',
@@ -14,25 +13,6 @@ __all__ = [
'STANDARD_INVENTORY_UPDATE_ENV',
]
-CLOUD_PROVIDERS = (
- 'azure_rm',
- 'ec2',
- 'gce',
- 'vmware',
- 'openstack',
- 'rhv',
- 'satellite6',
- 'controller',
- 'insights',
- 'terraform',
- 'openshift_virtualization',
- 'controller_supported',
- 'rhv_supported',
- 'openshift_virtualization_supported',
- 'insights_supported',
- 'satellite6_supported',
-)
-
PRIVILEGE_ESCALATION_METHODS = [
('sudo', _('Sudo')),
('su', _('Su')),
diff --git a/awx/main/migrations/0198_alter_inventorysource_source_and_more.py b/awx/main/migrations/0198_alter_inventorysource_source_and_more.py
new file mode 100644
index 0000000000..cdedde61c2
--- /dev/null
+++ b/awx/main/migrations/0198_alter_inventorysource_source_and_more.py
@@ -0,0 +1,23 @@
+# Generated by Django 4.2.10 on 2024-10-22 15:58
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('main', '0197_remove_sso_app_content'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='inventorysource',
+ name='source',
+ field=models.CharField(default=None, max_length=32),
+ ),
+ migrations.AlterField(
+ model_name='inventoryupdate',
+ name='source',
+ field=models.CharField(default=None, max_length=32),
+ ),
+ ]
diff --git a/awx/main/models/__init__.py b/awx/main/models/__init__.py
index a63cc31bf8..f8dbc2d50f 100644
--- a/awx/main/models/__init__.py
+++ b/awx/main/models/__init__.py
@@ -16,7 +16,7 @@ from ansible_base.lib.utils.models import prevent_search
from ansible_base.lib.utils.models import user_summary_fields
# AWX
-from awx.main.models.base import BaseModel, PrimordialModel, accepts_json, CLOUD_INVENTORY_SOURCES, VERBOSITY_CHOICES # noqa
+from awx.main.models.base import BaseModel, PrimordialModel, accepts_json, VERBOSITY_CHOICES # noqa
from awx.main.models.unified_jobs import UnifiedJob, UnifiedJobTemplate, StdoutMaxBytesExceeded # noqa
from awx.main.models.organization import Organization, Team, UserSessionMembership # noqa
from awx.main.models.credential import Credential, CredentialType, CredentialInputSource, ManagedCredentialType, build_safe_env # noqa
diff --git a/awx/main/models/base.py b/awx/main/models/base.py
index 1d80923ee2..9319d1fb8d 100644
--- a/awx/main/models/base.py
+++ b/awx/main/models/base.py
@@ -15,7 +15,6 @@ from crum import get_current_user
# AWX
from awx.main.utils import encrypt_field, parse_yaml_or_json
-from awx.main.constants import CLOUD_PROVIDERS
__all__ = [
'VarsDictProperty',
@@ -32,7 +31,6 @@ __all__ = [
'JOB_TYPE_CHOICES',
'AD_HOC_JOB_TYPE_CHOICES',
'PROJECT_UPDATE_JOB_TYPE_CHOICES',
- 'CLOUD_INVENTORY_SOURCES',
'VERBOSITY_CHOICES',
]
@@ -61,7 +59,6 @@ PROJECT_UPDATE_JOB_TYPE_CHOICES = [
(PERM_INVENTORY_CHECK, _('Check')),
]
-CLOUD_INVENTORY_SOURCES = list(CLOUD_PROVIDERS) + ['scm']
VERBOSITY_CHOICES = [
(0, '0 (Normal)'),
diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py
index fd0796ad25..d14e81543d 100644
--- a/awx/main/models/inventory.py
+++ b/awx/main/models/inventory.py
@@ -28,7 +28,7 @@ from awx_plugins.inventory.plugins import PluginFileInjector
# AWX
from awx.api.versioning import reverse
-from awx.main.constants import CLOUD_PROVIDERS
+from awx.main.utils.plugins import discover_available_cloud_provider_plugin_names, compute_cloud_inventory_sources
from awx.main.consumers import emit_channel_notification
from awx.main.fields import (
ImplicitRoleField,
@@ -36,7 +36,7 @@ from awx.main.fields import (
OrderedManyToManyField,
)
from awx.main.managers import HostManager, HostMetricActiveManager
-from awx.main.models.base import BaseModel, CommonModelNameNotUnique, VarsDictProperty, CLOUD_INVENTORY_SOURCES, accepts_json
+from awx.main.models.base import BaseModel, CommonModelNameNotUnique, VarsDictProperty, accepts_json
from awx.main.models.events import InventoryUpdateEvent, UnpartitionedInventoryUpdateEvent
from awx.main.models.unified_jobs import UnifiedJob, UnifiedJobTemplate
from awx.main.models.mixins import (
@@ -394,7 +394,7 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin, RelatedJobsMixin):
if self.kind == 'smart':
active_inventory_sources = self.inventory_sources.none()
else:
- active_inventory_sources = self.inventory_sources.filter(source__in=CLOUD_INVENTORY_SOURCES)
+ active_inventory_sources = self.inventory_sources.filter(source__in=compute_cloud_inventory_sources())
failed_inventory_sources = active_inventory_sources.filter(last_job_failed=True)
total_hosts = active_hosts.count()
# if total_hosts has changed, set update_task_impact to True
@@ -914,23 +914,6 @@ class InventorySourceOptions(BaseModel):
injectors = dict()
- SOURCE_CHOICES = [
- ('file', _('File, Directory or Script')),
- ('constructed', _('Template additional groups and hostvars at runtime')),
- ('scm', _('Sourced from a Project')),
- ('ec2', _('Amazon EC2')),
- ('gce', _('Google Compute Engine')),
- ('azure_rm', _('Microsoft Azure Resource Manager')),
- ('vmware', _('VMware vCenter')),
- ('satellite6', _('Red Hat Satellite 6')),
- ('openstack', _('OpenStack')),
- ('rhv', _('Red Hat Virtualization')),
- ('controller', _('Red Hat Ansible Automation Platform')),
- ('insights', _('Red Hat Insights')),
- ('terraform', _('Terraform State')),
- ('openshift_virtualization', _('OpenShift Virtualization')),
- ]
-
# From the options of the Django management base command
INVENTORY_UPDATE_VERBOSITY_CHOICES = [
(0, '0 (WARNING)'),
@@ -943,7 +926,6 @@ class InventorySourceOptions(BaseModel):
source = models.CharField(
max_length=32,
- choices=SOURCE_CHOICES,
blank=False,
default=None,
)
@@ -1047,7 +1029,7 @@ class InventorySourceOptions(BaseModel):
# Allow an EC2 source to omit the credential. If Tower is running on
# an EC2 instance with an IAM Role assigned, boto will use credentials
# from the instance metadata instead of those explicitly provided.
- elif source in CLOUD_PROVIDERS and source not in ['ec2', 'openshift_virtualization']:
+ elif source in discover_available_cloud_provider_plugin_names() and source not in ['ec2', 'openshift_virtualization']:
return _('Credential is required for a cloud source.')
elif source == 'custom' and cred and cred.credential_type.kind in ('scm', 'ssh', 'insights', 'vault'):
return _('Credentials of type machine, source control, insights and vault are disallowed for custom inventory sources.')
@@ -1061,11 +1043,8 @@ class InventorySourceOptions(BaseModel):
"""Return the credential which is directly tied to the inventory source type."""
credential = None
for cred in self.credentials.all():
- if self.source in CLOUD_PROVIDERS:
- source = self.source.replace('ec2', 'aws')
- if source.endswith('_supported'):
- source = source[:-10]
- if cred.kind == source:
+ if self.source in discover_available_cloud_provider_plugin_names():
+ if cred.kind == self.source.replace('ec2', 'aws'):
credential = cred
break
else:
@@ -1080,7 +1059,7 @@ class InventorySourceOptions(BaseModel):
These are all credentials that should run their own inject_credential logic.
"""
special_cred = None
- if self.source in CLOUD_PROVIDERS:
+ if self.source in discover_available_cloud_provider_plugin_names():
# these have special injection logic associated with them
special_cred = self.get_cloud_credential()
extra_creds = []
diff --git a/awx/main/tests/functional/models/test_inventory.py b/awx/main/tests/functional/models/test_inventory.py
index a07ef1b21c..3a739a3b81 100644
--- a/awx/main/tests/functional/models/test_inventory.py
+++ b/awx/main/tests/functional/models/test_inventory.py
@@ -5,8 +5,8 @@ from unittest import mock
# AWX
from awx.main.models import Host, Inventory, InventorySource, InventoryUpdate, CredentialType, Credential, Job
-from awx.main.constants import CLOUD_PROVIDERS
from awx.main.utils.filters import SmartFilter
+from awx.main.utils.plugins import discover_available_cloud_provider_plugin_names
@pytest.mark.django_db
@@ -166,11 +166,11 @@ class TestInventorySourceInjectors:
def test_all_cloud_sources_covered(self):
"""Code in several places relies on the fact that the older
- CLOUD_PROVIDERS constant contains the same names as what are
+ discover_cloud_provider_plugin_names returns the same names as what are
defined within the injectors
"""
# slight exception case for constructed, because it has a FQCN but is not a cloud source
- assert set(CLOUD_PROVIDERS) | set(['constructed']) == set(InventorySource.injectors.keys())
+ assert set(discover_available_cloud_provider_plugin_names()) | set(['constructed']) == set(InventorySource.injectors.keys())
@pytest.mark.parametrize('source,filename', [('ec2', 'aws_ec2.yml'), ('openstack', 'openstack.yml'), ('gce', 'gcp_compute.yml')])
def test_plugin_filenames(self, source, filename):
diff --git a/awx/main/tests/functional/test_inventory_source_injectors.py b/awx/main/tests/functional/test_inventory_source_injectors.py
index 35e74f820d..83d9360ff5 100644
--- a/awx/main/tests/functional/test_inventory_source_injectors.py
+++ b/awx/main/tests/functional/test_inventory_source_injectors.py
@@ -9,9 +9,9 @@ from awx_plugins.interfaces._temporary_private_container_api import get_incontai
from awx.main.tasks.jobs import RunInventoryUpdate
from awx.main.models import InventorySource, Credential, CredentialType, UnifiedJob, ExecutionEnvironment
-from awx.main.constants import CLOUD_PROVIDERS, STANDARD_INVENTORY_UPDATE_ENV
+from awx.main.constants import STANDARD_INVENTORY_UPDATE_ENV
from awx.main.tests import data
-
+from awx.main.utils.plugins import discover_available_cloud_provider_plugin_names
from django.conf import settings
DATA = os.path.join(os.path.dirname(data.__file__), 'inventory')
@@ -193,7 +193,7 @@ def create_reference_data(source_dir, env, content):
@pytest.mark.django_db
-@pytest.mark.parametrize('this_kind', CLOUD_PROVIDERS)
+@pytest.mark.parametrize('this_kind', discover_available_cloud_provider_plugin_names())
def test_inventory_update_injected_content(this_kind, inventory, fake_credential_factory, mock_me):
if this_kind.endswith('_supported'):
this_kind = this_kind[:-10]
@@ -202,8 +202,6 @@ def test_inventory_update_injected_content(this_kind, inventory, fake_credential
ExecutionEnvironment.objects.create(name='Default Job EE', managed=False)
injector = InventorySource.injectors[this_kind]
- if injector.plugin_name is None:
- pytest.skip('Use of inventory plugin is not enabled for this source')
src_vars = dict(base_source_var='value_of_var')
src_vars['plugin'] = injector.get_proper_name()
diff --git a/awx/main/utils/plugins.py b/awx/main/utils/plugins.py
new file mode 100644
index 0000000000..fd891a474a
--- /dev/null
+++ b/awx/main/utils/plugins.py
@@ -0,0 +1,59 @@
+# Copyright (c) 2024 Ansible, Inc.
+# All Rights Reserved.
+
+"""
+This module contains the code responsible for extracting the lists of dynamically discovered plugins.
+"""
+
+from functools import cache
+
+
+@cache
+def discover_available_cloud_provider_plugin_names() -> list[str]:
+ """
+ Return a list of cloud plugin names available in runtime.
+
+ The discovery result is cached since it does not change throughout
+ the life cycle of the server run.
+
+ :returns: List of plugin cloud names.
+ :rtype: list[str]
+ """
+ from awx.main.models.inventory import InventorySourceOptions
+
+ plugin_names = list(InventorySourceOptions.injectors.keys())
+
+ plugin_names.remove('constructed')
+
+ return plugin_names
+
+
+@cache
+def compute_cloud_inventory_sources() -> dict[str, str]:
+ """
+ Return a dictionary of cloud provider plugin names
+ available plus source control management and constructed.
+
+ :returns: Dictionary of plugin cloud names plus source control.
+ :rtype: dict[str, str]
+ """
+
+ plugins = discover_available_cloud_provider_plugin_names()
+
+ return dict(zip(plugins, plugins), scm='scm', constructed='constructed')
+
+
+@cache
+def load_combined_inventory_source_options() -> dict[str, str]:
+ """
+ Return a dictionary of cloud provider plugin names and 'file'.
+
+ The 'file' entry is included separately since it needs to be consumed directly by the serializer.
+
+ :returns: A dictionary of cloud provider plugin names (as both keys and values) plus the 'file' entry.
+ :rtype: dict[str, str]
+ """
+
+ plugins = compute_cloud_inventory_sources()
+
+ return dict(zip(plugins, plugins), file='file')