diff options
39 files changed, 588 insertions, 144 deletions
diff --git a/.github/actions/upload_awx_devel_logs/action.yml b/.github/actions/upload_awx_devel_logs/action.yml index e8b80bc0a2..bad2cd34c3 100644 --- a/.github/actions/upload_awx_devel_logs/action.yml +++ b/.github/actions/upload_awx_devel_logs/action.yml @@ -13,7 +13,7 @@ runs: docker logs tools_awx_1 > ${{ inputs.log-filename }} - name: Upload AWX logs as artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: docker-compose-logs + name: docker-compose-logs-${{ inputs.log-filename }} path: ${{ inputs.log-filename }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e0627af4a6..d6f6e3b17c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -206,7 +206,7 @@ jobs: - name: Upload debug output if: failure() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: awx-operator-debug-output path: ${{ env.DEBUG_OUTPUT_DIR }} @@ -328,7 +328,7 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} # Upload coverage report as artifact - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: always() with: name: coverage-${{ matrix.target-regex.name }} @@ -359,19 +359,29 @@ jobs: - name: Upgrade ansible-core run: python3 -m pip install --upgrade ansible-core - - name: Download coverage artifacts - uses: actions/download-artifact@v3 + - name: Download coverage artifacts A to H + uses: actions/download-artifact@v4 with: + name: coverage-a-h + path: coverage + + - name: Download coverage artifacts I to P + uses: actions/download-artifact@v4 + with: + name: coverage-i-p + path: coverage + + - name: Download coverage artifacts Z to Z + uses: actions/download-artifact@v4 + with: + name: coverage-r-z0-9 path: coverage - name: Combine coverage run: | make COLLECTION_VERSION=100.100.100-git install_collection mkdir -p ~/.ansible/collections/ansible_collections/awx/awx/tests/output/coverage - cd coverage - for i in coverage-*; do - cp -rv $i/* ~/.ansible/collections/ansible_collections/awx/awx/tests/output/coverage/ - done + cp -rv coverage/* ~/.ansible/collections/ansible_collections/awx/awx/tests/output/coverage/ cd ~/.ansible/collections/ansible_collections/awx/awx ansible-test coverage combine --requirements ansible-test coverage html @@ -424,7 +434,7 @@ jobs: done - name: Upload coverage report as artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: awx-collection-integration-coverage-html path: ~/.ansible/collections/ansible_collections/awx/awx/tests/output/reports/coverage diff --git a/awx/main/tasks/jobs.py b/awx/main/tasks/jobs.py index 36493c4e21..2a8af43ecd 100644 --- a/awx/main/tasks/jobs.py +++ b/awx/main/tasks/jobs.py @@ -1281,6 +1281,7 @@ class RunProjectUpdate(BaseTask): 'local_path': os.path.basename(project_update.project.local_path), 'project_path': project_update.get_project_path(check_if_exists=False), # deprecated 'insights_url': settings.INSIGHTS_URL_BASE, + 'oidc_endpoint': settings.INSIGHTS_OIDC_ENDPOINT, 'awx_license_type': get_license().get('license_type', 'UNLICENSED'), 'awx_version': get_awx_version(), 'scm_url': scm_url, @@ -1447,6 +1448,11 @@ class RunProjectUpdate(BaseTask): ) return params + def build_credentials_list(self, project_update): + if project_update.scm_type == 'insights' and project_update.credential: + return [project_update.credential] + return [] + @task(queue=get_task_queuename) class RunInventoryUpdate(SourceControlMixin, BaseTask): diff --git a/awx/main/tasks/receptor.py b/awx/main/tasks/receptor.py index 2fbf6791ed..f3fb91c573 100644 --- a/awx/main/tasks/receptor.py +++ b/awx/main/tasks/receptor.py @@ -228,22 +228,24 @@ class RemoteJobError(RuntimeError): pass -def run_until_complete(node, timing_data=None, **kwargs): +def run_until_complete(node, timing_data=None, worktype='ansible-runner', ttl='20s', **kwargs): """ Runs an ansible-runner work_type on remote node, waits until it completes, then returns stdout. """ + config_data = read_receptor_config() receptor_ctl = get_receptor_ctl(config_data) use_stream_tls = getattr(get_conn_type(node, receptor_ctl), 'name', None) == "STREAMTLS" kwargs.setdefault('tlsclient', get_tls_client(config_data, use_stream_tls)) - kwargs.setdefault('ttl', '20s') + if ttl is not None: + kwargs['ttl'] = ttl kwargs.setdefault('payload', '') if work_signing_enabled(config_data): kwargs['signwork'] = True transmit_start = time.time() - result = receptor_ctl.submit_work(worktype='ansible-runner', node=node, **kwargs) + result = receptor_ctl.submit_work(worktype=worktype, node=node, **kwargs) unit_id = result['unitid'] run_start = time.time() @@ -371,7 +373,7 @@ def _convert_args_to_cli(vargs): return args -def worker_cleanup(node_name, vargs, timeout=300.0): +def worker_cleanup(node_name, vargs): args = _convert_args_to_cli(vargs) remote_command = ' '.join(args) diff --git a/awx/main/tasks/system.py b/awx/main/tasks/system.py index 6d161d2ef8..e9facd55a8 100644 --- a/awx/main/tasks/system.py +++ b/awx/main/tasks/system.py @@ -25,6 +25,7 @@ from django.utils.translation import gettext_lazy as _ from django.utils.translation import gettext_noop from django.core.cache import cache from django.core.exceptions import ObjectDoesNotExist +from django.db.models.query import QuerySet # Django-CRUM from crum import impersonate @@ -379,48 +380,68 @@ def purge_old_stdout_files(): logger.debug("Removing {}".format(os.path.join(settings.JOBOUTPUT_ROOT, f))) -def _cleanup_images_and_files(**kwargs): - if settings.IS_K8S: - return - this_inst = Instance.objects.me() - runner_cleanup_kwargs = this_inst.get_cleanup_task_kwargs(**kwargs) - if runner_cleanup_kwargs: - stdout = '' - with StringIO() as buffer: - with redirect_stdout(buffer): - ansible_runner.cleanup.run_cleanup(runner_cleanup_kwargs) - stdout = buffer.getvalue() - if '(changed: True)' in stdout: - logger.info(f'Performed local cleanup with kwargs {kwargs}, output:\n{stdout}') - - # if we are the first instance alphabetically, then run cleanup on execution nodes - checker_instance = ( - Instance.objects.filter(node_type__in=['hybrid', 'control'], node_state=Instance.States.READY, enabled=True, capacity__gt=0) - .order_by('-hostname') - .first() - ) - if checker_instance and this_inst.hostname == checker_instance.hostname: - for inst in Instance.objects.filter(node_type='execution', node_state=Instance.States.READY, enabled=True, capacity__gt=0): - runner_cleanup_kwargs = inst.get_cleanup_task_kwargs(**kwargs) - if not runner_cleanup_kwargs: - continue - try: - stdout = worker_cleanup(inst.hostname, runner_cleanup_kwargs) - if '(changed: True)' in stdout: - logger.info(f'Performed cleanup on execution node {inst.hostname} with output:\n{stdout}') - except RuntimeError: - logger.exception(f'Error running cleanup on execution node {inst.hostname}') +class CleanupImagesAndFiles: + @classmethod + def get_first_control_instance(cls) -> Instance | None: + return ( + Instance.objects.filter(node_type__in=['hybrid', 'control'], node_state=Instance.States.READY, enabled=True, capacity__gt=0) + .order_by('-hostname') + .first() + ) + + @classmethod + def get_execution_instances(cls) -> QuerySet[Instance]: + return Instance.objects.filter(node_type='execution', node_state=Instance.States.READY, enabled=True, capacity__gt=0) + + @classmethod + def run_local(cls, this_inst: Instance, **kwargs): + if settings.IS_K8S: + return + runner_cleanup_kwargs = this_inst.get_cleanup_task_kwargs(**kwargs) + if runner_cleanup_kwargs: + stdout = '' + with StringIO() as buffer: + with redirect_stdout(buffer): + ansible_runner.cleanup.run_cleanup(runner_cleanup_kwargs) + stdout = buffer.getvalue() + if '(changed: True)' in stdout: + logger.info(f'Performed local cleanup with kwargs {kwargs}, output:\n{stdout}') + + @classmethod + def run_remote(cls, this_inst: Instance, **kwargs): + # if we are the first instance alphabetically, then run cleanup on execution nodes + checker_instance = cls.get_first_control_instance() + + if checker_instance and this_inst.hostname == checker_instance.hostname: + for inst in cls.get_execution_instances(): + runner_cleanup_kwargs = inst.get_cleanup_task_kwargs(**kwargs) + if not runner_cleanup_kwargs: + continue + try: + stdout = worker_cleanup(inst.hostname, runner_cleanup_kwargs) + if '(changed: True)' in stdout: + logger.info(f'Performed cleanup on execution node {inst.hostname} with output:\n{stdout}') + except RuntimeError: + logger.exception(f'Error running cleanup on execution node {inst.hostname}') + + @classmethod + def run(cls, **kwargs): + if settings.IS_K8S: + return + this_inst = Instance.objects.me() + cls.run_local(this_inst, **kwargs) + cls.run_remote(this_inst, **kwargs) @task(queue='tower_broadcast_all') def handle_removed_image(remove_images=None): """Special broadcast invocation of this method to handle case of deleted EE""" - _cleanup_images_and_files(remove_images=remove_images, file_pattern='') + CleanupImagesAndFiles.run(remove_images=remove_images, file_pattern='') @task(queue=get_task_queuename) def cleanup_images_and_files(): - _cleanup_images_and_files(image_prune=True) + CleanupImagesAndFiles.run(image_prune=True) @task(queue=get_task_queuename) diff --git a/awx/main/tests/data/projects/README.md b/awx/main/tests/data/projects/README.md new file mode 100644 index 0000000000..26be7e425c --- /dev/null +++ b/awx/main/tests/data/projects/README.md @@ -0,0 +1,41 @@ +# Project data for live tests + +Each folder in this directory is usable as source for a project or role or collection, +which is used in tests, particularly the "awx/main/tests/live" tests. + +Although these are not git repositories, test fixtures will make copies, +and in the coppied folders, run `git init` type commands, turning them into +git repos. This is done in the locations + + - `/var/lib/awx/projects` + - `/tmp/live_tests` + +These can then be referenced for manual projects or git via the `file://` protocol. + +## debug + +This is the simplest possible case with 1 playbook with 1 debug task. + +## with_requirements + +This has a playbook that runs a task that uses a role. + +The role project is referenced in the `roles/requirements.yml` file. + +### role_requirement + +This is the source for the role that the `with_requirements` project uses. + +## test_host_query + +This has a playbook that runs a task from a custom collection module which +is registered for the host query feature. + +The collection is referenced in its `collections/requirements.yml` file. + +### host_query + +This can act as source code for a collection that enables host/event querying. + +It has a `meta/event_query.yml` file, which may provide you an example of how +to implement this in your own collection. diff --git a/awx/main/tests/data/projects/debug/debug.yml b/awx/main/tests/data/projects/debug/debug.yml new file mode 100644 index 0000000000..f4fdcb2f0e --- /dev/null +++ b/awx/main/tests/data/projects/debug/debug.yml @@ -0,0 +1,6 @@ +--- +- hosts: all + gather_facts: false + connection: local + tasks: + - debug: msg='hello' diff --git a/awx/main/tests/data/projects/host_query/galaxy.yml b/awx/main/tests/data/projects/host_query/galaxy.yml new file mode 100644 index 0000000000..a69203d416 --- /dev/null +++ b/awx/main/tests/data/projects/host_query/galaxy.yml @@ -0,0 +1,19 @@ +--- +authors: + - AWX Project Contributors <awx-project@googlegroups.com> +dependencies: {} +description: Indirect host counting example repo. Not for use in production. +documentation: https://github.com/ansible/awx +homepage: https://github.com/ansible/awx +issues: https://github.com/ansible/awx +license: + - GPL-3.0-or-later +name: query +namespace: demo +readme: README.md +repository: https://github.com/ansible/awx +tags: + - demo + - testing + - host_counting +version: 0.0.1 diff --git a/awx/main/tests/data/projects/host_query/meta/event_query.yml b/awx/main/tests/data/projects/host_query/meta/event_query.yml new file mode 100644 index 0000000000..0c9e398c66 --- /dev/null +++ b/awx/main/tests/data/projects/host_query/meta/event_query.yml @@ -0,0 +1,4 @@ +--- +{ + "demo.query.example": "" +} diff --git a/awx/main/tests/data/projects/host_query/plugins/modules/example.py b/awx/main/tests/data/projects/host_query/plugins/modules/example.py new file mode 100644 index 0000000000..aa323e7c28 --- /dev/null +++ b/awx/main/tests/data/projects/host_query/plugins/modules/example.py @@ -0,0 +1,74 @@ +#!/usr/bin/python + +# Same licensing as AWX +from __future__ import absolute_import, division, print_function + +__metaclass__ = type + +DOCUMENTATION = r''' +--- +module: example + +short_description: Module for specific live tests + +version_added: "2.0.0" + +description: This module is part of a test collection in local source. + +options: + host_name: + description: Name to return as the host name. + required: false + type: str + +author: + - AWX Live Tests +''' + +EXAMPLES = r''' +- name: Test with defaults + demo.query.example: + +- name: Test with custom host name + demo.query.example: + host_name: foo_host +''' + +RETURN = r''' +direct_host_name: + description: The name of the host, this will be collected with the feature. + type: str + returned: always + sample: 'foo_host' +''' + +from ansible.module_utils.basic import AnsibleModule + + +def run_module(): + module_args = dict( + host_name=dict(type='str', required=False, default='foo_host_default'), + ) + + result = dict( + changed=False, + other_data='sample_string', + ) + + module = AnsibleModule(argument_spec=module_args, supports_check_mode=True) + + if module.check_mode: + module.exit_json(**result) + + result['direct_host_name'] = module.params['host_name'] + result['nested_host_name'] = {'host_name': module.params['host_name']} + + module.exit_json(**result) + + +def main(): + run_module() + + +if __name__ == '__main__': + main() diff --git a/awx/main/tests/data/projects/role_requirement/meta/main.yml b/awx/main/tests/data/projects/role_requirement/meta/main.yml new file mode 100644 index 0000000000..25563e6d68 --- /dev/null +++ b/awx/main/tests/data/projects/role_requirement/meta/main.yml @@ -0,0 +1,19 @@ +--- +galaxy_info: + author: "For Test" + company: AWX + license: MIT + min_ansible_version: 1.4 + platforms: + - name: EL + versions: + - 8 + - 9 + - name: Fedora + versions: + - 39 + - 40 + - 41 + categories: + - stuff +dependencies: [] diff --git a/awx/main/tests/data/projects/role_requirement/tasks/main.yml b/awx/main/tests/data/projects/role_requirement/tasks/main.yml new file mode 100644 index 0000000000..2dbafc91cf --- /dev/null +++ b/awx/main/tests/data/projects/role_requirement/tasks/main.yml @@ -0,0 +1,4 @@ +--- +- name: debug variable + debug: + msg: "1234567890" diff --git a/awx/main/tests/data/projects/test_host_query/collections/requirements.yml b/awx/main/tests/data/projects/test_host_query/collections/requirements.yml new file mode 100644 index 0000000000..17e176ae39 --- /dev/null +++ b/awx/main/tests/data/projects/test_host_query/collections/requirements.yml @@ -0,0 +1,5 @@ +--- +collections: + - name: 'file:///tmp/live_tests/host_query' + type: git + version: devel diff --git a/awx/main/tests/data/projects/test_host_query/run_task.yml b/awx/main/tests/data/projects/test_host_query/run_task.yml new file mode 100644 index 0000000000..2d23555c63 --- /dev/null +++ b/awx/main/tests/data/projects/test_host_query/run_task.yml @@ -0,0 +1,8 @@ +--- +- hosts: all + gather_facts: false + connection: local + tasks: + - demo.query.example: + register: result + - debug: var=result diff --git a/awx/main/tests/data/projects/with_requirements/roles/requirements.yml b/awx/main/tests/data/projects/with_requirements/roles/requirements.yml new file mode 100644 index 0000000000..b4eb43576f --- /dev/null +++ b/awx/main/tests/data/projects/with_requirements/roles/requirements.yml @@ -0,0 +1,3 @@ +--- +- name: role_requirement + src: git+file:///tmp/live_tests/role_requirement diff --git a/awx/main/tests/data/projects/with_requirements/use_requirement.yml b/awx/main/tests/data/projects/with_requirements/use_requirement.yml new file mode 100644 index 0000000000..7907d662d2 --- /dev/null +++ b/awx/main/tests/data/projects/with_requirements/use_requirement.yml @@ -0,0 +1,7 @@ +--- +- hosts: all + connection: local + gather_facts: false + tasks: + - include_role: + name: role_requirement diff --git a/awx/main/tests/factories/fixtures.py b/awx/main/tests/factories/fixtures.py index 9f4229718d..6f9a3263ac 100644 --- a/awx/main/tests/factories/fixtures.py +++ b/awx/main/tests/factories/fixtures.py @@ -99,11 +99,19 @@ def mk_user(name, is_superuser=False, organization=None, team=None, persisted=Tr def mk_project(name, organization=None, description=None, persisted=True): description = description or '{}-description'.format(name) - project = Project(name=name, description=description, playbook_files=['helloworld.yml', 'alt-helloworld.yml']) + project = Project( + name=name, + description=description, + playbook_files=['helloworld.yml', 'alt-helloworld.yml'], + scm_type='git', + scm_url='https://foo.invalid', + scm_revision='1234567890123456789012345678901234567890', + scm_update_on_launch=False, + ) if organization is not None: project.organization = organization if persisted: - project.save() + project.save(skip_update=True) return project diff --git a/awx/main/tests/functional/test_api_generics.py b/awx/main/tests/functional/api/test_api_generics.py index 029413af51..029413af51 100644 --- a/awx/main/tests/functional/test_api_generics.py +++ b/awx/main/tests/functional/api/test_api_generics.py diff --git a/awx/main/tests/functional/api/test_unified_job_template.py b/awx/main/tests/functional/api/test_unified_job_template.py index 1a9adc3965..c293827e43 100644 --- a/awx/main/tests/functional/api/test_unified_job_template.py +++ b/awx/main/tests/functional/api/test_unified_job_template.py @@ -18,7 +18,7 @@ class TestUnifiedOrganization: def data_for_model(self, model, orm_style=False): data = {'name': 'foo', 'organization': None} if model == 'JobTemplate': - proj = models.Project.objects.create(name="test-proj", playbook_files=['helloworld.yml']) + proj = models.Project.objects.create(name="test-proj", playbook_files=['helloworld.yml'], scm_type='git', scm_url='https://foo.invalid') if orm_style: data['project_id'] = proj.id else: diff --git a/awx/main/tests/functional/conftest.py b/awx/main/tests/functional/conftest.py index 288e5264c8..8fca93e7f9 100644 --- a/awx/main/tests/functional/conftest.py +++ b/awx/main/tests/functional/conftest.py @@ -114,20 +114,6 @@ def team_member(user, team): return ret -@pytest.fixture(scope="session", autouse=True) -def project_playbooks(): - """ - Return playbook_files as playbooks for manual projects when testing. - """ - - class PlaybooksMock(mock.PropertyMock): - def __get__(self, obj, obj_type): - return obj.playbook_files - - mocked = mock.patch.object(Project, 'playbooks', new_callable=PlaybooksMock) - mocked.start() - - @pytest.fixture def run_computed_fields_right_away(request): def run_me(inventory_id): diff --git a/awx/main/tests/functional/test_db_credential.py b/awx/main/tests/functional/models/test_db_credential.py index 9ed823deb9..9ed823deb9 100644 --- a/awx/main/tests/functional/test_db_credential.py +++ b/awx/main/tests/functional/models/test_db_credential.py diff --git a/awx/main/tests/functional/test_ha.py b/awx/main/tests/functional/models/test_ha.py index 0b4ced53c2..0b4ced53c2 100644 --- a/awx/main/tests/functional/test_ha.py +++ b/awx/main/tests/functional/models/test_ha.py diff --git a/awx/main/tests/functional/test_labels.py b/awx/main/tests/functional/rbac/test_rbac_labels.py index aaf74e41e4..aaf74e41e4 100644 --- a/awx/main/tests/functional/test_labels.py +++ b/awx/main/tests/functional/rbac/test_rbac_labels.py diff --git a/awx/main/tests/functional/tasks/test_tasks_jobs.py b/awx/main/tests/functional/tasks/test_tasks_jobs.py new file mode 100644 index 0000000000..a627731bbb --- /dev/null +++ b/awx/main/tests/functional/tasks/test_tasks_jobs.py @@ -0,0 +1,27 @@ +import pytest +import os + +from awx.main.tasks.jobs import RunJob +from awx.main.models import Job + + +@pytest.fixture +def scm_revision_file(tmpdir_factory): + # Returns path to temporary testing revision file + revision_file = tmpdir_factory.mktemp('revisions').join('revision.txt') + with open(str(revision_file), 'w') as f: + f.write('1234567890123456789012345678901234567890') + return os.path.join(revision_file.dirname, 'revision.txt') + + +@pytest.mark.django_db +def test_does_not_run_reaped_job(mocker, mock_me): + job = Job.objects.create(status='failed', job_explanation='This job has been reaped.') + mock_run = mocker.patch('awx.main.tasks.jobs.ansible_runner.interface.run') + try: + RunJob().run(job.id) + except Exception: + pass + job.refresh_from_db() + assert job.status == 'failed' + mock_run.assert_not_called() diff --git a/awx/main/tests/functional/test_tasks.py b/awx/main/tests/functional/tasks/test_tasks_system.py index ba252080c2..6fb3acbfba 100644 --- a/awx/main/tests/functional/test_tasks.py +++ b/awx/main/tests/functional/tasks/test_tasks_system.py @@ -1,28 +1,37 @@ -import pytest import os import tempfile import shutil -from awx.main.tasks.jobs import RunJob -from awx.main.tasks.system import execution_node_health_check, _cleanup_images_and_files -from awx.main.models import Instance, Job +import pytest - -@pytest.fixture -def scm_revision_file(tmpdir_factory): - # Returns path to temporary testing revision file - revision_file = tmpdir_factory.mktemp('revisions').join('revision.txt') - with open(str(revision_file), 'w') as f: - f.write('1234567890123456789012345678901234567890') - return os.path.join(revision_file.dirname, 'revision.txt') +from awx.main.tasks.system import CleanupImagesAndFiles, execution_node_health_check, inspect_established_receptor_connections +from awx.main.models import Instance, Job, ReceptorAddress, InstanceLink @pytest.mark.django_db -@pytest.mark.parametrize('node_type', ('control. hybrid')) -def test_no_worker_info_on_AWX_nodes(node_type): - hostname = 'us-south-3-compute.invalid' - Instance.objects.create(hostname=hostname, node_type=node_type) - assert execution_node_health_check(hostname) is None +class TestLinkState: + @pytest.fixture(autouse=True) + def configure_settings(self, settings): + settings.IS_K8S = True + + def test_inspect_established_receptor_connections(self): + ''' + Change link state from ADDING to ESTABLISHED + if the receptor status KnownConnectionCosts field + has an entry for the source and target node. + ''' + hop1 = Instance.objects.create(hostname='hop1') + hop2 = Instance.objects.create(hostname='hop2') + hop2addr = ReceptorAddress.objects.create(instance=hop2, address='hop2', port=5678) + InstanceLink.objects.create(source=hop1, target=hop2addr, link_state=InstanceLink.States.ADDING) + + # calling with empty KnownConnectionCosts should not change the link state + inspect_established_receptor_connections({"KnownConnectionCosts": {}}) + assert InstanceLink.objects.get(source=hop1, target=hop2addr).link_state == InstanceLink.States.ADDING + + mesh_state = {"KnownConnectionCosts": {"hop1": {"hop2": 1}}} + inspect_established_receptor_connections(mesh_state) + assert InstanceLink.objects.get(source=hop1, target=hop2addr).link_state == InstanceLink.States.ESTABLISHED @pytest.fixture @@ -47,23 +56,31 @@ def mock_job_folder(job_folder_factory): @pytest.mark.django_db +@pytest.mark.parametrize('node_type', ('control. hybrid')) +def test_no_worker_info_on_AWX_nodes(node_type): + hostname = 'us-south-3-compute.invalid' + Instance.objects.create(hostname=hostname, node_type=node_type) + assert execution_node_health_check(hostname) is None + + +@pytest.mark.django_db def test_folder_cleanup_stale_file(mock_job_folder, mock_me): - _cleanup_images_and_files() + CleanupImagesAndFiles.run() assert os.path.exists(mock_job_folder) # grace period should protect folder from deletion - _cleanup_images_and_files(grace_period=0) + CleanupImagesAndFiles.run(grace_period=0) assert not os.path.exists(mock_job_folder) # should be deleted @pytest.mark.django_db def test_folder_cleanup_running_job(mock_job_folder, me_inst): job = Job.objects.create(id=1234, controller_node=me_inst.hostname, status='running') - _cleanup_images_and_files(grace_period=0) + CleanupImagesAndFiles.run(grace_period=0) assert os.path.exists(mock_job_folder) # running job should prevent folder from getting deleted job.status = 'failed' job.save(update_fields=['status']) - _cleanup_images_and_files(grace_period=0) + CleanupImagesAndFiles.run(grace_period=0) assert not os.path.exists(mock_job_folder) # job is finished and no grace period, should delete @@ -78,19 +95,6 @@ def test_folder_cleanup_multiple_running_jobs(job_folder_factory, me_inst): dirs.append(job_folder_factory(job.id)) jobs.append(job) - _cleanup_images_and_files(grace_period=0) + CleanupImagesAndFiles.run(grace_period=0) assert [os.path.exists(d) for d in dirs] == [True for i in range(num_jobs)] - - -@pytest.mark.django_db -def test_does_not_run_reaped_job(mocker, mock_me): - job = Job.objects.create(status='failed', job_explanation='This job has been reaped.') - mock_run = mocker.patch('awx.main.tasks.jobs.ansible_runner.interface.run') - try: - RunJob().run(job.id) - except Exception: - pass - job.refresh_from_db() - assert job.status == 'failed' - mock_run.assert_not_called() diff --git a/awx/main/tests/functional/test_linkstate.py b/awx/main/tests/functional/test_linkstate.py deleted file mode 100644 index 478883870a..0000000000 --- a/awx/main/tests/functional/test_linkstate.py +++ /dev/null @@ -1,30 +0,0 @@ -import pytest - -from awx.main.models import Instance, ReceptorAddress, InstanceLink -from awx.main.tasks.system import inspect_established_receptor_connections - - -@pytest.mark.django_db -class TestLinkState: - @pytest.fixture(autouse=True) - def configure_settings(self, settings): - settings.IS_K8S = True - - def test_inspect_established_receptor_connections(self): - ''' - Change link state from ADDING to ESTABLISHED - if the receptor status KnownConnectionCosts field - has an entry for the source and target node. - ''' - hop1 = Instance.objects.create(hostname='hop1') - hop2 = Instance.objects.create(hostname='hop2') - hop2addr = ReceptorAddress.objects.create(instance=hop2, address='hop2', port=5678) - InstanceLink.objects.create(source=hop1, target=hop2addr, link_state=InstanceLink.States.ADDING) - - # calling with empty KnownConnectionCosts should not change the link state - inspect_established_receptor_connections({"KnownConnectionCosts": {}}) - assert InstanceLink.objects.get(source=hop1, target=hop2addr).link_state == InstanceLink.States.ADDING - - mesh_state = {"KnownConnectionCosts": {"hop1": {"hop2": 1}}} - inspect_established_receptor_connections(mesh_state) - assert InstanceLink.objects.get(source=hop1, target=hop2addr).link_state == InstanceLink.States.ESTABLISHED diff --git a/awx/main/tests/functional/test_projects.py b/awx/main/tests/functional/test_projects.py index 17eda7f58f..f4ea052596 100644 --- a/awx/main/tests/functional/test_projects.py +++ b/awx/main/tests/functional/test_projects.py @@ -335,7 +335,7 @@ def test_team_project_list(get, team_project_list): @pytest.mark.parametrize("u,expected_status_code", [('rando', 403), ('org_member', 403), ('org_admin', 201), ('admin', 201)]) -@pytest.mark.django_db() +@pytest.mark.django_db def test_create_project(post, organization, org_admin, org_member, admin, rando, u, expected_status_code): if u == 'rando': u = rando @@ -353,11 +353,12 @@ def test_create_project(post, organization, org_admin, org_member, admin, rando, 'organization': organization.id, }, u, + expect=expected_status_code, ) - print(result.data) - assert result.status_code == expected_status_code if expected_status_code == 201: assert Project.objects.filter(name='Project', organization=organization).exists() + elif expected_status_code == 403: + assert 'do not have permission' in str(result.data['detail']) @pytest.mark.django_db diff --git a/awx/main/tests/live/tests/conftest.py b/awx/main/tests/live/tests/conftest.py index f7c197199f..796e37cb0b 100644 --- a/awx/main/tests/live/tests/conftest.py +++ b/awx/main/tests/live/tests/conftest.py @@ -1,3 +1,4 @@ +import subprocess import time import pytest @@ -7,7 +8,7 @@ import pytest from awx.main.tests.functional.conftest import * # noqa from awx.main.tests.conftest import load_all_credentials # noqa: F401; pylint: disable=unused-import -from awx.main.models import Organization +from awx.main.models import Organization, Inventory def wait_to_leave_status(job, status, timeout=30, sleep_time=0.1): @@ -25,12 +26,26 @@ def wait_to_leave_status(job, status, timeout=30, sleep_time=0.1): raise RuntimeError(f'Job failed to exit {status} in {timeout} seconds. job_explanation={job.job_explanation} tb={job.result_traceback}') +def wait_for_events(uj, timeout=2): + start = time.time() + while uj.event_processing_finished is False: + time.sleep(0.2) + uj.refresh_from_db() + if time.time() - start > timeout: + break + + +def unified_job_stdout(uj): + wait_for_events(uj) + return '\n'.join([event.stdout for event in uj.get_event_queryset().order_by('created')]) + + def wait_for_job(job, final_status='successful', running_timeout=800): wait_to_leave_status(job, 'pending') wait_to_leave_status(job, 'waiting') wait_to_leave_status(job, 'running', timeout=running_timeout) - assert job.status == final_status, f'Job was not successful id={job.id} status={job.status}' + assert job.status == final_status, f'Job was not successful id={job.id} status={job.status} tb={job.result_traceback} output=\n{unified_job_stdout(job)}' @pytest.fixture(scope='session') @@ -39,3 +54,26 @@ def default_org(): if org is None: raise Exception('Tests expect Default org to already be created and it is not') return org + + +@pytest.fixture(scope='session') +def demo_inv(default_org): + inventory, _ = Inventory.objects.get_or_create(name='Demo Inventory', defaults={'organization': default_org}) + return inventory + + +@pytest.fixture +def podman_image_generator(): + """ + Generate a tagless podman image from awx base EE + """ + + def fn(): + dockerfile = """ + FROM quay.io/ansible/awx-ee:latest + RUN echo "Hello, Podman!" > /tmp/hello.txt + """ + cmd = ['podman', 'build', '-f', '-'] # Create an image without a tag + subprocess.run(cmd, capture_output=True, input=dockerfile, text=True, check=True) + + return fn diff --git a/awx/main/tests/live/tests/projects/conftest.py b/awx/main/tests/live/tests/projects/conftest.py new file mode 100644 index 0000000000..cfa8b07cb5 --- /dev/null +++ b/awx/main/tests/live/tests/projects/conftest.py @@ -0,0 +1,115 @@ +import pytest +import os +import shutil +import tempfile +import subprocess + +from django.conf import settings + +from awx.api.versioning import reverse +from awx.main.tests import data +from awx.main.models import Project, JobTemplate + +from awx.main.tests.live.tests.conftest import wait_for_job + +PROJ_DATA = os.path.join(os.path.dirname(data.__file__), 'projects') + + +def _copy_folders(source_path, dest_path, clear=False): + "folder-by-folder, copy dirs in the source root dir to the destination root dir" + for dirname in os.listdir(source_path): + source_dir = os.path.join(source_path, dirname) + expected_dir = os.path.join(dest_path, dirname) + if clear and os.path.exists(expected_dir): + shutil.rmtree(expected_dir) + if (not os.path.isdir(source_dir)) or os.path.exists(expected_dir): + continue + shutil.copytree(source_dir, expected_dir) + + +@pytest.fixture(scope='session') +def copy_project_folders(): + proj_root = settings.PROJECTS_ROOT + if not os.path.exists(proj_root): + os.mkdir(proj_root) + _copy_folders(PROJ_DATA, proj_root, clear=True) + + +GIT_COMMANDS = ( + 'git config --global init.defaultBranch devel; ' + 'git init; ' + 'git config user.email jenkins@ansible.com; ' + 'git config user.name DoneByTest; ' + 'git add .; ' + 'git commit -m "initial commit"' +) + + +@pytest.fixture(scope='session') +def live_tmp_folder(): + path = os.path.join(tempfile.gettempdir(), 'live_tests') + if os.path.exists(path): + shutil.rmtree(path) + os.mkdir(path) + _copy_folders(PROJ_DATA, path) + for dirname in os.listdir(path): + source_dir = os.path.join(path, dirname) + subprocess.run(GIT_COMMANDS, cwd=source_dir, shell=True) + if path not in settings.AWX_ISOLATION_SHOW_PATHS: + settings.AWX_ISOLATION_SHOW_PATHS = settings.AWX_ISOLATION_SHOW_PATHS + [path] + return path + + +@pytest.fixture +def run_job_from_playbook(default_org, demo_inv, post, admin): + def _rf(test_name, playbook, local_path=None, scm_url=None): + project_name = f'{test_name} project' + jt_name = f'{test_name} JT: {playbook}' + + old_proj = Project.objects.filter(name=project_name).first() + if old_proj: + old_proj.delete() + + old_jt = JobTemplate.objects.filter(name=jt_name).first() + if old_jt: + old_jt.delete() + + proj_kwargs = {'name': project_name, 'organization': default_org.id} + if local_path: + # manual path + proj_kwargs['scm_type'] = '' + proj_kwargs['local_path'] = local_path + elif scm_url: + proj_kwargs['scm_type'] = 'git' + proj_kwargs['scm_url'] = scm_url + else: + raise RuntimeError('Need to provide scm_url or local_path') + + result = post( + reverse('api:project_list'), + proj_kwargs, + admin, + expect=201, + ) + proj = Project.objects.get(id=result.data['id']) + + if proj.current_job: + wait_for_job(proj.current_job) + + assert proj.get_project_path() + assert playbook in proj.playbooks + + result = post( + reverse('api:job_template_list'), + {'name': jt_name, 'project': proj.id, 'playbook': playbook, 'inventory': demo_inv.id}, + admin, + expect=201, + ) + jt = JobTemplate.objects.get(id=result.data['id']) + job = jt.create_unified_job() + job.signal_start() + + wait_for_job(job) + assert job.status == 'successful' + + return _rf diff --git a/awx/main/tests/live/tests/projects/test_file_projects.py b/awx/main/tests/live/tests/projects/test_file_projects.py new file mode 100644 index 0000000000..a2872745b3 --- /dev/null +++ b/awx/main/tests/live/tests/projects/test_file_projects.py @@ -0,0 +1,2 @@ +def test_git_file_project(live_tmp_folder, run_job_from_playbook): + run_job_from_playbook('test_git_file_project', 'debug.yml', scm_url=f'file://{live_tmp_folder}/debug') diff --git a/awx/main/tests/live/tests/projects/test_indirect_host_counting.py b/awx/main/tests/live/tests/projects/test_indirect_host_counting.py new file mode 100644 index 0000000000..1a2027dcfe --- /dev/null +++ b/awx/main/tests/live/tests/projects/test_indirect_host_counting.py @@ -0,0 +1,3 @@ +def test_indirect_host_counting(live_tmp_folder, run_job_from_playbook): + run_job_from_playbook('test_indirect_host_counting', 'run_task.yml', scm_url=f'file://{live_tmp_folder}/test_host_query') + # TODO: add assertions that the host query data is populated diff --git a/awx/main/tests/live/tests/projects/test_manual_project.py b/awx/main/tests/live/tests/projects/test_manual_project.py new file mode 100644 index 0000000000..11aeb76cf8 --- /dev/null +++ b/awx/main/tests/live/tests/projects/test_manual_project.py @@ -0,0 +1,2 @@ +def test_manual_project(copy_project_folders, run_job_from_playbook): + run_job_from_playbook('test_manual_project', 'debug.yml', local_path='debug') diff --git a/awx/main/tests/live/tests/projects/test_requirements.py b/awx/main/tests/live/tests/projects/test_requirements.py index c82ccbec80..c0d3929969 100644 --- a/awx/main/tests/live/tests/projects/test_requirements.py +++ b/awx/main/tests/live/tests/projects/test_requirements.py @@ -5,9 +5,9 @@ import pytest from django.conf import settings -from awx.main.tests.live.tests.conftest import wait_for_job +from awx.main.tests.live.tests.conftest import wait_for_job, wait_for_events -from awx.main.models import Project, SystemJobTemplate +from awx.main.models import Project, SystemJobTemplate, Job @pytest.fixture(scope='session') @@ -54,3 +54,11 @@ def test_cache_is_populated_after_cleanup_job(project_with_requirements): # Now, we still have a populated cache assert project_cache_is_populated(project_with_requirements) + + +def test_git_file_collection_requirement(live_tmp_folder, copy_project_folders, run_job_from_playbook): + # this behaves differently, as use_requirements.yml references only the folder, does not include the github name + run_job_from_playbook('test_git_file_collection_requirement', 'use_requirement.yml', scm_url=f'file://{live_tmp_folder}/with_requirements') + job = Job.objects.filter(name__icontains='test_git_file_collection_requirement').order_by('-created').first() + wait_for_events(job) + assert '1234567890' in job.job_events.filter(task='debug variable', event='runner_on_ok').first().stdout diff --git a/awx/main/tests/live/tests/test_cleanup_task.py b/awx/main/tests/live/tests/test_cleanup_task.py index 137032b48c..e9af90b961 100644 --- a/awx/main/tests/live/tests/test_cleanup_task.py +++ b/awx/main/tests/live/tests/test_cleanup_task.py @@ -1,11 +1,21 @@ import os +import json +import pytest import tempfile import subprocess -from awx.main.tasks.receptor import _convert_args_to_cli +from unittest import mock + +from awx.main.tasks.receptor import _convert_args_to_cli, run_until_complete +from awx.main.tasks.system import CleanupImagesAndFiles from awx.main.models import Instance, JobTemplate +def get_podman_images(): + cmd = ['podman', 'images', '--format', 'json'] + return json.loads((subprocess.run(cmd, capture_output=True, text=True, check=True)).stdout) + + def test_folder_cleanup_multiple_running_jobs_execution_node(request): demo_jt = JobTemplate.objects.get(name='Demo Job Template') @@ -37,3 +47,36 @@ def test_folder_cleanup_multiple_running_jobs_execution_node(request): print('ansible-runner worker ' + remote_command) assert [os.path.exists(job_dir) for job_dir in job_dirs] == [True for i in range(3)] + + +@pytest.mark.parametrize( + 'worktype', + ('remote', 'local'), +) +def test_tagless_image(podman_image_generator, worktype: str): + """ + Ensure podman images on Control and Hybrid nodes are deleted during cleanup. + """ + podman_image_generator() + + dangling_image = next((image for image in get_podman_images() if image.get('Dangling', False)), None) + assert dangling_image + + instance_me = Instance.objects.me() + + match worktype: + case 'local': + CleanupImagesAndFiles.run_local(instance_me, image_prune=True) + case 'remote': + with ( + mock.patch( + 'awx.main.tasks.receptor.run_until_complete', lambda *args, **kwargs: run_until_complete(*args, worktype='local', ttl=None, **kwargs) + ), + mock.patch('awx.main.tasks.system.CleanupImagesAndFiles.get_execution_instances', lambda: [Instance.objects.me()]), + ): + CleanupImagesAndFiles.run_remote(instance_me, image_prune=True) + case _: + raise ValueError(f'worktype "{worktype}" not supported.') + + for image in get_podman_images(): + assert image['Id'] != dangling_image['Id'] diff --git a/awx/playbooks/action_plugins/insights.py b/awx/playbooks/action_plugins/insights.py index c2e63789b6..e3f9b9b6e8 100644 --- a/awx/playbooks/action_plugins/insights.py +++ b/awx/playbooks/action_plugins/insights.py @@ -6,10 +6,11 @@ import os import re import requests -from urllib.parse import urljoin from ansible.plugins.action import ActionBase +DEFAULT_OIDC_ENDPOINT = 'https://sso.redhat.com/auth/realms/redhat-external' + class ActionModule(ActionBase): def save_playbook(self, proj_path, remediation, content): @@ -36,7 +37,9 @@ class ActionModule(ActionBase): f.write(etag) def _obtain_auth_token(self, oidc_endpoint, client_id, client_secret): - main_url = urljoin(oidc_endpoint, '/.well-known/openid-configuration') + if oidc_endpoint.endswith('/'): + oidc_endpoint = oidc_endpoint.rstrip('/') + main_url = oidc_endpoint + '/.well-known/openid-configuration' response = requests.get(url=main_url, headers={'Accept': 'application/json'}) data = {} if response.status_code != 200: @@ -80,7 +83,7 @@ class ActionModule(ActionBase): password = self._task.args.get('password', None) client_id = self._task.args.get('client_id', None) client_secret = self._task.args.get('client_secret', None) - oidc_endpoint = self._task.args.get('oidc_endpoint', None) + oidc_endpoint = self._task.args.get('oidc_endpoint', DEFAULT_OIDC_ENDPOINT) session.headers.update( { @@ -95,7 +98,7 @@ class ActionModule(ActionBase): result['failed'] = data['failed'] result['msg'] = data['msg'] return result - session.headers.update({'Authorization': f'{result['token_type']} {result['token']}'}) + session.headers.update({'Authorization': f'{data["token_type"]} {data["token"]}'}) elif authentication == 'basic' or (username and password): session.auth = requests.auth.HTTPBasicAuth(username, password) diff --git a/awx/playbooks/project_update.yml b/awx/playbooks/project_update.yml index b95796f0bb..2f4ab183c7 100644 --- a/awx/playbooks/project_update.yml +++ b/awx/playbooks/project_update.yml @@ -21,7 +21,9 @@ # gpg_pubkey: the GPG public key to use for validation, when enabled # client_id: Red Hat service account client ID; required for the 'service_account' authentication method used against the Insights API # client_secret: Red Hat service account client secret; required for the 'service_account' authentication method used against the Insights API -# oidc_endpoint: OpenID Connect URL for 'service_account' authentication method. +# authentication: The authentication method to use against the Insights API +# client_id and client_secret are required for the 'service_account' authentication method +# scm_username and scm_password are required for the 'basic' authentication method - hosts: localhost gather_facts: false diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index fe4364e03c..b5d22b1c37 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -697,6 +697,7 @@ DISABLE_LOCAL_AUTH = False TOWER_URL_BASE = "https://platformhost" INSIGHTS_URL_BASE = "https://example.org" +INSIGHTS_OIDC_ENDPOINT = "https://sso.example.org" INSIGHTS_AGENT_MIME = 'application/example' # See https://github.com/ansible/awx-facts-playbooks INSIGHTS_SYSTEM_ID_FILE = '/etc/redhat-access-insights/machine-id' @@ -1060,4 +1061,4 @@ ANSIBLE_BASE_ALLOW_SINGLETON_ROLES_API = False # Do not allow creating user-def SYSTEM_USERNAME = None # feature flags -FLAGS = {} +FLAGS = {'FEATURE_INDIRECT_NODE_COUNTING_ENABLED': [{'condition': 'boolean', 'value': False}]} diff --git a/awxkit/awxkit/api/pages/credentials.py b/awxkit/awxkit/api/pages/credentials.py index 75c724a8d5..d7d95aa18b 100644 --- a/awxkit/awxkit/api/pages/credentials.py +++ b/awxkit/awxkit/api/pages/credentials.py @@ -134,6 +134,8 @@ def get_payload_field_and_value_from_kwargs_or_config_cred(field, kind, kwargs, config_field = 'ad_user' elif field == 'client': config_field = 'client_id' + elif field == 'client_id' and 'azure' in kind: # Needed to avoid service account client_id collision + config_field = '' elif field == 'authorize_password': config_field = 'authorize' else: diff --git a/licenses/psycopg-3.1.18.tar.gz b/licenses/psycopg-3.1.18.tar.gz Binary files differdeleted file mode 100644 index 81ccd5fecd..0000000000 --- a/licenses/psycopg-3.1.18.tar.gz +++ /dev/null |