Create a new pytest folder for live system testing with normal services (#15688)

* PoC for running dev env tests

* Replace in github actions

* Try non interactive

* Move folder to better location

* Further streamlining of new test folders

* Consolidate fixture, add writeup docs

* Use star import

* Push the wait-for-job to the conftest

8 files changed, 105 insertions, 62 deletions

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 08479c3946..4b2ec43e6a 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -139,8 +139,8 @@ jobs:
           build-ui: false
           github-token: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Run smoke test
-        run: ansible-playbook tools/docker-compose/ansible/smoke-test.yml -v
+      - name: Run live dev env tests
+        run: docker exec tools_awx_1 /bin/bash -c "make live_test"
 
   awx-operator:
     runs-on: ubuntu-latest
diff --git a/Makefile b/Makefile
index e9a1495644..3bf198f417 100644
--- a/Makefile
+++ b/Makefile
@@ -350,6 +350,9 @@ test:
 	cd awxkit && $(VENV_BASE)/awx/bin/tox -re py3
 	awx-manage check_migrations --dry-run --check  -n 'missing_migration_file'
 
+live_test:
+	cd awx/main/tests/live && py.test tests/
+
 ## Run all API unit tests with coverage enabled.
 test_coverage:
 	$(MAKE) test PYTEST_ARGS="--create-db --cov --cov-report=xml --junitxml=reports/junit.xml"
diff --git a/awx/main/tests/README.md b/awx/main/tests/README.md
new file mode 100644
index 0000000000..f6aac6418e
--- /dev/null
+++ b/awx/main/tests/README.md
@@ -0,0 +1,42 @@
+## Test Environments
+
+Several of the subfolders of `awx/main/tests/` indicate a different required _environment_
+where you can run the tests. Those folders are:
+
+ - `functional/` - requires a test database and no other services running
+ - `live/` - must run in `tools_awx_1` container launched by `make docker-compose`
+ - `unit/` - does not require a test database or any active services
+
+### Functional and unit test environment
+
+The functional and unit tests have an invocation in `make test`,
+and this attaches several other things like schema that piggybacks on requests.
+These tests are ran from the root AWX folder.
+
+#### Functional tests
+
+Only tests in the `functional/` folder should use the `@pytest.mark.django_db` decorator.
+This is the only difference between the functional and unit folders,
+the test environment is otherwise the same for both.
+
+Functional tests use a sqlite3 database, so the postgres service is not necessary.
+
+### Live tests
+
+The live tests have an invocation in `make live_test` which will change
+directory before running, which is required to pick up a different pytest
+configuration.
+
+This will use the postges container from `make docker-compose` for the database,
+and will disable the pytest-django features of running with a test database
+and running tests in transactions.
+This means that any changes done in the course of the test could potentially
+be seen in your browser via the API or UI, and anything the test fails
+to clean up will remain in the database.
+
+### Folders that should not contain tests
+
+ - `data/` - just files other tests use
+ - `docs/` - utilities for schema generation
+ - `factories/` - general utilities
+ - `manual/` - python files to be ran directly
diff --git a/awx/main/tests/live/pytest.ini b/awx/main/tests/live/pytest.ini
new file mode 100644
index 0000000000..cc1e9c5a91
--- /dev/null
+++ b/awx/main/tests/live/pytest.ini
@@ -0,0 +1,3 @@
+# This file is needed to undo the pytest settings from the project root
+[pytest]
+addopts =  -p no:django -p awx.main.tests.live.pytest_django_config
diff --git a/awx/main/tests/live/pytest_django_config.py b/awx/main/tests/live/pytest_django_config.py
new file mode 100644
index 0000000000..8c4eac2474
--- /dev/null
+++ b/awx/main/tests/live/pytest_django_config.py
@@ -0,0 +1,12 @@
+import django
+
+from awx import prepare_env
+
+
+def pytest_load_initial_conftests(args):
+    """Replacement for same-named method in pytest_django plugin
+
+    Instead of setting up a test database, this just sets up Django normally
+    this will give access to the postgres database as-is, for better and worse"""
+    prepare_env()
+    django.setup()
diff --git a/awx/main/tests/live/tests/conftest.py b/awx/main/tests/live/tests/conftest.py
new file mode 100644
index 0000000000..848e8fd7d2
--- /dev/null
+++ b/awx/main/tests/live/tests/conftest.py
@@ -0,0 +1,28 @@
+import time
+
+# These tests are invoked from the awx/main/tests/live/ subfolder
+# so any fixtures from higher-up conftest files must be explicitly included
+from awx.main.tests.functional.conftest import *  # noqa
+
+
+def wait_to_leave_status(job, status, timeout=25, sleep_time=0.1):
+    """Wait until the job does NOT have the specified status with some timeout
+
+    the default timeout of 25 if chosen because the task manager runs on a 20 second
+    schedule, and the API does not guarentee working jobs faster than this
+    """
+    start = time.time()
+    while time.time() - start < timeout:
+        job.refresh_from_db()
+        if job.status != status:
+            return
+        time.sleep(sleep_time)
+    raise RuntimeError(f'Job failed to exit {status} in {timeout} seconds. job_explanation={job.job_explanation} tb={job.result_traceback}')
+
+
+def wait_for_job(job, final_status='successful', running_timeout=800):
+    wait_to_leave_status(job, 'pending')
+    wait_to_leave_status(job, 'waiting')
+    wait_to_leave_status(job, 'running', timeout=running_timeout)
+
+    assert job.status == final_status, f'Job was not successful id={job.id} status={job.status}'
diff --git a/awx/main/tests/live/tests/test_demo_data.py b/awx/main/tests/live/tests/test_demo_data.py
new file mode 100644
index 0000000000..fa9ee5eb97
--- /dev/null
+++ b/awx/main/tests/live/tests/test_demo_data.py
@@ -0,0 +1,15 @@
+from awx.api.versioning import reverse
+
+from awx.main.models import JobTemplate, Job
+
+from awx.main.tests.live.tests.conftest import wait_for_job
+
+
+def test_launch_demo_jt(post, admin):
+    jt = JobTemplate.objects.get(name='Demo Job Template')
+
+    url = reverse('api:job_template_launch', kwargs={'pk': jt.id})
+
+    r = post(url=url, data={}, user=admin, expect=201)
+    job = Job.objects.get(pk=r.data['id'])
+    wait_for_job(job)
diff --git a/tools/docker-compose/ansible/smoke-test.yml b/tools/docker-compose/ansible/smoke-test.yml
deleted file mode 100644
index 591ceed396..0000000000
--- a/tools/docker-compose/ansible/smoke-test.yml
+++ /dev/null
@@ -1,60 +0,0 @@
----
-#
-# This is used by a CI check in GitHub Actions and isnt really
-# meant to be run locally.
-#
-# The development environment does some unfortunate things to
-# make rootless podman work inside of a docker container.
-# The goal here is to essentially tests that the awx user is
-# able to run `podman run`.
-#
-- name: Test that the development environment is able to launch a job
-  hosts: localhost
-  tasks:
-    - name: Reset admin password
-      shell: |
-        docker exec -i tools_awx_1 bash <<EOSH
-          awx-manage update_password --username=admin --password=password
-          awx-manage create_preload_data
-        EOSH
-
-    - block:
-        - name: Launch Demo Job Template
-          awx.awx.job_launch:
-            name: Demo Job Template
-            wait: yes
-            validate_certs: no
-            controller_host: "http://localhost:8013"
-            controller_username: "admin"
-            controller_password: "password"
-      rescue:
-        - name: Get list of project updates and jobs
-          uri:
-            url: "http://localhost:8013/api/v2/{{ resource }}/"
-            user: admin
-            password: "password"
-            force_basic_auth: yes
-          register: job_lists
-          loop:
-            - project_updates
-            - jobs
-          loop_control:
-            loop_var: resource
-
-        - name: Get all job and project details
-          uri:
-            url: "http://localhost:8013{{ endpoint }}"
-            user: admin
-            password: "password"
-            force_basic_auth: yes
-          loop: |
-            {{ job_lists.results | map(attribute='json') | map(attribute='results') | flatten | map(attribute='url') }}
-          loop_control:
-            loop_var: endpoint
-
-        - name: Re-emit failure
-          vars:
-            failed_task:
-              result: '{{ ansible_failed_result }}'
-          fail:
-            msg: '{{ failed_task }}'