AAP-36536 Send job_lifecycle logs to external loggers (#15701)

* Send job_lifecycle logs to external loggers

* Include structured data in message

* Attach the organization_id of the job

5 files changed, 11 insertions, 4 deletions

diff --git a/awx/main/conf.py b/awx/main/conf.py
index 3f601b6d09..c710913d36 100644
--- a/awx/main/conf.py
+++ b/awx/main/conf.py
@@ -592,7 +592,7 @@ register(
 register(
     'LOG_AGGREGATOR_LOGGERS',
     field_class=fields.StringListField,
-    default=['awx', 'activity_stream', 'job_events', 'system_tracking', 'broadcast_websocket'],
+    default=['awx', 'activity_stream', 'job_events', 'system_tracking', 'broadcast_websocket', 'job_lifecycle'],
     label=_('Loggers Sending Data to Log Aggregator Form'),
     help_text=_(
         'List of loggers that will send HTTP logs to the collector, these can '
@@ -602,6 +602,7 @@ register(
         'job_events - callback data from Ansible job events\n'
         'system_tracking - facts gathered from scan jobs\n'
         'broadcast_websocket - errors pertaining to websockets broadcast metrics\n'
+        'job_lifecycle - logs related to processing of a job\n'
     ),
     category=_('Logging'),
     category_slug='logging',
diff --git a/awx/main/models/unified_jobs.py b/awx/main/models/unified_jobs.py
index 1cced0a5b4..da9bca0df2 100644
--- a/awx/main/models/unified_jobs.py
+++ b/awx/main/models/unified_jobs.py
@@ -1625,7 +1625,7 @@ class UnifiedJob(
         elif state == "execution_node_chosen":
             extra["execution_node"] = self.execution_node or "NOT_SET"
 
-        logger_job_lifecycle.info(f"{msg} {json.dumps(extra)}")
+        logger_job_lifecycle.info(f"{msg} {json.dumps(extra)}", extra={'lifecycle_data': extra, 'organization_id': self.organization_id})
 
     @property
     def launched_by(self):
diff --git a/awx/main/utils/formatters.py b/awx/main/utils/formatters.py
index 5b6b5d785d..5cf5e17a29 100644
--- a/awx/main/utils/formatters.py
+++ b/awx/main/utils/formatters.py
@@ -160,6 +160,11 @@ class LogstashFormatter(LogstashFormatterBase):
             data = json.loads(data)
         data_for_log = {}
 
+        # For the job_lifecycle logger, copy some raw data fields directly
+        for key in ('lifecycle_data', 'organization_id'):
+            if key in raw_data:
+                data_for_log[key] = raw_data[key]
+
         if kind == 'job_events' and raw_data.get('python_objects', {}).get('job_event'):
             job_event = raw_data['python_objects']['job_event']
             guid = job_event.event_data.pop('guid', None)
diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py
index 1406d70e26..ef488a831c 100644
--- a/awx/settings/defaults.py
+++ b/awx/settings/defaults.py
@@ -783,7 +783,7 @@ LOGGING = {
         'awx.analytics': {'handlers': ['external_logger'], 'level': 'INFO', 'propagate': False},
         'awx.analytics.broadcast_websocket': {'handlers': ['console', 'file', 'wsrelay', 'external_logger'], 'level': 'INFO', 'propagate': False},
         'awx.analytics.performance': {'handlers': ['console', 'file', 'tower_warnings', 'external_logger'], 'level': 'DEBUG', 'propagate': False},
-        'awx.analytics.job_lifecycle': {'handlers': ['console', 'job_lifecycle'], 'level': 'DEBUG', 'propagate': False},
+        'awx.analytics.job_lifecycle': {'handlers': ['console', 'job_lifecycle', 'external_logger'], 'level': 'DEBUG', 'propagate': False},
         'social': {'handlers': ['console', 'file', 'tower_warnings'], 'level': 'DEBUG'},
         'system_tracking_migrations': {'handlers': ['console', 'file', 'tower_warnings'], 'level': 'DEBUG'},
         'rbac_migrations': {'handlers': ['console', 'file', 'tower_warnings'], 'level': 'DEBUG'},
diff --git a/tools/docker-compose/docs/logstash.md b/tools/docker-compose/docs/logstash.md
index f7f280875a..a659dff038 100644
--- a/tools/docker-compose/docs/logstash.md
+++ b/tools/docker-compose/docs/logstash.md
@@ -22,7 +22,8 @@ authentication set up inside of the logstash configuration file).
         "awx",
         "activity_stream",
         "job_events",
-        "system_tracking"
+        "system_tracking",
+        "job_lifecycle"
     ],
     "LOG_AGGREGATOR_INDIVIDUAL_FACTS": false,
     "LOG_AGGREGATOR_TOWER_UUID": "991ac7e9-6d68-48c8-bbde-7ca1096653c6",