feat: add limited execution tracing support

- For every process that is spawned (every new non-trivial goroutine
such as http requests, queues or tasks) start a [execution
tracer](https://pkg.go.dev/runtime/trace). This allows very precise
diagnosis of how each individual process over a time period.
- It's safe and [fast](https://go.dev/blog/execution-traces-2024#low-overhead-tracing) to
be run in production, hence no setting to disable this. There's only
noticable overhead when tracing is actually performed and not continuous.
- Proper tracing support would mean the codebase would be full of
`trace.WithRegion` and `trace.Log`, which feels premature for this patch
as there's no real-world usage yet to indicate which places would need
this the most. So far only Git commands and SQL queries receive somewhat
proper tracing support given that these are used throughout the codebase.
- Make git commands a new process type.
- Add tracing to diagnosis zip file.

3 files changed, 27 insertions, 5 deletions

diff --git a/modules/git/command.go b/modules/git/command.go
index a3d43aaec6..605816b7a2 100644
--- a/modules/git/command.go
+++ b/modules/git/command.go
@@ -13,6 +13,7 @@ import (
 	"os"
 	"os/exec"
 	"runtime"
+	"runtime/trace"
 	"strings"
 	"time"
 
@@ -317,12 +318,13 @@ func (c *Command) Run(opts *RunOpts) error {
 	var finished context.CancelFunc
 
 	if opts.UseContextTimeout {
-		ctx, cancel, finished = process.GetManager().AddContext(c.parentContext, desc)
+		ctx, cancel, finished = process.GetManager().AddTypedContext(c.parentContext, desc, process.GitProcessType, true)
 	} else {
-		ctx, cancel, finished = process.GetManager().AddContextTimeout(c.parentContext, timeout, desc)
+		ctx, cancel, finished = process.GetManager().AddTypedContextTimeout(c.parentContext, timeout, desc, process.GitProcessType, true)
 	}
 	defer finished()
 
+	trace.Log(ctx, "command", desc)
 	startTime := time.Now()
 
 	cmd := exec.CommandContext(ctx, c.prog, c.args...)
diff --git a/modules/process/manager.go b/modules/process/manager.go
index 37098ad92f..062ee1482f 100644
--- a/modules/process/manager.go
+++ b/modules/process/manager.go
@@ -7,6 +7,7 @@ package process
 import (
 	"context"
 	"runtime/pprof"
+	"runtime/trace"
 	"strconv"
 	"sync"
 	"sync/atomic"
@@ -126,7 +127,7 @@ func (pm *Manager) AddTypedContext(parent context.Context, description, processT
 	return ctx, cancel, finished
 }
 
-// AddContextTimeout creates a new context and add it as a process. Once the process is finished, finished must be called
+// AddTypedContextTimeout creates a new context and adds it as a process. Once the process is finished, finished must be called
 // to remove the process from the process table. It should not be called until the process is finished but must always be called.
 //
 // cancel should be used to cancel the returned context, however it will not remove the process from the process table.
@@ -134,7 +135,7 @@ func (pm *Manager) AddTypedContext(parent context.Context, description, processT
 //
 // Most processes will not need to use the cancel function but there will be cases whereby you want to cancel the process but not immediately remove it from the
 // process table.
-func (pm *Manager) AddContextTimeout(parent context.Context, timeout time.Duration, description string) (ctx context.Context, cancel context.CancelFunc, finished FinishedFunc) {
+func (pm *Manager) AddTypedContextTimeout(parent context.Context, timeout time.Duration, description, processType string, currentlyRunning bool) (ctx context.Context, cancel context.CancelFunc, finished FinishedFunc) {
 	if timeout <= 0 {
 		// it's meaningless to use timeout <= 0, and it must be a bug! so we must panic here to tell developers to make the timeout correct
 		panic("the timeout must be greater than zero, otherwise the context will be cancelled immediately")
@@ -142,11 +143,23 @@ func (pm *Manager) AddContextTimeout(parent context.Context, timeout time.Durati
 
 	ctx, cancel = context.WithTimeout(parent, timeout)
 
-	ctx, _, finished = pm.Add(ctx, description, cancel, NormalProcessType, true)
+	ctx, _, finished = pm.Add(ctx, description, cancel, processType, currentlyRunning)
 
 	return ctx, cancel, finished
 }
 
+// AddContextTimeout creates a new context and add it as a process. Once the process is finished, finished must be called
+// to remove the process from the process table. It should not be called until the process is finished but must always be called.
+//
+// cancel should be used to cancel the returned context, however it will not remove the process from the process table.
+// finished will cancel the returned context and remove it from the process table.
+//
+// Most processes will not need to use the cancel function but there will be cases whereby you want to cancel the process but not immediately remove it from the
+// process table.
+func (pm *Manager) AddContextTimeout(parent context.Context, timeout time.Duration, description string) (ctx context.Context, cancel context.CancelFunc, finished FinishedFunc) {
+	return pm.AddTypedContextTimeout(parent, timeout, description, NormalProcessType, true)
+}
+
 // Add create a new process
 func (pm *Manager) Add(ctx context.Context, description string, cancel context.CancelFunc, processType string, currentlyRunning bool) (context.Context, IDType, FinishedFunc) {
 	parentPID := GetParentPID(ctx)
@@ -159,6 +172,8 @@ func (pm *Manager) Add(ctx context.Context, description string, cancel context.C
 		parentPID = ""
 	}
 
+	ctx, traceTask := trace.NewTask(ctx, processType)
+
 	process := &process{
 		PID:         pid,
 		ParentPID:   parentPID,
@@ -166,6 +181,7 @@ func (pm *Manager) Add(ctx context.Context, description string, cancel context.C
 		Start:       start,
 		Cancel:      cancel,
 		Type:        processType,
+		TraceTrask:  traceTask,
 	}
 
 	var finished FinishedFunc
@@ -218,6 +234,7 @@ func (pm *Manager) nextPID() (start time.Time, pid IDType) {
 }
 
 func (pm *Manager) remove(process *process) {
+	process.TraceTrask.End()
 	deleted := false
 
 	pm.mutex.Lock()
diff --git a/modules/process/process.go b/modules/process/process.go
index 06a28c4a60..8947eb252f 100644
--- a/modules/process/process.go
+++ b/modules/process/process.go
@@ -5,12 +5,14 @@ package process
 
 import (
 	"context"
+	"runtime/trace"
 	"time"
 )
 
 var (
 	SystemProcessType  = "system"
 	RequestProcessType = "request"
+	GitProcessType     = "git"
 	NormalProcessType  = "normal"
 	NoneProcessType    = "none"
 )
@@ -23,6 +25,7 @@ type process struct {
 	Start       time.Time
 	Cancel      context.CancelFunc
 	Type        string
+	TraceTrask  *trace.Task
 }
 
 // ToProcess converts a process to a externally usable Process