Open telemetry integration (#3972)

This PR adds opentelemetry and chi wrapper to have basic instrumentation

<!--start release-notes-assistant-->

## Draft release notes
<!--URL:https://codeberg.org/forgejo/forgejo-->
- Features
  - [PR](https://codeberg.org/forgejo/forgejo/pulls/3972): <!--number 3972 --><!--line 0 --><!--description YWRkIHN1cHBvcnQgZm9yIGJhc2ljIHJlcXVlc3QgdHJhY2luZyB3aXRoIG9wZW50ZWxlbWV0cnk=-->add support for basic request tracing with opentelemetry<!--description-->
<!--end release-notes-assistant-->

Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/3972
Reviewed-by: Earl Warren <earl-warren@noreply.codeberg.org>
Co-authored-by: TheFox0x7 <thefox0x7@gmail.com>
Co-committed-by: TheFox0x7 <thefox0x7@gmail.com>

9 files changed, 1031 insertions, 0 deletions

diff --git a/modules/opentelemetry/otel.go b/modules/opentelemetry/otel.go
new file mode 100644
index 0000000000..963b696a54
--- /dev/null
+++ b/modules/opentelemetry/otel.go
@@ -0,0 +1,96 @@
+// Copyright 2024 TheFox0x7. All rights reserved.
+// SPDX-License-Identifier: EUPL-1.2
+
+package opentelemetry
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"os"
+
+	"code.gitea.io/gitea/modules/graceful"
+	"code.gitea.io/gitea/modules/log"
+
+	"github.com/go-logr/logr/funcr"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/propagation"
+)
+
+func Init(ctx context.Context) error {
+	// Redirect otel logger to write to common forgejo log at info
+	logWrap := funcr.New(func(prefix, args string) {
+		log.Info(fmt.Sprint(prefix, args))
+	}, funcr.Options{})
+	otel.SetLogger(logWrap)
+	// Redirect error handling to forgejo log as well
+	otel.SetErrorHandler(otel.ErrorHandlerFunc(func(cause error) {
+		log.Error("internal opentelemetry error was raised: %s", cause)
+	}))
+	var shutdownFuncs []func(context.Context) error
+	shutdownCtx := context.Background()
+
+	otel.SetTextMapPropagator(newPropagator())
+
+	res, err := newResource(ctx)
+	if err != nil {
+		return err
+	}
+
+	traceShutdown, err := setupTraceProvider(ctx, res)
+	if err != nil {
+		log.Warn("OpenTelemetry trace setup failed, err=%s", err)
+	} else {
+		shutdownFuncs = append(shutdownFuncs, traceShutdown)
+	}
+
+	graceful.GetManager().RunAtShutdown(ctx, func() {
+		for _, fn := range shutdownFuncs {
+			if err := fn(shutdownCtx); err != nil {
+				log.Warn("exporter shutdown failed, err=%s", err)
+			}
+		}
+		shutdownFuncs = nil
+	})
+
+	return nil
+}
+
+func newPropagator() propagation.TextMapPropagator {
+	return propagation.NewCompositeTextMapPropagator(
+		propagation.TraceContext{},
+		propagation.Baggage{},
+	)
+}
+
+func withCertPool(path string, tlsConf *tls.Config) {
+	if path == "" {
+		return
+	}
+	b, err := os.ReadFile(path)
+	if err != nil {
+		log.Warn("Otel: reading ca cert failed path=%s, err=%s", path, err)
+		return
+	}
+	cp := x509.NewCertPool()
+	if ok := cp.AppendCertsFromPEM(b); !ok {
+		log.Warn("Otel: no valid PEM certificate found path=%s", path)
+		return
+	}
+	tlsConf.RootCAs = cp
+}
+
+func withClientCert(nc, nk string, tlsConf *tls.Config) {
+	if nc == "" || nk == "" {
+		return
+	}
+
+	crt, err := tls.LoadX509KeyPair(nc, nk)
+	if err != nil {
+		log.Warn("Otel: create tls client key pair failed")
+		return
+	}
+
+	tlsConf.Certificates = append(tlsConf.Certificates, crt)
+}
diff --git a/modules/opentelemetry/otel_test.go b/modules/opentelemetry/otel_test.go
new file mode 100644
index 0000000000..d40146f9cb
--- /dev/null
+++ b/modules/opentelemetry/otel_test.go
@@ -0,0 +1,121 @@
+// Copyright 2024 TheFox0x7. All rights reserved.
+// SPDX-License-Identifier: EUPL-1.2
+
+package opentelemetry
+
+import (
+	"context"
+	"crypto/ed25519"
+	"crypto/rand"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"math/big"
+	"net"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/test"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	"go.opentelemetry.io/otel/sdk/trace/tracetest"
+)
+
+func TestNoopDefault(t *testing.T) {
+	inMem := tracetest.NewInMemoryExporter()
+	called := false
+	exp := func(ctx context.Context) (sdktrace.SpanExporter, error) {
+		called = true
+		return inMem, nil
+	}
+	exporter["inmemory"] = exp
+	t.Cleanup(func() {
+		delete(exporter, "inmemory")
+	})
+	defer test.MockVariableValue(&setting.OpenTelemetry.Traces, "inmemory")
+
+	ctx := context.Background()
+	require.NoError(t, Init(ctx))
+	tracer := otel.Tracer("test_noop")
+
+	_, span := tracer.Start(ctx, "test span")
+	defer span.End()
+
+	assert.False(t, span.SpanContext().HasTraceID())
+	assert.False(t, span.SpanContext().HasSpanID())
+	assert.False(t, called)
+}
+
+func generateTestTLS(t *testing.T, path, host string) *tls.Config {
+	_, priv, err := ed25519.GenerateKey(rand.Reader)
+	require.NoError(t, err, "Failed to generate private key: %v", err)
+
+	keyUsage := x509.KeyUsageDigitalSignature
+
+	notBefore := time.Now()
+	notAfter := notBefore.Add(time.Hour)
+
+	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
+	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
+	require.NoError(t, err, "Failed to generate serial number: %v", err)
+
+	template := x509.Certificate{
+		SerialNumber: serialNumber,
+		Subject: pkix.Name{
+			Organization: []string{"Forgejo Testing"},
+		},
+		NotBefore: notBefore,
+		NotAfter:  notAfter,
+
+		KeyUsage:              keyUsage,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
+		BasicConstraintsValid: true,
+	}
+
+	hosts := strings.Split(host, ",")
+	for _, h := range hosts {
+		if ip := net.ParseIP(h); ip != nil {
+			template.IPAddresses = append(template.IPAddresses, ip)
+		} else {
+			template.DNSNames = append(template.DNSNames, h)
+		}
+	}
+
+	derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, priv.Public(), priv)
+	require.NoError(t, err, "Failed to create certificate: %v", err)
+
+	certOut, err := os.Create(path + "/cert.pem")
+	require.NoError(t, err, "Failed to open cert.pem for writing: %v", err)
+
+	if err := pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil {
+		t.Fatalf("Failed to write data to cert.pem: %v", err)
+	}
+	if err := certOut.Close(); err != nil {
+		t.Fatalf("Error closing cert.pem: %v", err)
+	}
+	keyOut, err := os.OpenFile(path+"/key.pem", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600)
+	require.NoError(t, err, "Failed to open key.pem for writing: %v", err)
+
+	privBytes, err := x509.MarshalPKCS8PrivateKey(priv)
+	require.NoError(t, err, "Unable to marshal private key: %v", err)
+
+	if err := pem.Encode(keyOut, &pem.Block{Type: "PRIVATE KEY", Bytes: privBytes}); err != nil {
+		t.Fatalf("Failed to write data to key.pem: %v", err)
+	}
+	if err := keyOut.Close(); err != nil {
+		t.Fatalf("Error closing key.pem: %v", err)
+	}
+	serverCert, err := tls.LoadX509KeyPair(path+"/cert.pem", path+"/key.pem")
+	require.NoError(t, err, "failed to load the key pair")
+	return &tls.Config{
+		Certificates: []tls.Certificate{serverCert},
+		ClientAuth:   tls.RequireAnyClientCert,
+	}
+}
diff --git a/modules/opentelemetry/resource.go b/modules/opentelemetry/resource.go
new file mode 100644
index 0000000000..419c98a074
--- /dev/null
+++ b/modules/opentelemetry/resource.go
@@ -0,0 +1,90 @@
+// Copyright 2024 TheFox0x7. All rights reserved.
+// SPDX-License-Identifier: EUPL-1.2
+
+package opentelemetry
+
+import (
+	"context"
+	"net/url"
+	"strings"
+
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/sdk/resource"
+	semconv "go.opentelemetry.io/otel/semconv/v1.25.0"
+)
+
+const (
+	decoderTelemetrySdk = "sdk"
+	decoderProcess      = "process"
+	decoderOS           = "os"
+	decoderHost         = "host"
+)
+
+func newResource(ctx context.Context) (*resource.Resource, error) {
+	opts := []resource.Option{
+		resource.WithAttributes(parseSettingAttributes(setting.OpenTelemetry.ResourceAttributes)...),
+	}
+	opts = append(opts, parseDecoderOpts()...)
+	opts = append(opts, resource.WithAttributes(
+		semconv.ServiceName(setting.OpenTelemetry.ServiceName),
+		semconv.ServiceVersion(setting.ForgejoVersion),
+	))
+	return resource.New(ctx, opts...)
+}
+
+func parseDecoderOpts() []resource.Option {
+	var opts []resource.Option
+	for _, v := range strings.Split(setting.OpenTelemetry.ResourceDetectors, ",") {
+		switch v {
+		case decoderTelemetrySdk:
+			opts = append(opts, resource.WithTelemetrySDK())
+		case decoderProcess:
+			opts = append(opts, resource.WithProcess())
+		case decoderOS:
+			opts = append(opts, resource.WithOS())
+		case decoderHost:
+			opts = append(opts, resource.WithHost())
+		case "": // Don't warn on empty string
+		default:
+			log.Warn("Ignoring unknown resource decoder option: %s", v)
+		}
+	}
+	return opts
+}
+
+func parseSettingAttributes(s string) []attribute.KeyValue {
+	var attrs []attribute.KeyValue
+	rawAttrs := strings.TrimSpace(s)
+
+	if rawAttrs == "" {
+		return attrs
+	}
+
+	pairs := strings.Split(rawAttrs, ",")
+
+	var invalid []string
+	for _, p := range pairs {
+		k, v, found := strings.Cut(p, "=")
+		if !found {
+			invalid = append(invalid, p)
+			continue
+		}
+		key := strings.TrimSpace(k)
+		val, err := url.PathUnescape(strings.TrimSpace(v))
+		if err != nil {
+			// Retain original value if decoding fails, otherwise it will be
+			// an empty string.
+			val = v
+			log.Warn("Otel resource attribute decoding error, retaining unescaped value. key=%s, val=%s", key, val)
+		}
+		attrs = append(attrs, attribute.String(key, val))
+	}
+	if len(invalid) > 0 {
+		log.Warn("Partial resource, missing values: %v", invalid)
+	}
+
+	return attrs
+}
diff --git a/modules/opentelemetry/resource_test.go b/modules/opentelemetry/resource_test.go
new file mode 100644
index 0000000000..9a1733bac1
--- /dev/null
+++ b/modules/opentelemetry/resource_test.go
@@ -0,0 +1,73 @@
+// Copyright 2024 TheFox0x7. All rights reserved.
+// SPDX-License-Identifier: EUPL-1.2
+
+package opentelemetry
+
+import (
+	"context"
+	"slices"
+	"testing"
+
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/test"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/sdk/resource"
+	semconv "go.opentelemetry.io/otel/semconv/v1.25.0"
+)
+
+func TestResourceServiceName(t *testing.T) {
+	ctx := context.Background()
+
+	resource, err := newResource(ctx)
+	require.NoError(t, err)
+	serviceKeyIdx := slices.IndexFunc(resource.Attributes(), func(v attribute.KeyValue) bool {
+		return v.Key == semconv.ServiceNameKey
+	})
+	require.NotEqual(t, -1, serviceKeyIdx)
+
+	assert.Equal(t, "forgejo", resource.Attributes()[serviceKeyIdx].Value.AsString())
+
+	defer test.MockVariableValue(&setting.OpenTelemetry.ServiceName, "non-default value")()
+	resource, err = newResource(ctx)
+	require.NoError(t, err)
+
+	serviceKeyIdx = slices.IndexFunc(resource.Attributes(), func(v attribute.KeyValue) bool {
+		return v.Key == semconv.ServiceNameKey
+	})
+	require.NotEqual(t, -1, serviceKeyIdx)
+
+	assert.Equal(t, "non-default value", resource.Attributes()[serviceKeyIdx].Value.AsString())
+}
+
+func TestResourceAttributes(t *testing.T) {
+	ctx := context.Background()
+	defer test.MockVariableValue(&setting.OpenTelemetry.ResourceDetectors, "foo")()
+	defer test.MockVariableValue(&setting.OpenTelemetry.ResourceAttributes, "Test=LABEL,broken,unescape=%XXlabel")()
+	res, err := newResource(ctx)
+	require.NoError(t, err)
+	expected, err := resource.New(ctx, resource.WithAttributes(
+		semconv.ServiceName(setting.OpenTelemetry.ServiceName),
+		semconv.ServiceVersion(setting.ForgejoVersion),
+		attribute.String("Test", "LABEL"),
+		attribute.String("unescape", "%XXlabel"),
+	))
+	require.NoError(t, err)
+	assert.Equal(t, expected, res)
+}
+
+func TestDecoderParity(t *testing.T) {
+	ctx := context.Background()
+	defer test.MockVariableValue(&setting.OpenTelemetry.ResourceDetectors, "sdk,process,os,host")()
+	exp, err := resource.New(
+		ctx, resource.WithTelemetrySDK(), resource.WithOS(), resource.WithProcess(), resource.WithHost(), resource.WithAttributes(
+			semconv.ServiceName(setting.OpenTelemetry.ServiceName), semconv.ServiceVersion(setting.ForgejoVersion),
+		),
+	)
+	require.NoError(t, err)
+	res2, err := newResource(ctx)
+	require.NoError(t, err)
+	assert.Equal(t, exp, res2)
+}
diff --git a/modules/opentelemetry/traces.go b/modules/opentelemetry/traces.go
new file mode 100644
index 0000000000..30d9436392
--- /dev/null
+++ b/modules/opentelemetry/traces.go
@@ -0,0 +1,98 @@
+// Copyright 2024 TheFox0x7. All rights reserved.
+// SPDX-License-Identifier: EUPL-1.2
+
+package opentelemetry
+
+import (
+	"context"
+	"crypto/tls"
+
+	"code.gitea.io/gitea/modules/setting"
+
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
+	"go.opentelemetry.io/otel/sdk/resource"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	"google.golang.org/grpc/credentials"
+)
+
+func newGrpcExporter(ctx context.Context) (sdktrace.SpanExporter, error) {
+	endpoint := setting.OpenTelemetry.OtelTraces.Endpoint
+
+	opts := []otlptracegrpc.Option{}
+
+	tlsConf := &tls.Config{}
+	opts = append(opts, otlptracegrpc.WithEndpoint(endpoint.Host))
+	opts = append(opts, otlptracegrpc.WithTimeout(setting.OpenTelemetry.OtelTraces.Timeout))
+	switch setting.OpenTelemetry.OtelTraces.Endpoint.Scheme {
+	case "http", "unix":
+		opts = append(opts, otlptracegrpc.WithInsecure())
+	}
+
+	if setting.OpenTelemetry.OtelTraces.Compression != "" {
+		opts = append(opts, otlptracegrpc.WithCompressor(setting.OpenTelemetry.OtelTraces.Compression))
+	}
+	withCertPool(setting.OpenTelemetry.OtelTraces.Certificate, tlsConf)
+	withClientCert(setting.OpenTelemetry.OtelTraces.ClientCertificate, setting.OpenTelemetry.OtelTraces.ClientKey, tlsConf)
+	if tlsConf.RootCAs != nil || len(tlsConf.Certificates) > 0 {
+		opts = append(opts, otlptracegrpc.WithTLSCredentials(
+			credentials.NewTLS(tlsConf),
+		))
+	}
+	opts = append(opts, otlptracegrpc.WithHeaders(setting.OpenTelemetry.OtelTraces.Headers))
+
+	return otlptracegrpc.New(ctx, opts...)
+}
+
+func newHTTPExporter(ctx context.Context) (sdktrace.SpanExporter, error) {
+	endpoint := setting.OpenTelemetry.OtelTraces.Endpoint
+	opts := []otlptracehttp.Option{}
+	tlsConf := &tls.Config{}
+	opts = append(opts, otlptracehttp.WithEndpoint(endpoint.Host))
+	switch setting.OpenTelemetry.OtelTraces.Endpoint.Scheme {
+	case "http", "unix":
+		opts = append(opts, otlptracehttp.WithInsecure())
+	}
+	switch setting.OpenTelemetry.OtelTraces.Compression {
+	case "gzip":
+		opts = append(opts, otlptracehttp.WithCompression(otlptracehttp.GzipCompression))
+	default:
+		opts = append(opts, otlptracehttp.WithCompression(otlptracehttp.NoCompression))
+	}
+	withCertPool(setting.OpenTelemetry.OtelTraces.Certificate, tlsConf)
+	withClientCert(setting.OpenTelemetry.OtelTraces.ClientCertificate, setting.OpenTelemetry.OtelTraces.ClientKey, tlsConf)
+	if tlsConf.RootCAs != nil || len(tlsConf.Certificates) > 0 {
+		opts = append(opts, otlptracehttp.WithTLSClientConfig(tlsConf))
+	}
+	opts = append(opts, otlptracehttp.WithHeaders(setting.OpenTelemetry.OtelTraces.Headers))
+
+	return otlptracehttp.New(ctx, opts...)
+}
+
+var exporter = map[string]func(context.Context) (sdktrace.SpanExporter, error){
+	"http/protobuf": newHTTPExporter,
+	"grpc":          newGrpcExporter,
+}
+
+// Create new and register trace provider from user defined configuration
+func setupTraceProvider(ctx context.Context, r *resource.Resource) (func(context.Context) error, error) {
+	var shutdown func(context.Context) error
+	switch setting.OpenTelemetry.Traces {
+	case "otlp":
+		traceExporter, err := exporter[setting.OpenTelemetry.OtelTraces.Protocol](ctx)
+		if err != nil {
+			return nil, err
+		}
+		traceProvider := sdktrace.NewTracerProvider(
+			sdktrace.WithSampler(setting.OpenTelemetry.Sampler),
+			sdktrace.WithBatcher(traceExporter),
+			sdktrace.WithResource(r),
+		)
+		otel.SetTracerProvider(traceProvider)
+		shutdown = traceProvider.Shutdown
+	default:
+		shutdown = func(ctx context.Context) error { return nil }
+	}
+	return shutdown, nil
+}
diff --git a/modules/opentelemetry/traces_test.go b/modules/opentelemetry/traces_test.go
new file mode 100644
index 0000000000..dcc3c57394
--- /dev/null
+++ b/modules/opentelemetry/traces_test.go
@@ -0,0 +1,114 @@
+// Copyright 2024 TheFox0x7. All rights reserved.
+// SPDX-License-Identifier: EUPL-1.2
+
+package opentelemetry
+
+import (
+	"context"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"os"
+	"testing"
+	"time"
+
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/test"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials"
+)
+
+func TestTraceGrpcExporter(t *testing.T) {
+	grpcMethods := make(chan string)
+	tlsConfig := generateTestTLS(t, os.TempDir(), "localhost,127.0.0.1")
+	assert.NotNil(t, tlsConfig)
+
+	collector := grpc.NewServer(grpc.Creds(credentials.NewTLS(tlsConfig)), grpc.UnknownServiceHandler(func(srv any, stream grpc.ServerStream) error {
+		method, _ := grpc.Method(stream.Context())
+		grpcMethods <- method
+		return nil
+	}))
+	defer collector.GracefulStop()
+	ln, err := net.Listen("tcp", "localhost:0")
+	require.NoError(t, err)
+	defer ln.Close()
+	go collector.Serve(ln)
+
+	traceEndpoint, err := url.Parse("https://" + ln.Addr().String())
+	require.NoError(t, err)
+	config := &setting.OtelExporter{
+		Endpoint:          traceEndpoint,
+		Certificate:       os.TempDir() + "/cert.pem",
+		ClientCertificate: os.TempDir() + "/cert.pem",
+		ClientKey:         os.TempDir() + "/key.pem",
+		Protocol:          "grpc",
+	}
+
+	defer test.MockVariableValue(&setting.OpenTelemetry.ServiceName, "forgejo-certs")()
+	defer test.MockVariableValue(&setting.OpenTelemetry.Enabled, true)()
+	defer test.MockVariableValue(&setting.OpenTelemetry.Traces, "otlp")()
+	defer test.MockVariableValue(&setting.OpenTelemetry.OtelTraces, config)()
+	ctx := context.Background()
+	require.NoError(t, Init(ctx))
+
+	tracer := otel.Tracer("test_tls")
+	_, span := tracer.Start(ctx, "test span")
+	assert.True(t, span.SpanContext().HasTraceID())
+	assert.True(t, span.SpanContext().HasSpanID())
+
+	span.End()
+	// Give the exporter time to send the span
+	select {
+	case method := <-grpcMethods:
+		assert.Equal(t, "/opentelemetry.proto.collector.trace.v1.TraceService/Export", method)
+	case <-time.After(10 * time.Second):
+		t.Fatal("no grpc call within 10s")
+	}
+}
+
+func TestTraceHttpExporter(t *testing.T) {
+	httpCalls := make(chan string)
+	tlsConfig := generateTestTLS(t, os.TempDir(), "localhost,127.0.0.1")
+	assert.NotNil(t, tlsConfig)
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		httpCalls <- r.URL.Path
+		w.WriteHeader(http.StatusOK)
+		w.Write([]byte(`{"success": true}`))
+	}))
+	server.TLS = tlsConfig
+
+	traceEndpoint, err := url.Parse("http://" + server.Listener.Addr().String())
+	require.NoError(t, err)
+	config := &setting.OtelExporter{
+		Endpoint:          traceEndpoint,
+		Certificate:       os.TempDir() + "/cert.pem",
+		ClientCertificate: os.TempDir() + "/cert.pem",
+		ClientKey:         os.TempDir() + "/key.pem",
+		Protocol:          "http/protobuf",
+	}
+
+	defer test.MockVariableValue(&setting.OpenTelemetry.ServiceName, "forgejo-certs")()
+	defer test.MockVariableValue(&setting.OpenTelemetry.Enabled, true)()
+	defer test.MockVariableValue(&setting.OpenTelemetry.Traces, "otlp")()
+	defer test.MockVariableValue(&setting.OpenTelemetry.OtelTraces, config)()
+	ctx := context.Background()
+	require.NoError(t, Init(ctx))
+
+	tracer := otel.Tracer("test_tls")
+	_, span := tracer.Start(ctx, "test span")
+	assert.True(t, span.SpanContext().HasTraceID())
+	assert.True(t, span.SpanContext().HasSpanID())
+
+	span.End()
+	select {
+	case path := <-httpCalls:
+		assert.Equal(t, "/v1/traces", path)
+	case <-time.After(10 * time.Second):
+		t.Fatal("no http call within 10s")
+	}
+}
diff --git a/modules/setting/opentelemetry.go b/modules/setting/opentelemetry.go
new file mode 100644
index 0000000000..810cb58f5f
--- /dev/null
+++ b/modules/setting/opentelemetry.go
@@ -0,0 +1,199 @@
+// Copyright 2024 TheFox0x7. All rights reserved.
+// SPDX-License-Identifier: EUPL-1.2
+
+package setting
+
+import (
+	"net/url"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"code.gitea.io/gitea/modules/log"
+
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+)
+
+const (
+	opentelemetrySectionName string = "opentelemetry"
+	exporter                 string = ".exporter"
+	otlp                     string = ".otlp"
+	alwaysOn                 string = "always_on"
+	alwaysOff                string = "always_off"
+	traceIDRatio             string = "traceidratio"
+	parentBasedAlwaysOn      string = "parentbased_always_on"
+	parentBasedAlwaysOff     string = "parentbased_always_off"
+	parentBasedTraceIDRatio  string = "parentbased_traceidratio"
+)
+
+var OpenTelemetry = struct {
+	// Inverse of OTEL_SDK_DISABLE, skips telemetry setup
+	Enabled            bool
+	ServiceName        string
+	ResourceAttributes string
+	ResourceDetectors  string
+	Sampler            sdktrace.Sampler
+	Traces             string
+
+	OtelTraces *OtelExporter
+}{
+	ServiceName: "forgejo",
+	Traces:      "otel",
+}
+
+type OtelExporter struct {
+	Endpoint          *url.URL          `ini:"ENDPOINT"`
+	Headers           map[string]string `ini:"-"`
+	Compression       string            `ini:"COMPRESSION"`
+	Certificate       string            `ini:"CERTIFICATE"`
+	ClientKey         string            `ini:"CLIENT_KEY"`
+	ClientCertificate string            `ini:"CLIENT_CERTIFICATE"`
+	Timeout           time.Duration     `ini:"TIMEOUT"`
+	Protocol          string            `ini:"-"`
+}
+
+func createOtlpExporterConfig(rootCfg ConfigProvider, section string) *OtelExporter {
+	protocols := []string{"http/protobuf", "grpc"}
+	endpoint, _ := url.Parse("http://localhost:4318/")
+	exp := &OtelExporter{
+		Endpoint: endpoint,
+		Timeout:  10 * time.Second,
+		Headers:  map[string]string{},
+		Protocol: "http/protobuf",
+	}
+
+	loadSection := func(name string) {
+		otlp := rootCfg.Section(name)
+		if otlp.HasKey("ENDPOINT") {
+			endpoint, err := url.Parse(otlp.Key("ENDPOINT").String())
+			if err != nil {
+				log.Warn("Endpoint parsing failed, section: %s, err %v", name, err)
+			} else {
+				exp.Endpoint = endpoint
+			}
+		}
+		if err := otlp.MapTo(exp); err != nil {
+			log.Warn("Mapping otlp settings failed, section: %s, err: %v", name, err)
+		}
+
+		exp.Protocol = otlp.Key("PROTOCOL").In(exp.Protocol, protocols)
+
+		headers := otlp.Key("HEADERS").String()
+		if headers != "" {
+			for k, v := range _stringToHeader(headers) {
+				exp.Headers[k] = v
+			}
+		}
+	}
+	loadSection("opentelemetry.exporter.otlp")
+
+	loadSection("opentelemetry.exporter.otlp" + section)
+
+	if len(exp.Certificate) > 0 && !filepath.IsAbs(exp.Certificate) {
+		exp.Certificate = filepath.Join(CustomPath, exp.Certificate)
+	}
+	if len(exp.ClientCertificate) > 0 && !filepath.IsAbs(exp.ClientCertificate) {
+		exp.ClientCertificate = filepath.Join(CustomPath, exp.ClientCertificate)
+	}
+	if len(exp.ClientKey) > 0 && !filepath.IsAbs(exp.ClientKey) {
+		exp.ClientKey = filepath.Join(CustomPath, exp.ClientKey)
+	}
+
+	return exp
+}
+
+func loadOpenTelemetryFrom(rootCfg ConfigProvider) {
+	sec := rootCfg.Section(opentelemetrySectionName)
+	OpenTelemetry.Enabled = sec.Key("ENABLED").MustBool(false)
+	if !OpenTelemetry.Enabled {
+		return
+	}
+
+	// Load resource related settings
+	OpenTelemetry.ServiceName = sec.Key("SERVICE_NAME").MustString("forgejo")
+	OpenTelemetry.ResourceAttributes = sec.Key("RESOURCE_ATTRIBUTES").String()
+	OpenTelemetry.ResourceDetectors = strings.ToLower(sec.Key("RESOURCE_DETECTORS").String())
+
+	// Load tracing related settings
+	samplers := make([]string, 0, len(sampler))
+	for k := range sampler {
+		samplers = append(samplers, k)
+	}
+
+	samplerName := sec.Key("TRACES_SAMPLER").In(parentBasedAlwaysOn, samplers)
+	samplerArg := sec.Key("TRACES_SAMPLER_ARG").MustString("")
+	OpenTelemetry.Sampler = sampler[samplerName](samplerArg)
+
+	switch sec.Key("TRACES_EXPORTER").MustString("otlp") {
+	case "none":
+		OpenTelemetry.Traces = "none"
+	default:
+		OpenTelemetry.Traces = "otlp"
+		OpenTelemetry.OtelTraces = createOtlpExporterConfig(rootCfg, ".traces")
+	}
+}
+
+var sampler = map[string]func(arg string) sdktrace.Sampler{
+	alwaysOff: func(_ string) sdktrace.Sampler {
+		return sdktrace.NeverSample()
+	},
+	alwaysOn: func(_ string) sdktrace.Sampler {
+		return sdktrace.AlwaysSample()
+	},
+	traceIDRatio: func(arg string) sdktrace.Sampler {
+		ratio, err := strconv.ParseFloat(arg, 64)
+		if err != nil {
+			ratio = 1
+		}
+		return sdktrace.TraceIDRatioBased(ratio)
+	},
+	parentBasedAlwaysOff: func(_ string) sdktrace.Sampler {
+		return sdktrace.ParentBased(sdktrace.NeverSample())
+	},
+	parentBasedAlwaysOn: func(_ string) sdktrace.Sampler {
+		return sdktrace.ParentBased(sdktrace.AlwaysSample())
+	},
+	parentBasedTraceIDRatio: func(arg string) sdktrace.Sampler {
+		ratio, err := strconv.ParseFloat(arg, 64)
+		if err != nil {
+			ratio = 1
+		}
+		return sdktrace.ParentBased(sdktrace.TraceIDRatioBased(ratio))
+	},
+}
+
+// Opentelemetry SDK function port
+
+func _stringToHeader(value string) map[string]string {
+	headersPairs := strings.Split(value, ",")
+	headers := make(map[string]string)
+
+	for _, header := range headersPairs {
+		n, v, found := strings.Cut(header, "=")
+		if !found {
+			log.Warn("Otel header ignored on %q: missing '='", header)
+			continue
+		}
+		name, err := url.PathUnescape(n)
+		if err != nil {
+			log.Warn("Otel header ignored on %q, invalid header key: %s", header, n)
+			continue
+		}
+		trimmedName := strings.TrimSpace(name)
+		value, err := url.PathUnescape(v)
+		if err != nil {
+			log.Warn("Otel header ignored on %q, invalid header value: %s", header, v)
+			continue
+		}
+		trimmedValue := strings.TrimSpace(value)
+
+		headers[trimmedName] = trimmedValue
+	}
+
+	return headers
+}
+
+func IsOpenTelemetryEnabled() bool {
+	return OpenTelemetry.Enabled
+}
diff --git a/modules/setting/opentelemetry_test.go b/modules/setting/opentelemetry_test.go
new file mode 100644
index 0000000000..21da3837c7
--- /dev/null
+++ b/modules/setting/opentelemetry_test.go
@@ -0,0 +1,239 @@
+// Copyright 2024 TheFox0x7. All rights reserved.
+// SPDX-License-Identifier: EUPL-1.2
+
+package setting
+
+import (
+	"net/url"
+	"testing"
+	"time"
+
+	"code.gitea.io/gitea/modules/test"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+)
+
+func TestExporterLoad(t *testing.T) {
+	globalSetting := `
+	[opentelemetry.exporter.otlp]
+ENDPOINT=http://example.org:4318/
+CERTIFICATE=/boo/bar
+CLIENT_CERTIFICATE=/foo/bar
+CLIENT_KEY=/bar/bar
+COMPRESSION=
+HEADERS=key=val,val=key
+PROTOCOL=http/protobuf
+TIMEOUT=20s
+	`
+	endpoint, err := url.Parse("http://example.org:4318/")
+	require.NoError(t, err)
+	expected := &OtelExporter{
+		Endpoint:          endpoint,
+		Certificate:       "/boo/bar",
+		ClientCertificate: "/foo/bar",
+		ClientKey:         "/bar/bar",
+		Headers: map[string]string{
+			"key": "val", "val": "key",
+		},
+		Timeout:  20 * time.Second,
+		Protocol: "http/protobuf",
+	}
+	cfg, err := NewConfigProviderFromData(globalSetting)
+	require.NoError(t, err)
+	exp := createOtlpExporterConfig(cfg, ".traces")
+	assert.Equal(t, expected, exp)
+	localSetting := `
+[opentelemetry.exporter.otlp.traces]
+ENDPOINT=http://example.com:4318/
+CERTIFICATE=/boo
+CLIENT_CERTIFICATE=/foo
+CLIENT_KEY=/bar
+COMPRESSION=gzip
+HEADERS=key=val2,val1=key
+PROTOCOL=grpc
+TIMEOUT=5s
+	`
+	endpoint, err = url.Parse("http://example.com:4318/")
+	require.NoError(t, err)
+	expected = &OtelExporter{
+		Endpoint:          endpoint,
+		Certificate:       "/boo",
+		ClientCertificate: "/foo",
+		ClientKey:         "/bar",
+		Compression:       "gzip",
+		Headers: map[string]string{
+			"key": "val2", "val1": "key", "val": "key",
+		},
+		Timeout:  5 * time.Second,
+		Protocol: "grpc",
+	}
+
+	cfg, err = NewConfigProviderFromData(globalSetting + localSetting)
+	require.NoError(t, err)
+	exp = createOtlpExporterConfig(cfg, ".traces")
+	require.NoError(t, err)
+	assert.Equal(t, expected, exp)
+}
+
+func TestOpenTelemetryConfiguration(t *testing.T) {
+	defer test.MockProtect(&OpenTelemetry)()
+	iniStr := ``
+	cfg, err := NewConfigProviderFromData(iniStr)
+	require.NoError(t, err)
+	loadOpenTelemetryFrom(cfg)
+	assert.Nil(t, OpenTelemetry.OtelTraces)
+	assert.False(t, IsOpenTelemetryEnabled())
+
+	iniStr = `
+	[opentelemetry]
+	ENABLED=true
+	SERVICE_NAME = test service
+	RESOURCE_ATTRIBUTES = foo=bar
+	TRACES_SAMPLER = always_on
+
+	[opentelemetry.exporter.otlp]
+	ENDPOINT = http://jaeger:4317/
+	TIMEOUT = 30s
+	COMPRESSION = gzip
+	INSECURE = TRUE
+	HEADERS=foo=bar,overwrite=false
+	`
+	cfg, err = NewConfigProviderFromData(iniStr)
+	require.NoError(t, err)
+	loadOpenTelemetryFrom(cfg)
+
+	assert.True(t, IsOpenTelemetryEnabled())
+	assert.Equal(t, "test service", OpenTelemetry.ServiceName)
+	assert.Equal(t, "foo=bar", OpenTelemetry.ResourceAttributes)
+	assert.Equal(t, 30*time.Second, OpenTelemetry.OtelTraces.Timeout)
+	assert.Equal(t, "gzip", OpenTelemetry.OtelTraces.Compression)
+	assert.Equal(t, sdktrace.AlwaysSample(), OpenTelemetry.Sampler)
+	assert.Equal(t, "http://jaeger:4317/", OpenTelemetry.OtelTraces.Endpoint.String())
+	assert.Contains(t, OpenTelemetry.OtelTraces.Headers, "foo")
+	assert.Equal(t, "bar", OpenTelemetry.OtelTraces.Headers["foo"])
+	assert.Contains(t, OpenTelemetry.OtelTraces.Headers, "overwrite")
+	assert.Equal(t, "false", OpenTelemetry.OtelTraces.Headers["overwrite"])
+}
+
+func TestOpenTelemetryTraceDisable(t *testing.T) {
+	defer test.MockProtect(&OpenTelemetry)()
+	iniStr := ``
+	cfg, err := NewConfigProviderFromData(iniStr)
+	require.NoError(t, err)
+	loadOpenTelemetryFrom(cfg)
+	assert.False(t, OpenTelemetry.Enabled)
+	assert.False(t, IsOpenTelemetryEnabled())
+
+	iniStr = `
+	[opentelemetry]
+	ENABLED=true
+	EXPORTER_OTLP_ENDPOINT =
+	`
+	cfg, err = NewConfigProviderFromData(iniStr)
+	require.NoError(t, err)
+	loadOpenTelemetryFrom(cfg)
+
+	assert.True(t, IsOpenTelemetryEnabled())
+	endpoint, _ := url.Parse("http://localhost:4318/")
+	assert.Equal(t, endpoint, OpenTelemetry.OtelTraces.Endpoint)
+}
+
+func TestSamplerCombinations(t *testing.T) {
+	defer test.MockProtect(&OpenTelemetry)()
+	type config struct {
+		IniCfg   string
+		Expected sdktrace.Sampler
+	}
+	testSamplers := []config{
+		{`[opentelemetry]
+		ENABLED=true
+  TRACES_SAMPLER = always_on
+  TRACES_SAMPLER_ARG = nothing`, sdktrace.AlwaysSample()},
+		{`[opentelemetry]
+	ENABLED=true
+  TRACES_SAMPLER = always_off`, sdktrace.NeverSample()},
+		{`[opentelemetry]
+	ENABLED=true
+  TRACES_SAMPLER = traceidratio
+  TRACES_SAMPLER_ARG = 0.7`, sdktrace.TraceIDRatioBased(0.7)},
+		{`[opentelemetry]
+	ENABLED=true
+  TRACES_SAMPLER = traceidratio
+  TRACES_SAMPLER_ARG = badarg`, sdktrace.TraceIDRatioBased(1)},
+		{`[opentelemetry]
+	ENABLED=true
+  TRACES_SAMPLER = parentbased_always_off`, sdktrace.ParentBased(sdktrace.NeverSample())},
+		{`[opentelemetry]
+	ENABLED=true
+  TRACES_SAMPLER = parentbased_always_of`, sdktrace.ParentBased(sdktrace.AlwaysSample())},
+		{`[opentelemetry]
+	ENABLED=true
+  TRACES_SAMPLER = parentbased_traceidratio
+  TRACES_SAMPLER_ARG = 0.3`, sdktrace.ParentBased(sdktrace.TraceIDRatioBased(0.3))},
+		{`[opentelemetry]
+	ENABLED=true
+  TRACES_SAMPLER = parentbased_traceidratio
+  TRACES_SAMPLER_ARG = badarg`, sdktrace.ParentBased(sdktrace.TraceIDRatioBased(1))},
+		{`[opentelemetry]
+	ENABLED=true
+  TRACES_SAMPLER = not existing
+  TRACES_SAMPLER_ARG = badarg`, sdktrace.ParentBased(sdktrace.AlwaysSample())},
+	}
+
+	for _, sampler := range testSamplers {
+		cfg, err := NewConfigProviderFromData(sampler.IniCfg)
+		require.NoError(t, err)
+		loadOpenTelemetryFrom(cfg)
+		assert.Equal(t, sampler.Expected, OpenTelemetry.Sampler)
+	}
+}
+
+func TestOpentelemetryBadConfigs(t *testing.T) {
+	defer test.MockProtect(&OpenTelemetry)()
+	iniStr := `
+	[opentelemetry]
+	ENABLED=true
+
+	[opentelemetry.exporter.otlp]
+	ENDPOINT = jaeger:4317/
+	`
+	cfg, err := NewConfigProviderFromData(iniStr)
+	require.NoError(t, err)
+	loadOpenTelemetryFrom(cfg)
+
+	assert.True(t, IsOpenTelemetryEnabled())
+	assert.Equal(t, "jaeger:4317/", OpenTelemetry.OtelTraces.Endpoint.String())
+
+	iniStr = ``
+	cfg, err = NewConfigProviderFromData(iniStr)
+	require.NoError(t, err)
+	loadOpenTelemetryFrom(cfg)
+	assert.False(t, IsOpenTelemetryEnabled())
+
+	iniStr = `
+	[opentelemetry]
+	ENABLED=true
+	SERVICE_NAME =
+  TRACES_SAMPLER = not existing one
+	[opentelemetry.exporter.otlp]
+	ENDPOINT = http://jaeger:4317/
+
+	TIMEOUT = abc
+	COMPRESSION = foo
+	HEADERS=%s=bar,foo=%h,foo
+
+	`
+
+	cfg, err = NewConfigProviderFromData(iniStr)
+
+	require.NoError(t, err)
+	loadOpenTelemetryFrom(cfg)
+	assert.True(t, IsOpenTelemetryEnabled())
+	assert.Equal(t, "forgejo", OpenTelemetry.ServiceName)
+	assert.Equal(t, 10*time.Second, OpenTelemetry.OtelTraces.Timeout)
+	assert.Equal(t, sdktrace.ParentBased(sdktrace.AlwaysSample()), OpenTelemetry.Sampler)
+	assert.Equal(t, "http://jaeger:4317/", OpenTelemetry.OtelTraces.Endpoint.String())
+	assert.Empty(t, OpenTelemetry.OtelTraces.Headers)
+}
diff --git a/modules/setting/setting.go b/modules/setting/setting.go
index 892e41cddf..9c6f09c13b 100644
--- a/modules/setting/setting.go
+++ b/modules/setting/setting.go
@@ -150,6 +150,7 @@ func loadCommonSettingsFrom(cfg ConfigProvider) error {
 	loadAPIFrom(cfg)
 	loadBadgesFrom(cfg)
 	loadMetricsFrom(cfg)
+	loadOpenTelemetryFrom(cfg)
 	loadCamoFrom(cfg)
 	loadI18nFrom(cfg)
 	loadGitFrom(cfg)