summaryrefslogtreecommitdiffstats
path: root/routers/api/actions/artifacts.go
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--routers/api/actions/artifacts.go507
1 files changed, 507 insertions, 0 deletions
diff --git a/routers/api/actions/artifacts.go b/routers/api/actions/artifacts.go
new file mode 100644
index 0000000..bc29e44
--- /dev/null
+++ b/routers/api/actions/artifacts.go
@@ -0,0 +1,507 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package actions
+
+// GitHub Actions Artifacts API Simple Description
+//
+// 1. Upload artifact
+// 1.1. Post upload url
+// Post: /api/actions_pipeline/_apis/pipelines/workflows/{run_id}/artifacts?api-version=6.0-preview
+// Request:
+// {
+// "Type": "actions_storage",
+// "Name": "artifact"
+// }
+// Response:
+// {
+// "fileContainerResourceUrl":"/api/actions_pipeline/_apis/pipelines/workflows/{run_id}/artifacts/{artifact_id}/upload"
+// }
+// it acquires an upload url for artifact upload
+// 1.2. Upload artifact
+// PUT: /api/actions_pipeline/_apis/pipelines/workflows/{run_id}/artifacts/{artifact_id}/upload?itemPath=artifact%2Ffilename
+// it upload chunk with headers:
+// x-tfs-filelength: 1024 // total file length
+// content-length: 1024 // chunk length
+// x-actions-results-md5: md5sum // md5sum of chunk
+// content-range: bytes 0-1023/1024 // chunk range
+// we save all chunks to one storage directory after md5sum check
+// 1.3. Confirm upload
+// PATCH: /api/actions_pipeline/_apis/pipelines/workflows/{run_id}/artifacts/{artifact_id}/upload?itemPath=artifact%2Ffilename
+// it confirm upload and merge all chunks to one file, save this file to storage
+//
+// 2. Download artifact
+// 2.1 list artifacts
+// GET: /api/actions_pipeline/_apis/pipelines/workflows/{run_id}/artifacts?api-version=6.0-preview
+// Response:
+// {
+// "count": 1,
+// "value": [
+// {
+// "name": "artifact",
+// "fileContainerResourceUrl": "/api/actions_pipeline/_apis/pipelines/workflows/{run_id}/artifacts/{artifact_id}/path"
+// }
+// ]
+// }
+// 2.2 download artifact
+// GET: /api/actions_pipeline/_apis/pipelines/workflows/{run_id}/artifacts/{artifact_id}/path?api-version=6.0-preview
+// Response:
+// {
+// "value": [
+// {
+// "contentLocation": "/api/actions_pipeline/_apis/pipelines/workflows/{run_id}/artifacts/{artifact_id}/download",
+// "path": "artifact/filename",
+// "itemType": "file"
+// }
+// ]
+// }
+// 2.3 download artifact file
+// GET: /api/actions_pipeline/_apis/pipelines/workflows/{run_id}/artifacts/{artifact_id}/download?itemPath=artifact%2Ffilename
+// Response:
+// download file
+//
+
+import (
+ "crypto/md5"
+ "errors"
+ "fmt"
+ "net/http"
+ "strconv"
+ "strings"
+
+ "code.gitea.io/gitea/models/actions"
+ "code.gitea.io/gitea/models/db"
+ quota_model "code.gitea.io/gitea/models/quota"
+ "code.gitea.io/gitea/modules/json"
+ "code.gitea.io/gitea/modules/log"
+ "code.gitea.io/gitea/modules/setting"
+ "code.gitea.io/gitea/modules/storage"
+ "code.gitea.io/gitea/modules/util"
+ "code.gitea.io/gitea/modules/web"
+ web_types "code.gitea.io/gitea/modules/web/types"
+ actions_service "code.gitea.io/gitea/services/actions"
+ "code.gitea.io/gitea/services/context"
+)
+
+const artifactRouteBase = "/_apis/pipelines/workflows/{run_id}/artifacts"
+
+type artifactContextKeyType struct{}
+
+var artifactContextKey = artifactContextKeyType{}
+
+type ArtifactContext struct {
+ *context.Base
+
+ ActionTask *actions.ActionTask
+}
+
+func init() {
+ web.RegisterResponseStatusProvider[*ArtifactContext](func(req *http.Request) web_types.ResponseStatusProvider {
+ return req.Context().Value(artifactContextKey).(*ArtifactContext)
+ })
+}
+
+func ArtifactsRoutes(prefix string) *web.Route {
+ m := web.NewRoute()
+ m.Use(ArtifactContexter())
+
+ r := artifactRoutes{
+ prefix: prefix,
+ fs: storage.ActionsArtifacts,
+ }
+
+ m.Group(artifactRouteBase, func() {
+ // retrieve, list and confirm artifacts
+ m.Combo("").Get(r.listArtifacts).Post(r.getUploadArtifactURL).Patch(r.comfirmUploadArtifact)
+ // handle container artifacts list and download
+ m.Put("/{artifact_hash}/upload", r.uploadArtifact)
+ // handle artifacts download
+ m.Get("/{artifact_hash}/download_url", r.getDownloadArtifactURL)
+ m.Get("/{artifact_id}/download", r.downloadArtifact)
+ })
+
+ return m
+}
+
+func ArtifactContexter() func(next http.Handler) http.Handler {
+ return func(next http.Handler) http.Handler {
+ return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
+ base, baseCleanUp := context.NewBaseContext(resp, req)
+ defer baseCleanUp()
+
+ ctx := &ArtifactContext{Base: base}
+ ctx.AppendContextValue(artifactContextKey, ctx)
+
+ // action task call server api with Bearer ACTIONS_RUNTIME_TOKEN
+ // we should verify the ACTIONS_RUNTIME_TOKEN
+ authHeader := req.Header.Get("Authorization")
+ if len(authHeader) == 0 || !strings.HasPrefix(authHeader, "Bearer ") {
+ ctx.Error(http.StatusUnauthorized, "Bad authorization header")
+ return
+ }
+
+ // New act_runner uses jwt to authenticate
+ tID, err := actions_service.ParseAuthorizationToken(req)
+
+ var task *actions.ActionTask
+ if err == nil {
+ task, err = actions.GetTaskByID(req.Context(), tID)
+ if err != nil {
+ log.Error("Error runner api getting task by ID: %v", err)
+ ctx.Error(http.StatusInternalServerError, "Error runner api getting task by ID")
+ return
+ }
+ if task.Status != actions.StatusRunning {
+ log.Error("Error runner api getting task: task is not running")
+ ctx.Error(http.StatusInternalServerError, "Error runner api getting task: task is not running")
+ return
+ }
+ } else {
+ // Old act_runner uses GITEA_TOKEN to authenticate
+ authToken := strings.TrimPrefix(authHeader, "Bearer ")
+
+ task, err = actions.GetRunningTaskByToken(req.Context(), authToken)
+ if err != nil {
+ log.Error("Error runner api getting task: %v", err)
+ ctx.Error(http.StatusInternalServerError, "Error runner api getting task")
+ return
+ }
+ }
+
+ if err := task.LoadJob(req.Context()); err != nil {
+ log.Error("Error runner api getting job: %v", err)
+ ctx.Error(http.StatusInternalServerError, "Error runner api getting job")
+ return
+ }
+
+ ctx.ActionTask = task
+ next.ServeHTTP(ctx.Resp, ctx.Req)
+ })
+ }
+}
+
+type artifactRoutes struct {
+ prefix string
+ fs storage.ObjectStorage
+}
+
+func (ar artifactRoutes) buildArtifactURL(runID int64, artifactHash, suffix string) string {
+ uploadURL := strings.TrimSuffix(setting.AppURL, "/") + strings.TrimSuffix(ar.prefix, "/") +
+ strings.ReplaceAll(artifactRouteBase, "{run_id}", strconv.FormatInt(runID, 10)) +
+ "/" + artifactHash + "/" + suffix
+ return uploadURL
+}
+
+type getUploadArtifactRequest struct {
+ Type string
+ Name string
+ RetentionDays int64
+}
+
+type getUploadArtifactResponse struct {
+ FileContainerResourceURL string `json:"fileContainerResourceUrl"`
+}
+
+// getUploadArtifactURL generates a URL for uploading an artifact
+func (ar artifactRoutes) getUploadArtifactURL(ctx *ArtifactContext) {
+ _, runID, ok := validateRunID(ctx)
+ if !ok {
+ return
+ }
+
+ var req getUploadArtifactRequest
+ if err := json.NewDecoder(ctx.Req.Body).Decode(&req); err != nil {
+ log.Error("Error decode request body: %v", err)
+ ctx.Error(http.StatusInternalServerError, "Error decode request body")
+ return
+ }
+
+ // set retention days
+ retentionQuery := ""
+ if req.RetentionDays > 0 {
+ retentionQuery = fmt.Sprintf("?retentionDays=%d", req.RetentionDays)
+ }
+
+ // use md5(artifact_name) to create upload url
+ artifactHash := fmt.Sprintf("%x", md5.Sum([]byte(req.Name)))
+ resp := getUploadArtifactResponse{
+ FileContainerResourceURL: ar.buildArtifactURL(runID, artifactHash, "upload"+retentionQuery),
+ }
+ log.Debug("[artifact] get upload url: %s", resp.FileContainerResourceURL)
+ ctx.JSON(http.StatusOK, resp)
+}
+
+func (ar artifactRoutes) uploadArtifact(ctx *ArtifactContext) {
+ task, runID, ok := validateRunID(ctx)
+ if !ok {
+ return
+ }
+ artifactName, artifactPath, ok := parseArtifactItemPath(ctx)
+ if !ok {
+ return
+ }
+
+ // check the owner's quota
+ ok, err := quota_model.EvaluateForUser(ctx, ctx.ActionTask.OwnerID, quota_model.LimitSubjectSizeAssetsArtifacts)
+ if err != nil {
+ log.Error("quota_model.EvaluateForUser: %v", err)
+ ctx.Error(http.StatusInternalServerError, "Error checking quota")
+ return
+ }
+ if !ok {
+ ctx.Error(http.StatusRequestEntityTooLarge, "Quota exceeded")
+ return
+ }
+
+ // get upload file size
+ fileRealTotalSize, contentLength := getUploadFileSize(ctx)
+
+ // get artifact retention days
+ expiredDays := setting.Actions.ArtifactRetentionDays
+ if queryRetentionDays := ctx.Req.URL.Query().Get("retentionDays"); queryRetentionDays != "" {
+ var err error
+ expiredDays, err = strconv.ParseInt(queryRetentionDays, 10, 64)
+ if err != nil {
+ log.Error("Error parse retention days: %v", err)
+ ctx.Error(http.StatusBadRequest, "Error parse retention days")
+ return
+ }
+ }
+ log.Debug("[artifact] upload chunk, name: %s, path: %s, size: %d, retention days: %d",
+ artifactName, artifactPath, fileRealTotalSize, expiredDays)
+
+ // create or get artifact with name and path
+ artifact, err := actions.CreateArtifact(ctx, task, artifactName, artifactPath, expiredDays)
+ if err != nil {
+ log.Error("Error create or get artifact: %v", err)
+ ctx.Error(http.StatusInternalServerError, "Error create or get artifact")
+ return
+ }
+
+ // save chunk to storage, if success, return chunk stotal size
+ // if artifact is not gzip when uploading, chunksTotalSize == fileRealTotalSize
+ // if artifact is gzip when uploading, chunksTotalSize < fileRealTotalSize
+ chunksTotalSize, err := saveUploadChunk(ar.fs, ctx, artifact, contentLength, runID)
+ if err != nil {
+ log.Error("Error save upload chunk: %v", err)
+ ctx.Error(http.StatusInternalServerError, "Error save upload chunk")
+ return
+ }
+
+ // update artifact size if zero or not match, over write artifact size
+ if artifact.FileSize == 0 ||
+ artifact.FileCompressedSize == 0 ||
+ artifact.FileSize != fileRealTotalSize ||
+ artifact.FileCompressedSize != chunksTotalSize {
+ artifact.FileSize = fileRealTotalSize
+ artifact.FileCompressedSize = chunksTotalSize
+ artifact.ContentEncoding = ctx.Req.Header.Get("Content-Encoding")
+ if err := actions.UpdateArtifactByID(ctx, artifact.ID, artifact); err != nil {
+ log.Error("Error update artifact: %v", err)
+ ctx.Error(http.StatusInternalServerError, "Error update artifact")
+ return
+ }
+ log.Debug("[artifact] update artifact size, artifact_id: %d, size: %d, compressed size: %d",
+ artifact.ID, artifact.FileSize, artifact.FileCompressedSize)
+ }
+
+ ctx.JSON(http.StatusOK, map[string]string{
+ "message": "success",
+ })
+}
+
+// comfirmUploadArtifact confirm upload artifact.
+// if all chunks are uploaded, merge them to one file.
+func (ar artifactRoutes) comfirmUploadArtifact(ctx *ArtifactContext) {
+ _, runID, ok := validateRunID(ctx)
+ if !ok {
+ return
+ }
+ artifactName := ctx.Req.URL.Query().Get("artifactName")
+ if artifactName == "" {
+ log.Warn("Error artifact name is empty")
+ ctx.Error(http.StatusBadRequest, "Error artifact name is empty")
+ return
+ }
+ if err := mergeChunksForRun(ctx, ar.fs, runID, artifactName); err != nil {
+ log.Error("Error merge chunks: %v", err)
+ ctx.Error(http.StatusInternalServerError, "Error merge chunks")
+ return
+ }
+ ctx.JSON(http.StatusOK, map[string]string{
+ "message": "success",
+ })
+}
+
+type (
+ listArtifactsResponse struct {
+ Count int64 `json:"count"`
+ Value []listArtifactsResponseItem `json:"value"`
+ }
+ listArtifactsResponseItem struct {
+ Name string `json:"name"`
+ FileContainerResourceURL string `json:"fileContainerResourceUrl"`
+ }
+)
+
+func (ar artifactRoutes) listArtifacts(ctx *ArtifactContext) {
+ _, runID, ok := validateRunID(ctx)
+ if !ok {
+ return
+ }
+
+ artifacts, err := db.Find[actions.ActionArtifact](ctx, actions.FindArtifactsOptions{RunID: runID})
+ if err != nil {
+ log.Error("Error getting artifacts: %v", err)
+ ctx.Error(http.StatusInternalServerError, err.Error())
+ return
+ }
+ if len(artifacts) == 0 {
+ log.Debug("[artifact] handleListArtifacts, no artifacts")
+ ctx.Error(http.StatusNotFound)
+ return
+ }
+
+ var (
+ items []listArtifactsResponseItem
+ values = make(map[string]bool)
+ )
+
+ for _, art := range artifacts {
+ if values[art.ArtifactName] {
+ continue
+ }
+ artifactHash := fmt.Sprintf("%x", md5.Sum([]byte(art.ArtifactName)))
+ item := listArtifactsResponseItem{
+ Name: art.ArtifactName,
+ FileContainerResourceURL: ar.buildArtifactURL(runID, artifactHash, "download_url"),
+ }
+ items = append(items, item)
+ values[art.ArtifactName] = true
+
+ log.Debug("[artifact] handleListArtifacts, name: %s, url: %s", item.Name, item.FileContainerResourceURL)
+ }
+
+ respData := listArtifactsResponse{
+ Count: int64(len(items)),
+ Value: items,
+ }
+ ctx.JSON(http.StatusOK, respData)
+}
+
+type (
+ downloadArtifactResponse struct {
+ Value []downloadArtifactResponseItem `json:"value"`
+ }
+ downloadArtifactResponseItem struct {
+ Path string `json:"path"`
+ ItemType string `json:"itemType"`
+ ContentLocation string `json:"contentLocation"`
+ }
+)
+
+// getDownloadArtifactURL generates download url for each artifact
+func (ar artifactRoutes) getDownloadArtifactURL(ctx *ArtifactContext) {
+ _, runID, ok := validateRunID(ctx)
+ if !ok {
+ return
+ }
+
+ itemPath := util.PathJoinRel(ctx.Req.URL.Query().Get("itemPath"))
+ if !validateArtifactHash(ctx, itemPath) {
+ return
+ }
+
+ artifacts, err := db.Find[actions.ActionArtifact](ctx, actions.FindArtifactsOptions{
+ RunID: runID,
+ ArtifactName: itemPath,
+ })
+ if err != nil {
+ log.Error("Error getting artifacts: %v", err)
+ ctx.Error(http.StatusInternalServerError, err.Error())
+ return
+ }
+ if len(artifacts) == 0 {
+ log.Debug("[artifact] getDownloadArtifactURL, no artifacts")
+ ctx.Error(http.StatusNotFound)
+ return
+ }
+
+ if itemPath != artifacts[0].ArtifactName {
+ log.Error("Error mismatch artifact name, itemPath: %v, artifact: %v", itemPath, artifacts[0].ArtifactName)
+ ctx.Error(http.StatusBadRequest, "Error mismatch artifact name")
+ return
+ }
+
+ var items []downloadArtifactResponseItem
+ for _, artifact := range artifacts {
+ var downloadURL string
+ if setting.Actions.ArtifactStorage.MinioConfig.ServeDirect {
+ u, err := ar.fs.URL(artifact.StoragePath, artifact.ArtifactName)
+ if err != nil && !errors.Is(err, storage.ErrURLNotSupported) {
+ log.Error("Error getting serve direct url: %v", err)
+ }
+ if u != nil {
+ downloadURL = u.String()
+ }
+ }
+ if downloadURL == "" {
+ downloadURL = ar.buildArtifactURL(runID, strconv.FormatInt(artifact.ID, 10), "download")
+ }
+ item := downloadArtifactResponseItem{
+ Path: util.PathJoinRel(itemPath, artifact.ArtifactPath),
+ ItemType: "file",
+ ContentLocation: downloadURL,
+ }
+ log.Debug("[artifact] getDownloadArtifactURL, path: %s, url: %s", item.Path, item.ContentLocation)
+ items = append(items, item)
+ }
+ respData := downloadArtifactResponse{
+ Value: items,
+ }
+ ctx.JSON(http.StatusOK, respData)
+}
+
+// downloadArtifact downloads artifact content
+func (ar artifactRoutes) downloadArtifact(ctx *ArtifactContext) {
+ _, runID, ok := validateRunID(ctx)
+ if !ok {
+ return
+ }
+
+ artifactID := ctx.ParamsInt64("artifact_id")
+ artifact, exist, err := db.GetByID[actions.ActionArtifact](ctx, artifactID)
+ if err != nil {
+ log.Error("Error getting artifact: %v", err)
+ ctx.Error(http.StatusInternalServerError, err.Error())
+ return
+ }
+ if !exist {
+ log.Error("artifact with ID %d does not exist", artifactID)
+ ctx.Error(http.StatusNotFound, fmt.Sprintf("artifact with ID %d does not exist", artifactID))
+ return
+ }
+ if artifact.RunID != runID {
+ log.Error("Error mismatch runID and artifactID, task: %v, artifact: %v", runID, artifactID)
+ ctx.Error(http.StatusBadRequest)
+ return
+ }
+
+ fd, err := ar.fs.Open(artifact.StoragePath)
+ if err != nil {
+ log.Error("Error opening file: %v", err)
+ ctx.Error(http.StatusInternalServerError, err.Error())
+ return
+ }
+ defer fd.Close()
+
+ // if artifact is compressed, set content-encoding header to gzip
+ if artifact.ContentEncoding == "gzip" {
+ ctx.Resp.Header().Set("Content-Encoding", "gzip")
+ }
+ log.Debug("[artifact] downloadArtifact, name: %s, path: %s, storage: %s, size: %d", artifact.ArtifactName, artifact.ArtifactPath, artifact.StoragePath, artifact.FileSize)
+ ctx.ServeContent(fd, &context.ServeHeaderOptions{
+ Filename: artifact.ArtifactName,
+ LastModified: artifact.CreatedUnix.AsLocalTime(),
+ })
+}