diff options
Diffstat (limited to '')
| -rw-r--r-- | routers/api/actions/artifacts.go | 507 |
1 files changed, 507 insertions, 0 deletions
diff --git a/routers/api/actions/artifacts.go b/routers/api/actions/artifacts.go new file mode 100644 index 0000000..bc29e44 --- /dev/null +++ b/routers/api/actions/artifacts.go @@ -0,0 +1,507 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package actions + +// GitHub Actions Artifacts API Simple Description +// +// 1. Upload artifact +// 1.1. Post upload url +// Post: /api/actions_pipeline/_apis/pipelines/workflows/{run_id}/artifacts?api-version=6.0-preview +// Request: +// { +// "Type": "actions_storage", +// "Name": "artifact" +// } +// Response: +// { +// "fileContainerResourceUrl":"/api/actions_pipeline/_apis/pipelines/workflows/{run_id}/artifacts/{artifact_id}/upload" +// } +// it acquires an upload url for artifact upload +// 1.2. Upload artifact +// PUT: /api/actions_pipeline/_apis/pipelines/workflows/{run_id}/artifacts/{artifact_id}/upload?itemPath=artifact%2Ffilename +// it upload chunk with headers: +// x-tfs-filelength: 1024 // total file length +// content-length: 1024 // chunk length +// x-actions-results-md5: md5sum // md5sum of chunk +// content-range: bytes 0-1023/1024 // chunk range +// we save all chunks to one storage directory after md5sum check +// 1.3. Confirm upload +// PATCH: /api/actions_pipeline/_apis/pipelines/workflows/{run_id}/artifacts/{artifact_id}/upload?itemPath=artifact%2Ffilename +// it confirm upload and merge all chunks to one file, save this file to storage +// +// 2. Download artifact +// 2.1 list artifacts +// GET: /api/actions_pipeline/_apis/pipelines/workflows/{run_id}/artifacts?api-version=6.0-preview +// Response: +// { +// "count": 1, +// "value": [ +// { +// "name": "artifact", +// "fileContainerResourceUrl": "/api/actions_pipeline/_apis/pipelines/workflows/{run_id}/artifacts/{artifact_id}/path" +// } +// ] +// } +// 2.2 download artifact +// GET: /api/actions_pipeline/_apis/pipelines/workflows/{run_id}/artifacts/{artifact_id}/path?api-version=6.0-preview +// Response: +// { +// "value": [ +// { +// "contentLocation": "/api/actions_pipeline/_apis/pipelines/workflows/{run_id}/artifacts/{artifact_id}/download", +// "path": "artifact/filename", +// "itemType": "file" +// } +// ] +// } +// 2.3 download artifact file +// GET: /api/actions_pipeline/_apis/pipelines/workflows/{run_id}/artifacts/{artifact_id}/download?itemPath=artifact%2Ffilename +// Response: +// download file +// + +import ( + "crypto/md5" + "errors" + "fmt" + "net/http" + "strconv" + "strings" + + "code.gitea.io/gitea/models/actions" + "code.gitea.io/gitea/models/db" + quota_model "code.gitea.io/gitea/models/quota" + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/web" + web_types "code.gitea.io/gitea/modules/web/types" + actions_service "code.gitea.io/gitea/services/actions" + "code.gitea.io/gitea/services/context" +) + +const artifactRouteBase = "/_apis/pipelines/workflows/{run_id}/artifacts" + +type artifactContextKeyType struct{} + +var artifactContextKey = artifactContextKeyType{} + +type ArtifactContext struct { + *context.Base + + ActionTask *actions.ActionTask +} + +func init() { + web.RegisterResponseStatusProvider[*ArtifactContext](func(req *http.Request) web_types.ResponseStatusProvider { + return req.Context().Value(artifactContextKey).(*ArtifactContext) + }) +} + +func ArtifactsRoutes(prefix string) *web.Route { + m := web.NewRoute() + m.Use(ArtifactContexter()) + + r := artifactRoutes{ + prefix: prefix, + fs: storage.ActionsArtifacts, + } + + m.Group(artifactRouteBase, func() { + // retrieve, list and confirm artifacts + m.Combo("").Get(r.listArtifacts).Post(r.getUploadArtifactURL).Patch(r.comfirmUploadArtifact) + // handle container artifacts list and download + m.Put("/{artifact_hash}/upload", r.uploadArtifact) + // handle artifacts download + m.Get("/{artifact_hash}/download_url", r.getDownloadArtifactURL) + m.Get("/{artifact_id}/download", r.downloadArtifact) + }) + + return m +} + +func ArtifactContexter() func(next http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { + base, baseCleanUp := context.NewBaseContext(resp, req) + defer baseCleanUp() + + ctx := &ArtifactContext{Base: base} + ctx.AppendContextValue(artifactContextKey, ctx) + + // action task call server api with Bearer ACTIONS_RUNTIME_TOKEN + // we should verify the ACTIONS_RUNTIME_TOKEN + authHeader := req.Header.Get("Authorization") + if len(authHeader) == 0 || !strings.HasPrefix(authHeader, "Bearer ") { + ctx.Error(http.StatusUnauthorized, "Bad authorization header") + return + } + + // New act_runner uses jwt to authenticate + tID, err := actions_service.ParseAuthorizationToken(req) + + var task *actions.ActionTask + if err == nil { + task, err = actions.GetTaskByID(req.Context(), tID) + if err != nil { + log.Error("Error runner api getting task by ID: %v", err) + ctx.Error(http.StatusInternalServerError, "Error runner api getting task by ID") + return + } + if task.Status != actions.StatusRunning { + log.Error("Error runner api getting task: task is not running") + ctx.Error(http.StatusInternalServerError, "Error runner api getting task: task is not running") + return + } + } else { + // Old act_runner uses GITEA_TOKEN to authenticate + authToken := strings.TrimPrefix(authHeader, "Bearer ") + + task, err = actions.GetRunningTaskByToken(req.Context(), authToken) + if err != nil { + log.Error("Error runner api getting task: %v", err) + ctx.Error(http.StatusInternalServerError, "Error runner api getting task") + return + } + } + + if err := task.LoadJob(req.Context()); err != nil { + log.Error("Error runner api getting job: %v", err) + ctx.Error(http.StatusInternalServerError, "Error runner api getting job") + return + } + + ctx.ActionTask = task + next.ServeHTTP(ctx.Resp, ctx.Req) + }) + } +} + +type artifactRoutes struct { + prefix string + fs storage.ObjectStorage +} + +func (ar artifactRoutes) buildArtifactURL(runID int64, artifactHash, suffix string) string { + uploadURL := strings.TrimSuffix(setting.AppURL, "/") + strings.TrimSuffix(ar.prefix, "/") + + strings.ReplaceAll(artifactRouteBase, "{run_id}", strconv.FormatInt(runID, 10)) + + "/" + artifactHash + "/" + suffix + return uploadURL +} + +type getUploadArtifactRequest struct { + Type string + Name string + RetentionDays int64 +} + +type getUploadArtifactResponse struct { + FileContainerResourceURL string `json:"fileContainerResourceUrl"` +} + +// getUploadArtifactURL generates a URL for uploading an artifact +func (ar artifactRoutes) getUploadArtifactURL(ctx *ArtifactContext) { + _, runID, ok := validateRunID(ctx) + if !ok { + return + } + + var req getUploadArtifactRequest + if err := json.NewDecoder(ctx.Req.Body).Decode(&req); err != nil { + log.Error("Error decode request body: %v", err) + ctx.Error(http.StatusInternalServerError, "Error decode request body") + return + } + + // set retention days + retentionQuery := "" + if req.RetentionDays > 0 { + retentionQuery = fmt.Sprintf("?retentionDays=%d", req.RetentionDays) + } + + // use md5(artifact_name) to create upload url + artifactHash := fmt.Sprintf("%x", md5.Sum([]byte(req.Name))) + resp := getUploadArtifactResponse{ + FileContainerResourceURL: ar.buildArtifactURL(runID, artifactHash, "upload"+retentionQuery), + } + log.Debug("[artifact] get upload url: %s", resp.FileContainerResourceURL) + ctx.JSON(http.StatusOK, resp) +} + +func (ar artifactRoutes) uploadArtifact(ctx *ArtifactContext) { + task, runID, ok := validateRunID(ctx) + if !ok { + return + } + artifactName, artifactPath, ok := parseArtifactItemPath(ctx) + if !ok { + return + } + + // check the owner's quota + ok, err := quota_model.EvaluateForUser(ctx, ctx.ActionTask.OwnerID, quota_model.LimitSubjectSizeAssetsArtifacts) + if err != nil { + log.Error("quota_model.EvaluateForUser: %v", err) + ctx.Error(http.StatusInternalServerError, "Error checking quota") + return + } + if !ok { + ctx.Error(http.StatusRequestEntityTooLarge, "Quota exceeded") + return + } + + // get upload file size + fileRealTotalSize, contentLength := getUploadFileSize(ctx) + + // get artifact retention days + expiredDays := setting.Actions.ArtifactRetentionDays + if queryRetentionDays := ctx.Req.URL.Query().Get("retentionDays"); queryRetentionDays != "" { + var err error + expiredDays, err = strconv.ParseInt(queryRetentionDays, 10, 64) + if err != nil { + log.Error("Error parse retention days: %v", err) + ctx.Error(http.StatusBadRequest, "Error parse retention days") + return + } + } + log.Debug("[artifact] upload chunk, name: %s, path: %s, size: %d, retention days: %d", + artifactName, artifactPath, fileRealTotalSize, expiredDays) + + // create or get artifact with name and path + artifact, err := actions.CreateArtifact(ctx, task, artifactName, artifactPath, expiredDays) + if err != nil { + log.Error("Error create or get artifact: %v", err) + ctx.Error(http.StatusInternalServerError, "Error create or get artifact") + return + } + + // save chunk to storage, if success, return chunk stotal size + // if artifact is not gzip when uploading, chunksTotalSize == fileRealTotalSize + // if artifact is gzip when uploading, chunksTotalSize < fileRealTotalSize + chunksTotalSize, err := saveUploadChunk(ar.fs, ctx, artifact, contentLength, runID) + if err != nil { + log.Error("Error save upload chunk: %v", err) + ctx.Error(http.StatusInternalServerError, "Error save upload chunk") + return + } + + // update artifact size if zero or not match, over write artifact size + if artifact.FileSize == 0 || + artifact.FileCompressedSize == 0 || + artifact.FileSize != fileRealTotalSize || + artifact.FileCompressedSize != chunksTotalSize { + artifact.FileSize = fileRealTotalSize + artifact.FileCompressedSize = chunksTotalSize + artifact.ContentEncoding = ctx.Req.Header.Get("Content-Encoding") + if err := actions.UpdateArtifactByID(ctx, artifact.ID, artifact); err != nil { + log.Error("Error update artifact: %v", err) + ctx.Error(http.StatusInternalServerError, "Error update artifact") + return + } + log.Debug("[artifact] update artifact size, artifact_id: %d, size: %d, compressed size: %d", + artifact.ID, artifact.FileSize, artifact.FileCompressedSize) + } + + ctx.JSON(http.StatusOK, map[string]string{ + "message": "success", + }) +} + +// comfirmUploadArtifact confirm upload artifact. +// if all chunks are uploaded, merge them to one file. +func (ar artifactRoutes) comfirmUploadArtifact(ctx *ArtifactContext) { + _, runID, ok := validateRunID(ctx) + if !ok { + return + } + artifactName := ctx.Req.URL.Query().Get("artifactName") + if artifactName == "" { + log.Warn("Error artifact name is empty") + ctx.Error(http.StatusBadRequest, "Error artifact name is empty") + return + } + if err := mergeChunksForRun(ctx, ar.fs, runID, artifactName); err != nil { + log.Error("Error merge chunks: %v", err) + ctx.Error(http.StatusInternalServerError, "Error merge chunks") + return + } + ctx.JSON(http.StatusOK, map[string]string{ + "message": "success", + }) +} + +type ( + listArtifactsResponse struct { + Count int64 `json:"count"` + Value []listArtifactsResponseItem `json:"value"` + } + listArtifactsResponseItem struct { + Name string `json:"name"` + FileContainerResourceURL string `json:"fileContainerResourceUrl"` + } +) + +func (ar artifactRoutes) listArtifacts(ctx *ArtifactContext) { + _, runID, ok := validateRunID(ctx) + if !ok { + return + } + + artifacts, err := db.Find[actions.ActionArtifact](ctx, actions.FindArtifactsOptions{RunID: runID}) + if err != nil { + log.Error("Error getting artifacts: %v", err) + ctx.Error(http.StatusInternalServerError, err.Error()) + return + } + if len(artifacts) == 0 { + log.Debug("[artifact] handleListArtifacts, no artifacts") + ctx.Error(http.StatusNotFound) + return + } + + var ( + items []listArtifactsResponseItem + values = make(map[string]bool) + ) + + for _, art := range artifacts { + if values[art.ArtifactName] { + continue + } + artifactHash := fmt.Sprintf("%x", md5.Sum([]byte(art.ArtifactName))) + item := listArtifactsResponseItem{ + Name: art.ArtifactName, + FileContainerResourceURL: ar.buildArtifactURL(runID, artifactHash, "download_url"), + } + items = append(items, item) + values[art.ArtifactName] = true + + log.Debug("[artifact] handleListArtifacts, name: %s, url: %s", item.Name, item.FileContainerResourceURL) + } + + respData := listArtifactsResponse{ + Count: int64(len(items)), + Value: items, + } + ctx.JSON(http.StatusOK, respData) +} + +type ( + downloadArtifactResponse struct { + Value []downloadArtifactResponseItem `json:"value"` + } + downloadArtifactResponseItem struct { + Path string `json:"path"` + ItemType string `json:"itemType"` + ContentLocation string `json:"contentLocation"` + } +) + +// getDownloadArtifactURL generates download url for each artifact +func (ar artifactRoutes) getDownloadArtifactURL(ctx *ArtifactContext) { + _, runID, ok := validateRunID(ctx) + if !ok { + return + } + + itemPath := util.PathJoinRel(ctx.Req.URL.Query().Get("itemPath")) + if !validateArtifactHash(ctx, itemPath) { + return + } + + artifacts, err := db.Find[actions.ActionArtifact](ctx, actions.FindArtifactsOptions{ + RunID: runID, + ArtifactName: itemPath, + }) + if err != nil { + log.Error("Error getting artifacts: %v", err) + ctx.Error(http.StatusInternalServerError, err.Error()) + return + } + if len(artifacts) == 0 { + log.Debug("[artifact] getDownloadArtifactURL, no artifacts") + ctx.Error(http.StatusNotFound) + return + } + + if itemPath != artifacts[0].ArtifactName { + log.Error("Error mismatch artifact name, itemPath: %v, artifact: %v", itemPath, artifacts[0].ArtifactName) + ctx.Error(http.StatusBadRequest, "Error mismatch artifact name") + return + } + + var items []downloadArtifactResponseItem + for _, artifact := range artifacts { + var downloadURL string + if setting.Actions.ArtifactStorage.MinioConfig.ServeDirect { + u, err := ar.fs.URL(artifact.StoragePath, artifact.ArtifactName) + if err != nil && !errors.Is(err, storage.ErrURLNotSupported) { + log.Error("Error getting serve direct url: %v", err) + } + if u != nil { + downloadURL = u.String() + } + } + if downloadURL == "" { + downloadURL = ar.buildArtifactURL(runID, strconv.FormatInt(artifact.ID, 10), "download") + } + item := downloadArtifactResponseItem{ + Path: util.PathJoinRel(itemPath, artifact.ArtifactPath), + ItemType: "file", + ContentLocation: downloadURL, + } + log.Debug("[artifact] getDownloadArtifactURL, path: %s, url: %s", item.Path, item.ContentLocation) + items = append(items, item) + } + respData := downloadArtifactResponse{ + Value: items, + } + ctx.JSON(http.StatusOK, respData) +} + +// downloadArtifact downloads artifact content +func (ar artifactRoutes) downloadArtifact(ctx *ArtifactContext) { + _, runID, ok := validateRunID(ctx) + if !ok { + return + } + + artifactID := ctx.ParamsInt64("artifact_id") + artifact, exist, err := db.GetByID[actions.ActionArtifact](ctx, artifactID) + if err != nil { + log.Error("Error getting artifact: %v", err) + ctx.Error(http.StatusInternalServerError, err.Error()) + return + } + if !exist { + log.Error("artifact with ID %d does not exist", artifactID) + ctx.Error(http.StatusNotFound, fmt.Sprintf("artifact with ID %d does not exist", artifactID)) + return + } + if artifact.RunID != runID { + log.Error("Error mismatch runID and artifactID, task: %v, artifact: %v", runID, artifactID) + ctx.Error(http.StatusBadRequest) + return + } + + fd, err := ar.fs.Open(artifact.StoragePath) + if err != nil { + log.Error("Error opening file: %v", err) + ctx.Error(http.StatusInternalServerError, err.Error()) + return + } + defer fd.Close() + + // if artifact is compressed, set content-encoding header to gzip + if artifact.ContentEncoding == "gzip" { + ctx.Resp.Header().Set("Content-Encoding", "gzip") + } + log.Debug("[artifact] downloadArtifact, name: %s, path: %s, storage: %s, size: %d", artifact.ArtifactName, artifact.ArtifactPath, artifact.StoragePath, artifact.FileSize) + ctx.ServeContent(fd, &context.ServeHeaderOptions{ + Filename: artifact.ArtifactName, + LastModified: artifact.CreatedUnix.AsLocalTime(), + }) +} |