diff options
Diffstat (limited to 'services/repository/files')
| -rw-r--r-- | services/repository/files/cherry_pick.go | 128 | ||||
| -rw-r--r-- | services/repository/files/commit.go | 44 | ||||
| -rw-r--r-- | services/repository/files/content.go | 278 | ||||
| -rw-r--r-- | services/repository/files/content_test.go | 201 | ||||
| -rw-r--r-- | services/repository/files/diff.go | 42 | ||||
| -rw-r--r-- | services/repository/files/diff_test.go | 166 | ||||
| -rw-r--r-- | services/repository/files/file.go | 174 | ||||
| -rw-r--r-- | services/repository/files/file_test.go | 115 | ||||
| -rw-r--r-- | services/repository/files/patch.go | 199 | ||||
| -rw-r--r-- | services/repository/files/temp_repo.go | 406 | ||||
| -rw-r--r-- | services/repository/files/temp_repo_test.go | 28 | ||||
| -rw-r--r-- | services/repository/files/tree.go | 101 | ||||
| -rw-r--r-- | services/repository/files/tree_test.go | 52 | ||||
| -rw-r--r-- | services/repository/files/update.go | 501 | ||||
| -rw-r--r-- | services/repository/files/upload.go | 248 |
15 files changed, 2683 insertions, 0 deletions
diff --git a/services/repository/files/cherry_pick.go b/services/repository/files/cherry_pick.go new file mode 100644 index 0000000..451a182 --- /dev/null +++ b/services/repository/files/cherry_pick.go @@ -0,0 +1,128 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package files + +import ( + "context" + "fmt" + "strings" + + "code.gitea.io/gitea/models" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/services/pull" +) + +// CherryPick cherrypicks or reverts a commit to the given repository +func CherryPick(ctx context.Context, repo *repo_model.Repository, doer *user_model.User, revert bool, opts *ApplyDiffPatchOptions) (*structs.FileResponse, error) { + if err := opts.Validate(ctx, repo, doer); err != nil { + return nil, err + } + message := strings.TrimSpace(opts.Message) + + author, committer := GetAuthorAndCommitterUsers(opts.Author, opts.Committer, doer) + + t, err := NewTemporaryUploadRepository(ctx, repo) + if err != nil { + log.Error("NewTemporaryUploadRepository failed: %v", err) + } + defer t.Close() + if err := t.Clone(opts.OldBranch, false); err != nil { + return nil, err + } + if err := t.SetDefaultIndex(); err != nil { + return nil, err + } + if err := t.RefreshIndex(); err != nil { + return nil, err + } + + // Get the commit of the original branch + commit, err := t.GetBranchCommit(opts.OldBranch) + if err != nil { + return nil, err // Couldn't get a commit for the branch + } + + // Assigned LastCommitID in opts if it hasn't been set + if opts.LastCommitID == "" { + opts.LastCommitID = commit.ID.String() + } else { + lastCommitID, err := t.gitRepo.ConvertToGitID(opts.LastCommitID) + if err != nil { + return nil, fmt.Errorf("CherryPick: Invalid last commit ID: %w", err) + } + opts.LastCommitID = lastCommitID.String() + if commit.ID.String() != opts.LastCommitID { + return nil, models.ErrCommitIDDoesNotMatch{ + GivenCommitID: opts.LastCommitID, + CurrentCommitID: opts.LastCommitID, + } + } + } + + commit, err = t.GetCommit(strings.TrimSpace(opts.Content)) + if err != nil { + return nil, err + } + parent, err := commit.ParentID(0) + if err != nil { + parent = git.ObjectFormatFromName(repo.ObjectFormatName).EmptyTree() + } + + base, right := parent.String(), commit.ID.String() + + if revert { + right, base = base, right + } + + description := fmt.Sprintf("CherryPick %s onto %s", right, opts.OldBranch) + conflict, _, err := pull.AttemptThreeWayMerge(ctx, + t.basePath, t.gitRepo, base, opts.LastCommitID, right, description) + if err != nil { + return nil, fmt.Errorf("failed to three-way merge %s onto %s: %w", right, opts.OldBranch, err) + } + + if conflict { + return nil, fmt.Errorf("failed to merge due to conflicts") + } + + treeHash, err := t.WriteTree() + if err != nil { + // likely non-sensical tree due to merge conflicts... + return nil, err + } + + // Now commit the tree + var commitHash string + if opts.Dates != nil { + commitHash, err = t.CommitTreeWithDate("HEAD", author, committer, treeHash, message, opts.Signoff, opts.Dates.Author, opts.Dates.Committer) + } else { + commitHash, err = t.CommitTree("HEAD", author, committer, treeHash, message, opts.Signoff) + } + if err != nil { + return nil, err + } + + // Then push this tree to NewBranch + if err := t.Push(doer, commitHash, opts.NewBranch); err != nil { + return nil, err + } + + commit, err = t.GetCommit(commitHash) + if err != nil { + return nil, err + } + + fileCommitResponse, _ := GetFileCommitResponse(repo, commit) // ok if fails, then will be nil + verification := GetPayloadCommitVerification(ctx, commit) + fileResponse := &structs.FileResponse{ + Commit: fileCommitResponse, + Verification: verification, + } + + return fileResponse, nil +} diff --git a/services/repository/files/commit.go b/services/repository/files/commit.go new file mode 100644 index 0000000..e0dad29 --- /dev/null +++ b/services/repository/files/commit.go @@ -0,0 +1,44 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package files + +import ( + "context" + + asymkey_model "code.gitea.io/gitea/models/asymkey" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/structs" +) + +// CountDivergingCommits determines how many commits a branch is ahead or behind the repository's base branch +func CountDivergingCommits(ctx context.Context, repo *repo_model.Repository, branch string) (*git.DivergeObject, error) { + divergence, err := git.GetDivergingCommits(ctx, repo.RepoPath(), repo.DefaultBranch, branch) + if err != nil { + return nil, err + } + return &divergence, nil +} + +// GetPayloadCommitVerification returns the verification information of a commit +func GetPayloadCommitVerification(ctx context.Context, commit *git.Commit) *structs.PayloadCommitVerification { + verification := &structs.PayloadCommitVerification{} + commitVerification := asymkey_model.ParseCommitWithSignature(ctx, commit) + if commit.Signature != nil { + verification.Signature = commit.Signature.Signature + verification.Payload = commit.Signature.Payload + } + if commitVerification.SigningUser != nil { + verification.Signer = &structs.PayloadUser{ + Name: commitVerification.SigningUser.Name, + Email: commitVerification.SigningUser.Email, + } + } + verification.Verified = commitVerification.Verified + verification.Reason = commitVerification.Reason + if verification.Reason == "" && !verification.Verified { + verification.Reason = "gpg.error.not_signed_commit" + } + return verification +} diff --git a/services/repository/files/content.go b/services/repository/files/content.go new file mode 100644 index 0000000..8dfd8b8 --- /dev/null +++ b/services/repository/files/content.go @@ -0,0 +1,278 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package files + +import ( + "context" + "fmt" + "net/url" + "path" + "strings" + + "code.gitea.io/gitea/models" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/gitrepo" + "code.gitea.io/gitea/modules/setting" + api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/util" +) + +// ContentType repo content type +type ContentType string + +// The string representations of different content types +const ( + // ContentTypeRegular regular content type (file) + ContentTypeRegular ContentType = "file" + // ContentTypeDir dir content type (dir) + ContentTypeDir ContentType = "dir" + // ContentLink link content type (symlink) + ContentTypeLink ContentType = "symlink" + // ContentTag submodule content type (submodule) + ContentTypeSubmodule ContentType = "submodule" +) + +// String gets the string of ContentType +func (ct *ContentType) String() string { + return string(*ct) +} + +// GetContentsOrList gets the meta data of a file's contents (*ContentsResponse) if treePath not a tree +// directory, otherwise a listing of file contents ([]*ContentsResponse). Ref can be a branch, commit or tag +func GetContentsOrList(ctx context.Context, repo *repo_model.Repository, treePath, ref string) (any, error) { + if repo.IsEmpty { + return make([]any, 0), nil + } + if ref == "" { + ref = repo.DefaultBranch + } + origRef := ref + + // Check that the path given in opts.treePath is valid (not a git path) + cleanTreePath := CleanUploadFileName(treePath) + if cleanTreePath == "" && treePath != "" { + return nil, models.ErrFilenameInvalid{ + Path: treePath, + } + } + treePath = cleanTreePath + + gitRepo, closer, err := gitrepo.RepositoryFromContextOrOpen(ctx, repo) + if err != nil { + return nil, err + } + defer closer.Close() + + // Get the commit object for the ref + commit, err := gitRepo.GetCommit(ref) + if err != nil { + return nil, err + } + + entry, err := commit.GetTreeEntryByPath(treePath) + if err != nil { + return nil, err + } + + if entry.Type() != "tree" { + return GetContents(ctx, repo, treePath, origRef, false) + } + + // We are in a directory, so we return a list of FileContentResponse objects + var fileList []*api.ContentsResponse + + gitTree, err := commit.SubTree(treePath) + if err != nil { + return nil, err + } + entries, err := gitTree.ListEntries() + if err != nil { + return nil, err + } + for _, e := range entries { + subTreePath := path.Join(treePath, e.Name()) + fileContentResponse, err := GetContents(ctx, repo, subTreePath, origRef, true) + if err != nil { + return nil, err + } + fileList = append(fileList, fileContentResponse) + } + return fileList, nil +} + +// GetObjectTypeFromTreeEntry check what content is behind it +func GetObjectTypeFromTreeEntry(entry *git.TreeEntry) ContentType { + switch { + case entry.IsDir(): + return ContentTypeDir + case entry.IsSubModule(): + return ContentTypeSubmodule + case entry.IsExecutable(), entry.IsRegular(): + return ContentTypeRegular + case entry.IsLink(): + return ContentTypeLink + default: + return "" + } +} + +// GetContents gets the meta data on a file's contents. Ref can be a branch, commit or tag +func GetContents(ctx context.Context, repo *repo_model.Repository, treePath, ref string, forList bool) (*api.ContentsResponse, error) { + if ref == "" { + ref = repo.DefaultBranch + } + origRef := ref + + // Check that the path given in opts.treePath is valid (not a git path) + cleanTreePath := CleanUploadFileName(treePath) + if cleanTreePath == "" && treePath != "" { + return nil, models.ErrFilenameInvalid{ + Path: treePath, + } + } + treePath = cleanTreePath + + gitRepo, closer, err := gitrepo.RepositoryFromContextOrOpen(ctx, repo) + if err != nil { + return nil, err + } + defer closer.Close() + + // Get the commit object for the ref + commit, err := gitRepo.GetCommit(ref) + if err != nil { + return nil, err + } + commitID := commit.ID.String() + if len(ref) >= 4 && strings.HasPrefix(commitID, ref) { + ref = commit.ID.String() + } + + entry, err := commit.GetTreeEntryByPath(treePath) + if err != nil { + return nil, err + } + + refType := gitRepo.GetRefType(ref) + if refType == "invalid" { + return nil, fmt.Errorf("no commit found for the ref [ref: %s]", ref) + } + + selfURL, err := url.Parse(repo.APIURL() + "/contents/" + util.PathEscapeSegments(treePath) + "?ref=" + url.QueryEscape(origRef)) + if err != nil { + return nil, err + } + selfURLString := selfURL.String() + + err = gitRepo.AddLastCommitCache(repo.GetCommitsCountCacheKey(ref, refType != git.ObjectCommit), repo.FullName(), commitID) + if err != nil { + return nil, err + } + + lastCommit, err := commit.GetCommitByPath(treePath) + if err != nil { + return nil, err + } + + // All content types have these fields in populated + contentsResponse := &api.ContentsResponse{ + Name: entry.Name(), + Path: treePath, + SHA: entry.ID.String(), + LastCommitSHA: lastCommit.ID.String(), + Size: entry.Size(), + URL: &selfURLString, + Links: &api.FileLinksResponse{ + Self: &selfURLString, + }, + } + + // Now populate the rest of the ContentsResponse based on entry type + if entry.IsRegular() || entry.IsExecutable() { + contentsResponse.Type = string(ContentTypeRegular) + if blobResponse, err := GetBlobBySHA(ctx, repo, gitRepo, entry.ID.String()); err != nil { + return nil, err + } else if !forList { + // We don't show the content if we are getting a list of FileContentResponses + contentsResponse.Encoding = &blobResponse.Encoding + contentsResponse.Content = &blobResponse.Content + } + } else if entry.IsDir() { + contentsResponse.Type = string(ContentTypeDir) + } else if entry.IsLink() { + contentsResponse.Type = string(ContentTypeLink) + // The target of a symlink file is the content of the file + targetFromContent, err := entry.Blob().GetBlobContent(1024) + if err != nil { + return nil, err + } + contentsResponse.Target = &targetFromContent + } else if entry.IsSubModule() { + contentsResponse.Type = string(ContentTypeSubmodule) + submodule, err := commit.GetSubModule(treePath) + if err != nil { + return nil, err + } + if submodule != nil && submodule.URL != "" { + contentsResponse.SubmoduleGitURL = &submodule.URL + } + } + // Handle links + if entry.IsRegular() || entry.IsLink() || entry.IsExecutable() { + downloadURL, err := url.Parse(repo.HTMLURL() + "/raw/" + url.PathEscape(string(refType)) + "/" + util.PathEscapeSegments(ref) + "/" + util.PathEscapeSegments(treePath)) + if err != nil { + return nil, err + } + downloadURLString := downloadURL.String() + contentsResponse.DownloadURL = &downloadURLString + } + if !entry.IsSubModule() { + htmlURL, err := url.Parse(repo.HTMLURL() + "/src/" + url.PathEscape(string(refType)) + "/" + util.PathEscapeSegments(ref) + "/" + util.PathEscapeSegments(treePath)) + if err != nil { + return nil, err + } + htmlURLString := htmlURL.String() + contentsResponse.HTMLURL = &htmlURLString + contentsResponse.Links.HTMLURL = &htmlURLString + + gitURL, err := url.Parse(repo.APIURL() + "/git/blobs/" + url.PathEscape(entry.ID.String())) + if err != nil { + return nil, err + } + gitURLString := gitURL.String() + contentsResponse.GitURL = &gitURLString + contentsResponse.Links.GitURL = &gitURLString + } + + return contentsResponse, nil +} + +// GetBlobBySHA get the GitBlobResponse of a repository using a sha hash. +func GetBlobBySHA(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, sha string) (*api.GitBlobResponse, error) { + gitBlob, err := gitRepo.GetBlob(sha) + if err != nil { + return nil, err + } + content := "" + if gitBlob.Size() <= setting.API.DefaultMaxBlobSize { + content, err = gitBlob.GetBlobContentBase64() + if err != nil { + return nil, err + } + } + return &api.GitBlobResponse{ + SHA: gitBlob.ID.String(), + URL: repo.APIURL() + "/git/blobs/" + url.PathEscape(gitBlob.ID.String()), + Size: gitBlob.Size(), + Encoding: "base64", + Content: content, + }, nil +} + +// TryGetContentLanguage tries to get the (linguist) language of the file content +func TryGetContentLanguage(gitRepo *git.Repository, commitID, treePath string) (string, error) { + attribute, err := gitRepo.GitAttributeFirst(commitID, treePath, "linguist-language", "gitlab-language") + return attribute.Prefix(), err +} diff --git a/services/repository/files/content_test.go b/services/repository/files/content_test.go new file mode 100644 index 0000000..c22dcd2 --- /dev/null +++ b/services/repository/files/content_test.go @@ -0,0 +1,201 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package files + +import ( + "testing" + + "code.gitea.io/gitea/models/db" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/models/unittest" + "code.gitea.io/gitea/modules/gitrepo" + api "code.gitea.io/gitea/modules/structs" + + _ "code.gitea.io/gitea/models/actions" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMain(m *testing.M) { + unittest.MainTest(m) +} + +func getExpectedReadmeContentsResponse() *api.ContentsResponse { + treePath := "README.md" + sha := "4b4851ad51df6a7d9f25c979345979eaeb5b349f" + encoding := "base64" + content := "IyByZXBvMQoKRGVzY3JpcHRpb24gZm9yIHJlcG8x" + selfURL := "https://try.gitea.io/api/v1/repos/user2/repo1/contents/" + treePath + "?ref=master" + htmlURL := "https://try.gitea.io/user2/repo1/src/branch/master/" + treePath + gitURL := "https://try.gitea.io/api/v1/repos/user2/repo1/git/blobs/" + sha + downloadURL := "https://try.gitea.io/user2/repo1/raw/branch/master/" + treePath + return &api.ContentsResponse{ + Name: treePath, + Path: treePath, + SHA: "4b4851ad51df6a7d9f25c979345979eaeb5b349f", + LastCommitSHA: "65f1bf27bc3bf70f64657658635e66094edbcb4d", + Type: "file", + Size: 30, + Encoding: &encoding, + Content: &content, + URL: &selfURL, + HTMLURL: &htmlURL, + GitURL: &gitURL, + DownloadURL: &downloadURL, + Links: &api.FileLinksResponse{ + Self: &selfURL, + GitURL: &gitURL, + HTMLURL: &htmlURL, + }, + } +} + +func TestGetContents(t *testing.T) { + unittest.PrepareTestEnv(t) + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) + + treePath := "README.md" + ref := repo.DefaultBranch + + expectedContentsResponse := getExpectedReadmeContentsResponse() + + t.Run("Get README.md contents with GetContents(ctx, )", func(t *testing.T) { + fileContentResponse, err := GetContents(db.DefaultContext, repo, treePath, ref, false) + assert.EqualValues(t, expectedContentsResponse, fileContentResponse) + require.NoError(t, err) + }) + + t.Run("Get README.md contents with ref as empty string (should then use the repo's default branch) with GetContents(ctx, )", func(t *testing.T) { + fileContentResponse, err := GetContents(db.DefaultContext, repo, treePath, "", false) + assert.EqualValues(t, expectedContentsResponse, fileContentResponse) + require.NoError(t, err) + }) +} + +func TestGetContentsOrListForDir(t *testing.T) { + unittest.PrepareTestEnv(t) + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) + + treePath := "" // root dir + ref := repo.DefaultBranch + + readmeContentsResponse := getExpectedReadmeContentsResponse() + // because will be in a list, doesn't have encoding and content + readmeContentsResponse.Encoding = nil + readmeContentsResponse.Content = nil + + expectedContentsListResponse := []*api.ContentsResponse{ + readmeContentsResponse, + } + + t.Run("Get root dir contents with GetContentsOrList(ctx, )", func(t *testing.T) { + fileContentResponse, err := GetContentsOrList(db.DefaultContext, repo, treePath, ref) + assert.EqualValues(t, expectedContentsListResponse, fileContentResponse) + require.NoError(t, err) + }) + + t.Run("Get root dir contents with ref as empty string (should then use the repo's default branch) with GetContentsOrList(ctx, )", func(t *testing.T) { + fileContentResponse, err := GetContentsOrList(db.DefaultContext, repo, treePath, "") + assert.EqualValues(t, expectedContentsListResponse, fileContentResponse) + require.NoError(t, err) + }) +} + +func TestGetContentsOrListForFile(t *testing.T) { + unittest.PrepareTestEnv(t) + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) + + treePath := "README.md" + ref := repo.DefaultBranch + + expectedContentsResponse := getExpectedReadmeContentsResponse() + + t.Run("Get README.md contents with GetContentsOrList(ctx, )", func(t *testing.T) { + fileContentResponse, err := GetContentsOrList(db.DefaultContext, repo, treePath, ref) + assert.EqualValues(t, expectedContentsResponse, fileContentResponse) + require.NoError(t, err) + }) + + t.Run("Get README.md contents with ref as empty string (should then use the repo's default branch) with GetContentsOrList(ctx, )", func(t *testing.T) { + fileContentResponse, err := GetContentsOrList(db.DefaultContext, repo, treePath, "") + assert.EqualValues(t, expectedContentsResponse, fileContentResponse) + require.NoError(t, err) + }) +} + +func TestGetContentsErrors(t *testing.T) { + unittest.PrepareTestEnv(t) + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) + + treePath := "README.md" + ref := repo.DefaultBranch + + t.Run("bad treePath", func(t *testing.T) { + badTreePath := "bad/tree.md" + fileContentResponse, err := GetContents(db.DefaultContext, repo, badTreePath, ref, false) + require.EqualError(t, err, "object does not exist [id: , rel_path: bad]") + assert.Nil(t, fileContentResponse) + }) + + t.Run("bad ref", func(t *testing.T) { + badRef := "bad_ref" + fileContentResponse, err := GetContents(db.DefaultContext, repo, treePath, badRef, false) + require.EqualError(t, err, "object does not exist [id: "+badRef+", rel_path: ]") + assert.Nil(t, fileContentResponse) + }) +} + +func TestGetContentsOrListErrors(t *testing.T) { + unittest.PrepareTestEnv(t) + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) + + treePath := "README.md" + ref := repo.DefaultBranch + + t.Run("bad treePath", func(t *testing.T) { + badTreePath := "bad/tree.md" + fileContentResponse, err := GetContentsOrList(db.DefaultContext, repo, badTreePath, ref) + require.EqualError(t, err, "object does not exist [id: , rel_path: bad]") + assert.Nil(t, fileContentResponse) + }) + + t.Run("bad ref", func(t *testing.T) { + badRef := "bad_ref" + fileContentResponse, err := GetContentsOrList(db.DefaultContext, repo, treePath, badRef) + require.EqualError(t, err, "object does not exist [id: "+badRef+", rel_path: ]") + assert.Nil(t, fileContentResponse) + }) +} + +func TestGetContentsOrListOfEmptyRepos(t *testing.T) { + unittest.PrepareTestEnv(t) + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 52}) + + t.Run("empty repo", func(t *testing.T) { + contents, err := GetContentsOrList(db.DefaultContext, repo, "", "") + require.NoError(t, err) + assert.Empty(t, contents) + }) +} + +func TestGetBlobBySHA(t *testing.T) { + unittest.PrepareTestEnv(t) + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) + + gitRepo, err := gitrepo.OpenRepository(db.DefaultContext, repo) + require.NoError(t, err) + defer gitRepo.Close() + + gbr, err := GetBlobBySHA(db.DefaultContext, repo, gitRepo, "65f1bf27bc3bf70f64657658635e66094edbcb4d") + expectedGBR := &api.GitBlobResponse{ + Content: "dHJlZSAyYTJmMWQ0NjcwNzI4YTJlMTAwNDllMzQ1YmQ3YTI3NjQ2OGJlYWI2CmF1dGhvciB1c2VyMSA8YWRkcmVzczFAZXhhbXBsZS5jb20+IDE0ODk5NTY0NzkgLTA0MDAKY29tbWl0dGVyIEV0aGFuIEtvZW5pZyA8ZXRoYW50a29lbmlnQGdtYWlsLmNvbT4gMTQ4OTk1NjQ3OSAtMDQwMAoKSW5pdGlhbCBjb21taXQK", + Encoding: "base64", + URL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/blobs/65f1bf27bc3bf70f64657658635e66094edbcb4d", + SHA: "65f1bf27bc3bf70f64657658635e66094edbcb4d", + Size: 180, + } + require.NoError(t, err) + assert.Equal(t, expectedGBR, gbr) +} diff --git a/services/repository/files/diff.go b/services/repository/files/diff.go new file mode 100644 index 0000000..bf8b938 --- /dev/null +++ b/services/repository/files/diff.go @@ -0,0 +1,42 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package files + +import ( + "context" + "strings" + + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/services/gitdiff" +) + +// GetDiffPreview produces and returns diff result of a file which is not yet committed. +func GetDiffPreview(ctx context.Context, repo *repo_model.Repository, branch, treePath, content string) (*gitdiff.Diff, error) { + if branch == "" { + branch = repo.DefaultBranch + } + t, err := NewTemporaryUploadRepository(ctx, repo) + if err != nil { + return nil, err + } + defer t.Close() + if err := t.Clone(branch, true); err != nil { + return nil, err + } + if err := t.SetDefaultIndex(); err != nil { + return nil, err + } + + // Add the object to the database + objectHash, err := t.HashObject(strings.NewReader(content)) + if err != nil { + return nil, err + } + + // Add the object to the index + if err := t.AddObjectToIndex("100644", objectHash, treePath); err != nil { + return nil, err + } + return t.DiffIndex() +} diff --git a/services/repository/files/diff_test.go b/services/repository/files/diff_test.go new file mode 100644 index 0000000..95de10e --- /dev/null +++ b/services/repository/files/diff_test.go @@ -0,0 +1,166 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package files + +import ( + "testing" + + "code.gitea.io/gitea/models/db" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/models/unittest" + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/services/contexttest" + "code.gitea.io/gitea/services/gitdiff" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGetDiffPreview(t *testing.T) { + unittest.PrepareTestEnv(t) + ctx, _ := contexttest.MockContext(t, "user2/repo1") + ctx.SetParams(":id", "1") + contexttest.LoadRepo(t, ctx, 1) + contexttest.LoadUser(t, ctx, 2) + contexttest.LoadGitRepo(t, ctx) + defer ctx.Repo.GitRepo.Close() + + branch := ctx.Repo.Repository.DefaultBranch + treePath := "README.md" + content := "# repo1\n\nDescription for repo1\nthis is a new line" + + expectedDiff := &gitdiff.Diff{ + TotalAddition: 2, + TotalDeletion: 1, + Files: []*gitdiff.DiffFile{ + { + Name: "README.md", + OldName: "README.md", + NameHash: "8ec9a00bfd09b3190ac6b22251dbb1aa95a0579d", + Index: 1, + Addition: 2, + Deletion: 1, + Type: 2, + IsCreated: false, + IsDeleted: false, + IsBin: false, + IsLFSFile: false, + IsRenamed: false, + IsSubmodule: false, + Sections: []*gitdiff.DiffSection{ + { + FileName: "README.md", + Name: "", + Lines: []*gitdiff.DiffLine{ + { + LeftIdx: 0, + RightIdx: 0, + Type: 4, + Content: "@@ -1,3 +1,4 @@", + Conversations: nil, + SectionInfo: &gitdiff.DiffLineSectionInfo{ + Path: "README.md", + LastLeftIdx: 0, + LastRightIdx: 0, + LeftIdx: 1, + RightIdx: 1, + LeftHunkSize: 3, + RightHunkSize: 4, + }, + }, + { + LeftIdx: 1, + RightIdx: 1, + Type: 1, + Content: " # repo1", + Conversations: nil, + }, + { + LeftIdx: 2, + RightIdx: 2, + Type: 1, + Content: " ", + Conversations: nil, + }, + { + LeftIdx: 3, + RightIdx: 0, + Match: 4, + Type: 3, + Content: "-Description for repo1", + Conversations: nil, + }, + { + LeftIdx: 0, + RightIdx: 3, + Match: 3, + Type: 2, + Content: "+Description for repo1", + Conversations: nil, + }, + { + LeftIdx: 0, + RightIdx: 4, + Match: -1, + Type: 2, + Content: "+this is a new line", + Conversations: nil, + }, + }, + }, + }, + IsIncomplete: false, + }, + }, + IsIncomplete: false, + } + expectedDiff.NumFiles = len(expectedDiff.Files) + + t.Run("with given branch", func(t *testing.T) { + diff, err := GetDiffPreview(ctx, ctx.Repo.Repository, branch, treePath, content) + require.NoError(t, err) + expectedBs, err := json.Marshal(expectedDiff) + require.NoError(t, err) + bs, err := json.Marshal(diff) + require.NoError(t, err) + assert.EqualValues(t, string(expectedBs), string(bs)) + }) + + t.Run("empty branch, same results", func(t *testing.T) { + diff, err := GetDiffPreview(ctx, ctx.Repo.Repository, "", treePath, content) + require.NoError(t, err) + expectedBs, err := json.Marshal(expectedDiff) + require.NoError(t, err) + bs, err := json.Marshal(diff) + require.NoError(t, err) + assert.EqualValues(t, expectedBs, bs) + }) +} + +func TestGetDiffPreviewErrors(t *testing.T) { + unittest.PrepareTestEnv(t) + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) + branch := repo.DefaultBranch + treePath := "README.md" + content := "# repo1\n\nDescription for repo1\nthis is a new line" + + t.Run("empty repo", func(t *testing.T) { + diff, err := GetDiffPreview(db.DefaultContext, &repo_model.Repository{}, branch, treePath, content) + assert.Nil(t, diff) + assert.EqualError(t, err, "repository does not exist [id: 0, uid: 0, owner_name: , name: ]") + }) + + t.Run("bad branch", func(t *testing.T) { + badBranch := "bad_branch" + diff, err := GetDiffPreview(db.DefaultContext, repo, badBranch, treePath, content) + assert.Nil(t, diff) + assert.EqualError(t, err, "branch does not exist [name: "+badBranch+"]") + }) + + t.Run("empty treePath", func(t *testing.T) { + diff, err := GetDiffPreview(db.DefaultContext, repo, branch, "", content) + assert.Nil(t, diff) + assert.EqualError(t, err, "path is invalid [path: ]") + }) +} diff --git a/services/repository/files/file.go b/services/repository/files/file.go new file mode 100644 index 0000000..852cca0 --- /dev/null +++ b/services/repository/files/file.go @@ -0,0 +1,174 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package files + +import ( + "context" + "fmt" + "net/url" + "strings" + "time" + + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" + api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/util" +) + +func GetFilesResponseFromCommit(ctx context.Context, repo *repo_model.Repository, commit *git.Commit, branch string, treeNames []string) (*api.FilesResponse, error) { + files := []*api.ContentsResponse{} + for _, file := range treeNames { + fileContents, _ := GetContents(ctx, repo, file, branch, false) // ok if fails, then will be nil + files = append(files, fileContents) + } + fileCommitResponse, _ := GetFileCommitResponse(repo, commit) // ok if fails, then will be nil + verification := GetPayloadCommitVerification(ctx, commit) + filesResponse := &api.FilesResponse{ + Files: files, + Commit: fileCommitResponse, + Verification: verification, + } + return filesResponse, nil +} + +// GetFileResponseFromCommit Constructs a FileResponse from a Commit object +func GetFileResponseFromCommit(ctx context.Context, repo *repo_model.Repository, commit *git.Commit, branch, treeName string) (*api.FileResponse, error) { + fileContents, _ := GetContents(ctx, repo, treeName, branch, false) // ok if fails, then will be nil + fileCommitResponse, _ := GetFileCommitResponse(repo, commit) // ok if fails, then will be nil + verification := GetPayloadCommitVerification(ctx, commit) + fileResponse := &api.FileResponse{ + Content: fileContents, + Commit: fileCommitResponse, + Verification: verification, + } + return fileResponse, nil +} + +// constructs a FileResponse with the file at the index from FilesResponse +func GetFileResponseFromFilesResponse(filesResponse *api.FilesResponse, index int) *api.FileResponse { + content := &api.ContentsResponse{} + if len(filesResponse.Files) > index { + content = filesResponse.Files[index] + } + fileResponse := &api.FileResponse{ + Content: content, + Commit: filesResponse.Commit, + Verification: filesResponse.Verification, + } + return fileResponse +} + +// GetFileCommitResponse Constructs a FileCommitResponse from a Commit object +func GetFileCommitResponse(repo *repo_model.Repository, commit *git.Commit) (*api.FileCommitResponse, error) { + if repo == nil { + return nil, fmt.Errorf("repo cannot be nil") + } + if commit == nil { + return nil, fmt.Errorf("commit cannot be nil") + } + commitURL, _ := url.Parse(repo.APIURL() + "/git/commits/" + url.PathEscape(commit.ID.String())) + commitTreeURL, _ := url.Parse(repo.APIURL() + "/git/trees/" + url.PathEscape(commit.Tree.ID.String())) + parents := make([]*api.CommitMeta, commit.ParentCount()) + for i := 0; i <= commit.ParentCount(); i++ { + if parent, err := commit.Parent(i); err == nil && parent != nil { + parentCommitURL, _ := url.Parse(repo.APIURL() + "/git/commits/" + url.PathEscape(parent.ID.String())) + parents[i] = &api.CommitMeta{ + SHA: parent.ID.String(), + URL: parentCommitURL.String(), + } + } + } + commitHTMLURL, _ := url.Parse(repo.HTMLURL() + "/commit/" + url.PathEscape(commit.ID.String())) + fileCommit := &api.FileCommitResponse{ + CommitMeta: api.CommitMeta{ + SHA: commit.ID.String(), + URL: commitURL.String(), + }, + HTMLURL: commitHTMLURL.String(), + Author: &api.CommitUser{ + Identity: api.Identity{ + Name: commit.Author.Name, + Email: commit.Author.Email, + }, + Date: commit.Author.When.UTC().Format(time.RFC3339), + }, + Committer: &api.CommitUser{ + Identity: api.Identity{ + Name: commit.Committer.Name, + Email: commit.Committer.Email, + }, + Date: commit.Committer.When.UTC().Format(time.RFC3339), + }, + Message: commit.Message(), + Tree: &api.CommitMeta{ + URL: commitTreeURL.String(), + SHA: commit.Tree.ID.String(), + }, + Parents: parents, + } + return fileCommit, nil +} + +// GetAuthorAndCommitterUsers Gets the author and committer user objects from the IdentityOptions +func GetAuthorAndCommitterUsers(author, committer *IdentityOptions, doer *user_model.User) (authorUser, committerUser *user_model.User) { + // Committer and author are optional. If they are not the doer (not same email address) + // then we use bogus User objects for them to store their FullName and Email. + // If only one of the two are provided, we set both of them to it. + // If neither are provided, both are the doer. + if committer != nil && committer.Email != "" { + if doer != nil && strings.EqualFold(doer.Email, committer.Email) { + committerUser = doer // the committer is the doer, so will use their user object + if committer.Name != "" { + committerUser.FullName = committer.Name + } + // Use the provided email and not revert to placeholder mail. + committerUser.KeepEmailPrivate = false + } else { + committerUser = &user_model.User{ + FullName: committer.Name, + Email: committer.Email, + } + } + } + if author != nil && author.Email != "" { + if doer != nil && strings.EqualFold(doer.Email, author.Email) { + authorUser = doer // the author is the doer, so will use their user object + if authorUser.Name != "" { + authorUser.FullName = author.Name + } + // Use the provided email and not revert to placeholder mail. + authorUser.KeepEmailPrivate = false + } else { + authorUser = &user_model.User{ + FullName: author.Name, + Email: author.Email, + } + } + } + if authorUser == nil { + if committerUser != nil { + authorUser = committerUser // No valid author was given so use the committer + } else if doer != nil { + authorUser = doer // No valid author was given and no valid committer so use the doer + } + } + if committerUser == nil { + committerUser = authorUser // No valid committer so use the author as the committer (was set to a valid user above) + } + return authorUser, committerUser +} + +// CleanUploadFileName Trims a filename and returns empty string if it is a .git directory +func CleanUploadFileName(name string) string { + // Rebase the filename + name = util.PathJoinRel(name) + // Git disallows any filenames to have a .git directory in them. + for _, part := range strings.Split(name, "/") { + if strings.ToLower(part) == ".git" { + return "" + } + } + return name +} diff --git a/services/repository/files/file_test.go b/services/repository/files/file_test.go new file mode 100644 index 0000000..7c387e2 --- /dev/null +++ b/services/repository/files/file_test.go @@ -0,0 +1,115 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package files + +import ( + "testing" + + "code.gitea.io/gitea/models/db" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/models/unittest" + "code.gitea.io/gitea/modules/gitrepo" + "code.gitea.io/gitea/modules/setting" + api "code.gitea.io/gitea/modules/structs" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCleanUploadFileName(t *testing.T) { + t.Run("Clean regular file", func(t *testing.T) { + name := "this/is/test" + cleanName := CleanUploadFileName(name) + expectedCleanName := name + assert.EqualValues(t, expectedCleanName, cleanName) + }) + + t.Run("Clean a .git path", func(t *testing.T) { + name := "this/is/test/.git" + cleanName := CleanUploadFileName(name) + expectedCleanName := "" + assert.EqualValues(t, expectedCleanName, cleanName) + }) +} + +func getExpectedFileResponse() *api.FileResponse { + treePath := "README.md" + sha := "4b4851ad51df6a7d9f25c979345979eaeb5b349f" + encoding := "base64" + content := "IyByZXBvMQoKRGVzY3JpcHRpb24gZm9yIHJlcG8x" + selfURL := setting.AppURL + "api/v1/repos/user2/repo1/contents/" + treePath + "?ref=master" + htmlURL := setting.AppURL + "user2/repo1/src/branch/master/" + treePath + gitURL := setting.AppURL + "api/v1/repos/user2/repo1/git/blobs/" + sha + downloadURL := setting.AppURL + "user2/repo1/raw/branch/master/" + treePath + return &api.FileResponse{ + Content: &api.ContentsResponse{ + Name: treePath, + Path: treePath, + SHA: sha, + LastCommitSHA: "65f1bf27bc3bf70f64657658635e66094edbcb4d", + Type: "file", + Size: 30, + Encoding: &encoding, + Content: &content, + URL: &selfURL, + HTMLURL: &htmlURL, + GitURL: &gitURL, + DownloadURL: &downloadURL, + Links: &api.FileLinksResponse{ + Self: &selfURL, + GitURL: &gitURL, + HTMLURL: &htmlURL, + }, + }, + Commit: &api.FileCommitResponse{ + CommitMeta: api.CommitMeta{ + URL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/commits/65f1bf27bc3bf70f64657658635e66094edbcb4d", + SHA: "65f1bf27bc3bf70f64657658635e66094edbcb4d", + }, + HTMLURL: "https://try.gitea.io/user2/repo1/commit/65f1bf27bc3bf70f64657658635e66094edbcb4d", + Author: &api.CommitUser{ + Identity: api.Identity{ + Name: "user1", + Email: "address1@example.com", + }, + Date: "2017-03-19T20:47:59Z", + }, + Committer: &api.CommitUser{ + Identity: api.Identity{ + Name: "Ethan Koenig", + Email: "ethantkoenig@gmail.com", + }, + Date: "2017-03-19T20:47:59Z", + }, + Parents: []*api.CommitMeta{}, + Message: "Initial commit\n", + Tree: &api.CommitMeta{ + URL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/trees/2a2f1d4670728a2e10049e345bd7a276468beab6", + SHA: "2a2f1d4670728a2e10049e345bd7a276468beab6", + }, + }, + Verification: &api.PayloadCommitVerification{ + Verified: false, + Reason: "gpg.error.not_signed_commit", + Signature: "", + Payload: "", + }, + } +} + +func TestGetFileResponseFromCommit(t *testing.T) { + unittest.PrepareTestEnv(t) + + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) + branch := repo.DefaultBranch + treePath := "README.md" + gitRepo, _ := gitrepo.OpenRepository(db.DefaultContext, repo) + defer gitRepo.Close() + commit, _ := gitRepo.GetBranchCommit(branch) + expectedFileResponse := getExpectedFileResponse() + + fileResponse, err := GetFileResponseFromCommit(db.DefaultContext, repo, commit, branch, treePath) + require.NoError(t, err) + assert.EqualValues(t, expectedFileResponse, fileResponse) +} diff --git a/services/repository/files/patch.go b/services/repository/files/patch.go new file mode 100644 index 0000000..e5f7e2a --- /dev/null +++ b/services/repository/files/patch.go @@ -0,0 +1,199 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package files + +import ( + "context" + "fmt" + "strings" + + "code.gitea.io/gitea/models" + git_model "code.gitea.io/gitea/models/git" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/gitrepo" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/structs" + asymkey_service "code.gitea.io/gitea/services/asymkey" +) + +// ApplyDiffPatchOptions holds the repository diff patch update options +type ApplyDiffPatchOptions struct { + LastCommitID string + OldBranch string + NewBranch string + Message string + Content string + SHA string + Author *IdentityOptions + Committer *IdentityOptions + Dates *CommitDateOptions + Signoff bool +} + +// Validate validates the provided options +func (opts *ApplyDiffPatchOptions) Validate(ctx context.Context, repo *repo_model.Repository, doer *user_model.User) error { + // If no branch name is set, assume master + if opts.OldBranch == "" { + opts.OldBranch = repo.DefaultBranch + } + if opts.NewBranch == "" { + opts.NewBranch = opts.OldBranch + } + + gitRepo, closer, err := gitrepo.RepositoryFromContextOrOpen(ctx, repo) + if err != nil { + return err + } + defer closer.Close() + + // oldBranch must exist for this operation + if _, err := gitRepo.GetBranch(opts.OldBranch); err != nil { + return err + } + // A NewBranch can be specified for the patch to be applied to. + // Check to make sure the branch does not already exist, otherwise we can't proceed. + // If we aren't branching to a new branch, make sure user can commit to the given branch + if opts.NewBranch != opts.OldBranch { + existingBranch, err := gitRepo.GetBranch(opts.NewBranch) + if existingBranch != nil { + return git_model.ErrBranchAlreadyExists{ + BranchName: opts.NewBranch, + } + } + if err != nil && !git.IsErrBranchNotExist(err) { + return err + } + } else { + protectedBranch, err := git_model.GetFirstMatchProtectedBranchRule(ctx, repo.ID, opts.OldBranch) + if err != nil { + return err + } + if protectedBranch != nil { + protectedBranch.Repo = repo + if !protectedBranch.CanUserPush(ctx, doer) { + return models.ErrUserCannotCommit{ + UserName: doer.LowerName, + } + } + } + if protectedBranch != nil && protectedBranch.RequireSignedCommits { + _, _, _, err := asymkey_service.SignCRUDAction(ctx, repo.RepoPath(), doer, repo.RepoPath(), opts.OldBranch) + if err != nil { + if !asymkey_service.IsErrWontSign(err) { + return err + } + return models.ErrUserCannotCommit{ + UserName: doer.LowerName, + } + } + } + } + return nil +} + +// ApplyDiffPatch applies a patch to the given repository +func ApplyDiffPatch(ctx context.Context, repo *repo_model.Repository, doer *user_model.User, opts *ApplyDiffPatchOptions) (*structs.FileResponse, error) { + err := repo.MustNotBeArchived() + if err != nil { + return nil, err + } + + if err := opts.Validate(ctx, repo, doer); err != nil { + return nil, err + } + + message := strings.TrimSpace(opts.Message) + + author, committer := GetAuthorAndCommitterUsers(opts.Author, opts.Committer, doer) + + t, err := NewTemporaryUploadRepository(ctx, repo) + if err != nil { + log.Error("NewTemporaryUploadRepository failed: %v", err) + } + defer t.Close() + if err := t.Clone(opts.OldBranch, true); err != nil { + return nil, err + } + if err := t.SetDefaultIndex(); err != nil { + return nil, err + } + + // Get the commit of the original branch + commit, err := t.GetBranchCommit(opts.OldBranch) + if err != nil { + return nil, err // Couldn't get a commit for the branch + } + + // Assigned LastCommitID in opts if it hasn't been set + if opts.LastCommitID == "" { + opts.LastCommitID = commit.ID.String() + } else { + lastCommitID, err := t.gitRepo.ConvertToGitID(opts.LastCommitID) + if err != nil { + return nil, fmt.Errorf("ApplyPatch: Invalid last commit ID: %w", err) + } + opts.LastCommitID = lastCommitID.String() + if commit.ID.String() != opts.LastCommitID { + return nil, models.ErrCommitIDDoesNotMatch{ + GivenCommitID: opts.LastCommitID, + CurrentCommitID: opts.LastCommitID, + } + } + } + + stdout := &strings.Builder{} + stderr := &strings.Builder{} + + cmdApply := git.NewCommand(ctx, "apply", "--index", "--recount", "--cached", "--ignore-whitespace", "--whitespace=fix", "--binary") + if git.CheckGitVersionAtLeast("2.32") == nil { + cmdApply.AddArguments("-3") + } + + if err := cmdApply.Run(&git.RunOpts{ + Dir: t.basePath, + Stdout: stdout, + Stderr: stderr, + Stdin: strings.NewReader(opts.Content), + }); err != nil { + return nil, fmt.Errorf("Error: Stdout: %s\nStderr: %s\nErr: %w", stdout.String(), stderr.String(), err) + } + + // Now write the tree + treeHash, err := t.WriteTree() + if err != nil { + return nil, err + } + + // Now commit the tree + var commitHash string + if opts.Dates != nil { + commitHash, err = t.CommitTreeWithDate("HEAD", author, committer, treeHash, message, opts.Signoff, opts.Dates.Author, opts.Dates.Committer) + } else { + commitHash, err = t.CommitTree("HEAD", author, committer, treeHash, message, opts.Signoff) + } + if err != nil { + return nil, err + } + + // Then push this tree to NewBranch + if err := t.Push(doer, commitHash, opts.NewBranch); err != nil { + return nil, err + } + + commit, err = t.GetCommit(commitHash) + if err != nil { + return nil, err + } + + fileCommitResponse, _ := GetFileCommitResponse(repo, commit) // ok if fails, then will be nil + verification := GetPayloadCommitVerification(ctx, commit) + fileResponse := &structs.FileResponse{ + Commit: fileCommitResponse, + Verification: verification, + } + + return fileResponse, nil +} diff --git a/services/repository/files/temp_repo.go b/services/repository/files/temp_repo.go new file mode 100644 index 0000000..50b936c --- /dev/null +++ b/services/repository/files/temp_repo.go @@ -0,0 +1,406 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package files + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "regexp" + "strings" + "time" + + "code.gitea.io/gitea/models" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" + repo_module "code.gitea.io/gitea/modules/repository" + "code.gitea.io/gitea/modules/setting" + asymkey_service "code.gitea.io/gitea/services/asymkey" + "code.gitea.io/gitea/services/gitdiff" +) + +// TemporaryUploadRepository is a type to wrap our upload repositories as a shallow clone +type TemporaryUploadRepository struct { + ctx context.Context + repo *repo_model.Repository + gitRepo *git.Repository + basePath string +} + +// NewTemporaryUploadRepository creates a new temporary upload repository +func NewTemporaryUploadRepository(ctx context.Context, repo *repo_model.Repository) (*TemporaryUploadRepository, error) { + basePath, err := repo_module.CreateTemporaryPath("upload") + if err != nil { + return nil, err + } + t := &TemporaryUploadRepository{ctx: ctx, repo: repo, basePath: basePath} + return t, nil +} + +// Close the repository cleaning up all files +func (t *TemporaryUploadRepository) Close() { + defer t.gitRepo.Close() + if err := repo_module.RemoveTemporaryPath(t.basePath); err != nil { + log.Error("Failed to remove temporary path %s: %v", t.basePath, err) + } +} + +// Clone the base repository to our path and set branch as the HEAD +func (t *TemporaryUploadRepository) Clone(branch string, bare bool) error { + cmd := git.NewCommand(t.ctx, "clone", "-s", "-b").AddDynamicArguments(branch, t.repo.RepoPath(), t.basePath) + if bare { + cmd.AddArguments("--bare") + } + + if _, _, err := cmd.RunStdString(nil); err != nil { + stderr := err.Error() + if matched, _ := regexp.MatchString(".*Remote branch .* not found in upstream origin.*", stderr); matched { + return git.ErrBranchNotExist{ + Name: branch, + } + } else if matched, _ := regexp.MatchString(".* repository .* does not exist.*", stderr); matched { + return repo_model.ErrRepoNotExist{ + ID: t.repo.ID, + UID: t.repo.OwnerID, + OwnerName: t.repo.OwnerName, + Name: t.repo.Name, + } + } + return fmt.Errorf("Clone: %w %s", err, stderr) + } + gitRepo, err := git.OpenRepository(t.ctx, t.basePath) + if err != nil { + return err + } + t.gitRepo = gitRepo + return nil +} + +// Init the repository +func (t *TemporaryUploadRepository) Init(objectFormatName string) error { + if err := git.InitRepository(t.ctx, t.basePath, false, objectFormatName); err != nil { + return err + } + gitRepo, err := git.OpenRepository(t.ctx, t.basePath) + if err != nil { + return err + } + t.gitRepo = gitRepo + return nil +} + +// SetDefaultIndex sets the git index to our HEAD +func (t *TemporaryUploadRepository) SetDefaultIndex() error { + if _, _, err := git.NewCommand(t.ctx, "read-tree", "HEAD").RunStdString(&git.RunOpts{Dir: t.basePath}); err != nil { + return fmt.Errorf("SetDefaultIndex: %w", err) + } + return nil +} + +// RefreshIndex looks at the current index and checks to see if merges or updates are needed by checking stat() information. +func (t *TemporaryUploadRepository) RefreshIndex() error { + if _, _, err := git.NewCommand(t.ctx, "update-index", "--refresh").RunStdString(&git.RunOpts{Dir: t.basePath}); err != nil { + return fmt.Errorf("RefreshIndex: %w", err) + } + return nil +} + +// LsFiles checks if the given filename arguments are in the index +func (t *TemporaryUploadRepository) LsFiles(filenames ...string) ([]string, error) { + stdOut := new(bytes.Buffer) + stdErr := new(bytes.Buffer) + + if err := git.NewCommand(t.ctx, "ls-files", "-z").AddDashesAndList(filenames...). + Run(&git.RunOpts{ + Dir: t.basePath, + Stdout: stdOut, + Stderr: stdErr, + }); err != nil { + log.Error("Unable to run git ls-files for temporary repo: %s (%s) Error: %v\nstdout: %s\nstderr: %s", t.repo.FullName(), t.basePath, err, stdOut.String(), stdErr.String()) + err = fmt.Errorf("Unable to run git ls-files for temporary repo of: %s Error: %w\nstdout: %s\nstderr: %s", t.repo.FullName(), err, stdOut.String(), stdErr.String()) + return nil, err + } + + fileList := make([]string, 0, len(filenames)) + for _, line := range bytes.Split(stdOut.Bytes(), []byte{'\000'}) { + fileList = append(fileList, string(line)) + } + + return fileList, nil +} + +// RemoveFilesFromIndex removes the given files from the index +func (t *TemporaryUploadRepository) RemoveFilesFromIndex(filenames ...string) error { + objectFormat, err := t.gitRepo.GetObjectFormat() + if err != nil { + return err + } + + stdOut := new(bytes.Buffer) + stdErr := new(bytes.Buffer) + stdIn := new(bytes.Buffer) + for _, file := range filenames { + if file != "" { + stdIn.WriteString("0 ") + stdIn.WriteString(objectFormat.EmptyObjectID().String()) + stdIn.WriteByte('\t') + stdIn.WriteString(file) + stdIn.WriteByte('\000') + } + } + + if err := git.NewCommand(t.ctx, "update-index", "--remove", "-z", "--index-info"). + Run(&git.RunOpts{ + Dir: t.basePath, + Stdin: stdIn, + Stdout: stdOut, + Stderr: stdErr, + }); err != nil { + log.Error("Unable to update-index for temporary repo: %s (%s) Error: %v\nstdout: %s\nstderr: %s", t.repo.FullName(), t.basePath, err, stdOut.String(), stdErr.String()) + return fmt.Errorf("Unable to update-index for temporary repo: %s Error: %w\nstdout: %s\nstderr: %s", t.repo.FullName(), err, stdOut.String(), stdErr.String()) + } + return nil +} + +// HashObject writes the provided content to the object db and returns its hash +func (t *TemporaryUploadRepository) HashObject(content io.Reader) (string, error) { + stdOut := new(bytes.Buffer) + stdErr := new(bytes.Buffer) + + if err := git.NewCommand(t.ctx, "hash-object", "-w", "--stdin"). + Run(&git.RunOpts{ + Dir: t.basePath, + Stdin: content, + Stdout: stdOut, + Stderr: stdErr, + }); err != nil { + log.Error("Unable to hash-object to temporary repo: %s (%s) Error: %v\nstdout: %s\nstderr: %s", t.repo.FullName(), t.basePath, err, stdOut.String(), stdErr.String()) + return "", fmt.Errorf("Unable to hash-object to temporary repo: %s Error: %w\nstdout: %s\nstderr: %s", t.repo.FullName(), err, stdOut.String(), stdErr.String()) + } + + return strings.TrimSpace(stdOut.String()), nil +} + +// AddObjectToIndex adds the provided object hash to the index with the provided mode and path +func (t *TemporaryUploadRepository) AddObjectToIndex(mode, objectHash, objectPath string) error { + if _, _, err := git.NewCommand(t.ctx, "update-index", "--add", "--replace", "--cacheinfo").AddDynamicArguments(mode, objectHash, objectPath).RunStdString(&git.RunOpts{Dir: t.basePath}); err != nil { + stderr := err.Error() + if matched, _ := regexp.MatchString(".*Invalid path '.*", stderr); matched { + return models.ErrFilePathInvalid{ + Message: objectPath, + Path: objectPath, + } + } + log.Error("Unable to add object to index: %s %s %s in temporary repo %s(%s) Error: %v", mode, objectHash, objectPath, t.repo.FullName(), t.basePath, err) + return fmt.Errorf("Unable to add object to index at %s in temporary repo %s Error: %w", objectPath, t.repo.FullName(), err) + } + return nil +} + +// WriteTree writes the current index as a tree to the object db and returns its hash +func (t *TemporaryUploadRepository) WriteTree() (string, error) { + stdout, _, err := git.NewCommand(t.ctx, "write-tree").RunStdString(&git.RunOpts{Dir: t.basePath}) + if err != nil { + log.Error("Unable to write tree in temporary repo: %s(%s): Error: %v", t.repo.FullName(), t.basePath, err) + return "", fmt.Errorf("Unable to write-tree in temporary repo for: %s Error: %w", t.repo.FullName(), err) + } + return strings.TrimSpace(stdout), nil +} + +// GetLastCommit gets the last commit ID SHA of the repo +func (t *TemporaryUploadRepository) GetLastCommit() (string, error) { + return t.GetLastCommitByRef("HEAD") +} + +// GetLastCommitByRef gets the last commit ID SHA of the repo by ref +func (t *TemporaryUploadRepository) GetLastCommitByRef(ref string) (string, error) { + if ref == "" { + ref = "HEAD" + } + stdout, _, err := git.NewCommand(t.ctx, "rev-parse").AddDynamicArguments(ref).RunStdString(&git.RunOpts{Dir: t.basePath}) + if err != nil { + log.Error("Unable to get last ref for %s in temporary repo: %s(%s): Error: %v", ref, t.repo.FullName(), t.basePath, err) + return "", fmt.Errorf("Unable to rev-parse %s in temporary repo for: %s Error: %w", ref, t.repo.FullName(), err) + } + return strings.TrimSpace(stdout), nil +} + +// CommitTree creates a commit from a given tree for the user with provided message +func (t *TemporaryUploadRepository) CommitTree(parent string, author, committer *user_model.User, treeHash, message string, signoff bool) (string, error) { + return t.CommitTreeWithDate(parent, author, committer, treeHash, message, signoff, time.Now(), time.Now()) +} + +// CommitTreeWithDate creates a commit from a given tree for the user with provided message +func (t *TemporaryUploadRepository) CommitTreeWithDate(parent string, author, committer *user_model.User, treeHash, message string, signoff bool, authorDate, committerDate time.Time) (string, error) { + authorSig := author.NewGitSig() + committerSig := committer.NewGitSig() + + // Because this may call hooks we should pass in the environment + env := append(os.Environ(), + "GIT_AUTHOR_NAME="+authorSig.Name, + "GIT_AUTHOR_EMAIL="+authorSig.Email, + "GIT_AUTHOR_DATE="+authorDate.Format(time.RFC3339), + "GIT_COMMITTER_DATE="+committerDate.Format(time.RFC3339), + ) + + messageBytes := new(bytes.Buffer) + _, _ = messageBytes.WriteString(message) + _, _ = messageBytes.WriteString("\n") + + cmdCommitTree := git.NewCommand(t.ctx, "commit-tree").AddDynamicArguments(treeHash) + if parent != "" { + cmdCommitTree.AddOptionValues("-p", parent) + } + + var sign bool + var keyID string + var signer *git.Signature + if parent != "" { + sign, keyID, signer, _ = asymkey_service.SignCRUDAction(t.ctx, t.repo.RepoPath(), author, t.basePath, parent) + } else { + sign, keyID, signer, _ = asymkey_service.SignInitialCommit(t.ctx, t.repo.RepoPath(), author) + } + if sign { + cmdCommitTree.AddOptionFormat("-S%s", keyID) + if t.repo.GetTrustModel() == repo_model.CommitterTrustModel || t.repo.GetTrustModel() == repo_model.CollaboratorCommitterTrustModel { + if committerSig.Name != authorSig.Name || committerSig.Email != authorSig.Email { + // Add trailers + _, _ = messageBytes.WriteString("\n") + _, _ = messageBytes.WriteString("Co-authored-by: ") + _, _ = messageBytes.WriteString(committerSig.String()) + _, _ = messageBytes.WriteString("\n") + _, _ = messageBytes.WriteString("Co-committed-by: ") + _, _ = messageBytes.WriteString(committerSig.String()) + _, _ = messageBytes.WriteString("\n") + } + committerSig = signer + } + } else { + cmdCommitTree.AddArguments("--no-gpg-sign") + } + + if signoff { + // Signed-off-by + _, _ = messageBytes.WriteString("\n") + _, _ = messageBytes.WriteString("Signed-off-by: ") + _, _ = messageBytes.WriteString(committerSig.String()) + } + + env = append(env, + "GIT_COMMITTER_NAME="+committerSig.Name, + "GIT_COMMITTER_EMAIL="+committerSig.Email, + ) + + stdout := new(bytes.Buffer) + stderr := new(bytes.Buffer) + if err := cmdCommitTree. + Run(&git.RunOpts{ + Env: env, + Dir: t.basePath, + Stdin: messageBytes, + Stdout: stdout, + Stderr: stderr, + }); err != nil { + log.Error("Unable to commit-tree in temporary repo: %s (%s) Error: %v\nStdout: %s\nStderr: %s", + t.repo.FullName(), t.basePath, err, stdout, stderr) + return "", fmt.Errorf("Unable to commit-tree in temporary repo: %s Error: %w\nStdout: %s\nStderr: %s", + t.repo.FullName(), err, stdout, stderr) + } + return strings.TrimSpace(stdout.String()), nil +} + +// Push the provided commitHash to the repository branch by the provided user +func (t *TemporaryUploadRepository) Push(doer *user_model.User, commitHash, branch string) error { + // Because calls hooks we need to pass in the environment + env := repo_module.PushingEnvironment(doer, t.repo) + if err := git.Push(t.ctx, t.basePath, git.PushOptions{ + Remote: t.repo.RepoPath(), + Branch: strings.TrimSpace(commitHash) + ":" + git.BranchPrefix + strings.TrimSpace(branch), + Env: env, + }); err != nil { + if git.IsErrPushOutOfDate(err) { + return err + } else if git.IsErrPushRejected(err) { + rejectErr := err.(*git.ErrPushRejected) + log.Info("Unable to push back to repo from temporary repo due to rejection: %s (%s)\nStdout: %s\nStderr: %s\nError: %v", + t.repo.FullName(), t.basePath, rejectErr.StdOut, rejectErr.StdErr, rejectErr.Err) + return err + } + log.Error("Unable to push back to repo from temporary repo: %s (%s)\nError: %v", + t.repo.FullName(), t.basePath, err) + return fmt.Errorf("Unable to push back to repo from temporary repo: %s (%s) Error: %v", + t.repo.FullName(), t.basePath, err) + } + return nil +} + +// DiffIndex returns a Diff of the current index to the head +func (t *TemporaryUploadRepository) DiffIndex() (*gitdiff.Diff, error) { + stdoutReader, stdoutWriter, err := os.Pipe() + if err != nil { + log.Error("Unable to open stdout pipe: %v", err) + return nil, fmt.Errorf("Unable to open stdout pipe: %w", err) + } + defer func() { + _ = stdoutReader.Close() + _ = stdoutWriter.Close() + }() + stderr := new(bytes.Buffer) + var diff *gitdiff.Diff + var finalErr error + + if err := git.NewCommand(t.ctx, "diff-index", "--src-prefix=\\a/", "--dst-prefix=\\b/", "--cached", "-p", "HEAD"). + Run(&git.RunOpts{ + Timeout: 30 * time.Second, + Dir: t.basePath, + Stdout: stdoutWriter, + Stderr: stderr, + PipelineFunc: func(ctx context.Context, cancel context.CancelFunc) error { + _ = stdoutWriter.Close() + diff, finalErr = gitdiff.ParsePatch(t.ctx, setting.Git.MaxGitDiffLines, setting.Git.MaxGitDiffLineCharacters, setting.Git.MaxGitDiffFiles, stdoutReader, "") + if finalErr != nil { + log.Error("ParsePatch: %v", finalErr) + cancel() + } + _ = stdoutReader.Close() + return finalErr + }, + }); err != nil { + if finalErr != nil { + log.Error("Unable to ParsePatch in temporary repo %s (%s). Error: %v", t.repo.FullName(), t.basePath, finalErr) + return nil, finalErr + } + log.Error("Unable to run diff-index pipeline in temporary repo %s (%s). Error: %v\nStderr: %s", + t.repo.FullName(), t.basePath, err, stderr) + return nil, fmt.Errorf("Unable to run diff-index pipeline in temporary repo %s. Error: %w\nStderr: %s", + t.repo.FullName(), err, stderr) + } + + diff.NumFiles, diff.TotalAddition, diff.TotalDeletion, err = git.GetDiffShortStat(t.ctx, t.basePath, git.TrustedCmdArgs{"--cached"}, "HEAD") + if err != nil { + return nil, err + } + + return diff, nil +} + +// GetBranchCommit Gets the commit object of the given branch +func (t *TemporaryUploadRepository) GetBranchCommit(branch string) (*git.Commit, error) { + if t.gitRepo == nil { + return nil, fmt.Errorf("repository has not been cloned") + } + return t.gitRepo.GetBranchCommit(branch) +} + +// GetCommit Gets the commit object of the given commit ID +func (t *TemporaryUploadRepository) GetCommit(commitID string) (*git.Commit, error) { + if t.gitRepo == nil { + return nil, fmt.Errorf("repository has not been cloned") + } + return t.gitRepo.GetCommit(commitID) +} diff --git a/services/repository/files/temp_repo_test.go b/services/repository/files/temp_repo_test.go new file mode 100644 index 0000000..e7d85ea --- /dev/null +++ b/services/repository/files/temp_repo_test.go @@ -0,0 +1,28 @@ +// Copyright 2024 The Forgejo Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package files + +import ( + "testing" + + "code.gitea.io/gitea/models/db" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/models/unittest" + "code.gitea.io/gitea/modules/git" + + "github.com/stretchr/testify/require" +) + +func TestRemoveFilesFromIndexSha256(t *testing.T) { + if git.CheckGitVersionAtLeast("2.42") != nil { + t.Skip("skipping because installed Git version doesn't support SHA256") + } + unittest.PrepareTestEnv(t) + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) + + temp, err := NewTemporaryUploadRepository(db.DefaultContext, repo) + require.NoError(t, err) + require.NoError(t, temp.Init("sha256")) + require.NoError(t, temp.RemoveFilesFromIndex("README.md")) +} diff --git a/services/repository/files/tree.go b/services/repository/files/tree.go new file mode 100644 index 0000000..e3a7f3b --- /dev/null +++ b/services/repository/files/tree.go @@ -0,0 +1,101 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package files + +import ( + "context" + "fmt" + "net/url" + + "code.gitea.io/gitea/models" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" + api "code.gitea.io/gitea/modules/structs" +) + +// GetTreeBySHA get the GitTreeResponse of a repository using a sha hash. +func GetTreeBySHA(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, sha string, page, perPage int, recursive bool) (*api.GitTreeResponse, error) { + gitTree, err := gitRepo.GetTree(sha) + if err != nil || gitTree == nil { + return nil, models.ErrSHANotFound{ + SHA: sha, + } + } + tree := new(api.GitTreeResponse) + tree.SHA = gitTree.ResolvedID.String() + tree.URL = repo.APIURL() + "/git/trees/" + url.PathEscape(tree.SHA) + var entries git.Entries + if recursive { + entries, err = gitTree.ListEntriesRecursiveWithSize() + } else { + entries, err = gitTree.ListEntries() + } + if err != nil { + return nil, err + } + apiURL := repo.APIURL() + apiURLLen := len(apiURL) + objectFormat := git.ObjectFormatFromName(repo.ObjectFormatName) + hashLen := objectFormat.FullLength() + + const gitBlobsPath = "/git/blobs/" + blobURL := make([]byte, apiURLLen+hashLen+len(gitBlobsPath)) + copy(blobURL, apiURL) + copy(blobURL[apiURLLen:], []byte(gitBlobsPath)) + + const gitTreePath = "/git/trees/" + treeURL := make([]byte, apiURLLen+hashLen+len(gitTreePath)) + copy(treeURL, apiURL) + copy(treeURL[apiURLLen:], []byte(gitTreePath)) + + // copyPos is at the start of the hash + copyPos := len(treeURL) - hashLen + + if perPage <= 0 || perPage > setting.API.DefaultGitTreesPerPage { + perPage = setting.API.DefaultGitTreesPerPage + } + if page <= 0 { + page = 1 + } + tree.Page = page + tree.TotalCount = len(entries) + rangeStart := perPage * (page - 1) + if rangeStart >= len(entries) { + return tree, nil + } + var rangeEnd int + if len(entries) > perPage { + tree.Truncated = true + } + if rangeStart+perPage < len(entries) { + rangeEnd = rangeStart + perPage + } else { + rangeEnd = len(entries) + } + tree.Entries = make([]api.GitEntry, rangeEnd-rangeStart) + for e := rangeStart; e < rangeEnd; e++ { + i := e - rangeStart + + tree.Entries[i].Path = entries[e].Name() + tree.Entries[i].Mode = fmt.Sprintf("%06o", entries[e].Mode()) + tree.Entries[i].Type = entries[e].Type() + tree.Entries[i].Size = entries[e].Size() + tree.Entries[i].SHA = entries[e].ID.String() + + if entries[e].IsDir() { + copy(treeURL[copyPos:], entries[e].ID.String()) + tree.Entries[i].URL = string(treeURL) + } else if entries[e].IsSubModule() { + // In Github Rest API Version=2022-11-28, if a tree entry is a submodule, + // its url will be returned as an empty string. + // So the URL will be set to "" here. + tree.Entries[i].URL = "" + } else { + copy(blobURL[copyPos:], entries[e].ID.String()) + tree.Entries[i].URL = string(blobURL) + } + } + return tree, nil +} diff --git a/services/repository/files/tree_test.go b/services/repository/files/tree_test.go new file mode 100644 index 0000000..9e5c5c1 --- /dev/null +++ b/services/repository/files/tree_test.go @@ -0,0 +1,52 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package files + +import ( + "testing" + + "code.gitea.io/gitea/models/unittest" + api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/services/contexttest" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGetTreeBySHA(t *testing.T) { + unittest.PrepareTestEnv(t) + ctx, _ := contexttest.MockContext(t, "user2/repo1") + contexttest.LoadRepo(t, ctx, 1) + contexttest.LoadUser(t, ctx, 2) + contexttest.LoadGitRepo(t, ctx) + defer ctx.Repo.GitRepo.Close() + + sha := ctx.Repo.Repository.DefaultBranch + page := 1 + perPage := 10 + ctx.SetParams(":id", "1") + ctx.SetParams(":sha", sha) + + tree, err := GetTreeBySHA(ctx, ctx.Repo.Repository, ctx.Repo.GitRepo, ctx.Params(":sha"), page, perPage, true) + require.NoError(t, err) + expectedTree := &api.GitTreeResponse{ + SHA: "65f1bf27bc3bf70f64657658635e66094edbcb4d", + URL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/trees/65f1bf27bc3bf70f64657658635e66094edbcb4d", + Entries: []api.GitEntry{ + { + Path: "README.md", + Mode: "100644", + Type: "blob", + Size: 30, + SHA: "4b4851ad51df6a7d9f25c979345979eaeb5b349f", + URL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/blobs/4b4851ad51df6a7d9f25c979345979eaeb5b349f", + }, + }, + Truncated: false, + Page: 1, + TotalCount: 1, + } + + assert.EqualValues(t, expectedTree, tree) +} diff --git a/services/repository/files/update.go b/services/repository/files/update.go new file mode 100644 index 0000000..d6025b6 --- /dev/null +++ b/services/repository/files/update.go @@ -0,0 +1,501 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package files + +import ( + "context" + "fmt" + "io" + "path" + "strings" + "time" + + "code.gitea.io/gitea/models" + git_model "code.gitea.io/gitea/models/git" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/gitrepo" + "code.gitea.io/gitea/modules/lfs" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/structs" + asymkey_service "code.gitea.io/gitea/services/asymkey" +) + +// IdentityOptions for a person's identity like an author or committer +type IdentityOptions struct { + Name string + Email string +} + +// CommitDateOptions store dates for GIT_AUTHOR_DATE and GIT_COMMITTER_DATE +type CommitDateOptions struct { + Author time.Time + Committer time.Time +} + +type ChangeRepoFile struct { + Operation string + TreePath string + FromTreePath string + ContentReader io.ReadSeeker + SHA string + Options *RepoFileOptions +} + +// ChangeRepoFilesOptions holds the repository files update options +type ChangeRepoFilesOptions struct { + LastCommitID string + OldBranch string + NewBranch string + Message string + Files []*ChangeRepoFile + Author *IdentityOptions + Committer *IdentityOptions + Dates *CommitDateOptions + Signoff bool +} + +type RepoFileOptions struct { + treePath string + fromTreePath string + executable bool +} + +// ChangeRepoFiles adds, updates or removes multiple files in the given repository +func ChangeRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *user_model.User, opts *ChangeRepoFilesOptions) (*structs.FilesResponse, error) { + err := repo.MustNotBeArchived() + if err != nil { + return nil, err + } + + // If no branch name is set, assume default branch + if opts.OldBranch == "" { + opts.OldBranch = repo.DefaultBranch + } + if opts.NewBranch == "" { + opts.NewBranch = opts.OldBranch + } + + gitRepo, closer, err := gitrepo.RepositoryFromContextOrOpen(ctx, repo) + if err != nil { + return nil, err + } + defer closer.Close() + + // oldBranch must exist for this operation + if _, err := gitRepo.GetBranch(opts.OldBranch); err != nil && !repo.IsEmpty { + return nil, err + } + + var treePaths []string + for _, file := range opts.Files { + // If FromTreePath is not set, set it to the opts.TreePath + if file.TreePath != "" && file.FromTreePath == "" { + file.FromTreePath = file.TreePath + } + + // Check that the path given in opts.treePath is valid (not a git path) + treePath := CleanUploadFileName(file.TreePath) + if treePath == "" { + return nil, models.ErrFilenameInvalid{ + Path: file.TreePath, + } + } + // If there is a fromTreePath (we are copying it), also clean it up + fromTreePath := CleanUploadFileName(file.FromTreePath) + if fromTreePath == "" && file.FromTreePath != "" { + return nil, models.ErrFilenameInvalid{ + Path: file.FromTreePath, + } + } + + file.Options = &RepoFileOptions{ + treePath: treePath, + fromTreePath: fromTreePath, + executable: false, + } + treePaths = append(treePaths, treePath) + } + + // A NewBranch can be specified for the file to be created/updated in a new branch. + // Check to make sure the branch does not already exist, otherwise we can't proceed. + // If we aren't branching to a new branch, make sure user can commit to the given branch + if opts.NewBranch != opts.OldBranch { + existingBranch, err := gitRepo.GetBranch(opts.NewBranch) + if existingBranch != nil { + return nil, git_model.ErrBranchAlreadyExists{ + BranchName: opts.NewBranch, + } + } + if err != nil && !git.IsErrBranchNotExist(err) { + return nil, err + } + } else if err := VerifyBranchProtection(ctx, repo, doer, opts.OldBranch, treePaths); err != nil { + return nil, err + } + + message := strings.TrimSpace(opts.Message) + + author, committer := GetAuthorAndCommitterUsers(opts.Author, opts.Committer, doer) + + t, err := NewTemporaryUploadRepository(ctx, repo) + if err != nil { + log.Error("NewTemporaryUploadRepository failed: %v", err) + } + defer t.Close() + hasOldBranch := true + if err := t.Clone(opts.OldBranch, true); err != nil { + for _, file := range opts.Files { + if file.Operation == "delete" { + return nil, err + } + } + if !git.IsErrBranchNotExist(err) || !repo.IsEmpty { + return nil, err + } + if err := t.Init(repo.ObjectFormatName); err != nil { + return nil, err + } + hasOldBranch = false + opts.LastCommitID = "" + } + if hasOldBranch { + if err := t.SetDefaultIndex(); err != nil { + return nil, err + } + } + + for _, file := range opts.Files { + if file.Operation == "delete" { + // Get the files in the index + filesInIndex, err := t.LsFiles(file.TreePath) + if err != nil { + return nil, fmt.Errorf("DeleteRepoFile: %w", err) + } + + // Find the file we want to delete in the index + inFilelist := false + for _, indexFile := range filesInIndex { + if indexFile == file.TreePath { + inFilelist = true + break + } + } + if !inFilelist { + return nil, models.ErrRepoFileDoesNotExist{ + Path: file.TreePath, + } + } + } + } + + if hasOldBranch { + // Get the commit of the original branch + commit, err := t.GetBranchCommit(opts.OldBranch) + if err != nil { + return nil, err // Couldn't get a commit for the branch + } + + // Assigned LastCommitID in opts if it hasn't been set + if opts.LastCommitID == "" { + opts.LastCommitID = commit.ID.String() + } else { + lastCommitID, err := t.gitRepo.ConvertToGitID(opts.LastCommitID) + if err != nil { + return nil, fmt.Errorf("ConvertToSHA1: Invalid last commit ID: %w", err) + } + opts.LastCommitID = lastCommitID.String() + } + + for _, file := range opts.Files { + if err := handleCheckErrors(file, commit, opts); err != nil { + return nil, err + } + } + } + + contentStore := lfs.NewContentStore() + for _, file := range opts.Files { + switch file.Operation { + case "create", "update": + if err := CreateOrUpdateFile(ctx, t, file, contentStore, repo.ID, hasOldBranch); err != nil { + return nil, err + } + case "delete": + // Remove the file from the index + if err := t.RemoveFilesFromIndex(file.TreePath); err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("invalid file operation: %s %s, supported operations are create, update, delete", file.Operation, file.Options.treePath) + } + } + + // Now write the tree + treeHash, err := t.WriteTree() + if err != nil { + return nil, err + } + + // Now commit the tree + var commitHash string + if opts.Dates != nil { + commitHash, err = t.CommitTreeWithDate(opts.LastCommitID, author, committer, treeHash, message, opts.Signoff, opts.Dates.Author, opts.Dates.Committer) + } else { + commitHash, err = t.CommitTree(opts.LastCommitID, author, committer, treeHash, message, opts.Signoff) + } + if err != nil { + return nil, err + } + + // Then push this tree to NewBranch + if err := t.Push(doer, commitHash, opts.NewBranch); err != nil { + log.Error("%T %v", err, err) + return nil, err + } + + commit, err := t.GetCommit(commitHash) + if err != nil { + return nil, err + } + + filesResponse, err := GetFilesResponseFromCommit(ctx, repo, commit, opts.NewBranch, treePaths) + if err != nil { + return nil, err + } + + if repo.IsEmpty { + if isEmpty, err := gitRepo.IsEmpty(); err == nil && !isEmpty { + _ = repo_model.UpdateRepositoryCols(ctx, &repo_model.Repository{ID: repo.ID, IsEmpty: false, DefaultBranch: opts.NewBranch}, "is_empty", "default_branch") + } + } + + return filesResponse, nil +} + +// handles the check for various issues for ChangeRepoFiles +func handleCheckErrors(file *ChangeRepoFile, commit *git.Commit, opts *ChangeRepoFilesOptions) error { + if file.Operation == "update" || file.Operation == "delete" { + fromEntry, err := commit.GetTreeEntryByPath(file.Options.fromTreePath) + if err != nil { + return err + } + if file.SHA != "" { + // If a SHA was given and the SHA given doesn't match the SHA of the fromTreePath, throw error + if file.SHA != fromEntry.ID.String() { + return models.ErrSHADoesNotMatch{ + Path: file.Options.treePath, + GivenSHA: file.SHA, + CurrentSHA: fromEntry.ID.String(), + } + } + } else if opts.LastCommitID != "" { + // If a lastCommitID was given and it doesn't match the commitID of the head of the branch throw + // an error, but only if we aren't creating a new branch. + if commit.ID.String() != opts.LastCommitID && opts.OldBranch == opts.NewBranch { + if changed, err := commit.FileChangedSinceCommit(file.Options.treePath, opts.LastCommitID); err != nil { + return err + } else if changed { + return models.ErrCommitIDDoesNotMatch{ + GivenCommitID: opts.LastCommitID, + CurrentCommitID: opts.LastCommitID, + } + } + // The file wasn't modified, so we are good to delete it + } + } else { + // When updating a file, a lastCommitID or SHA needs to be given to make sure other commits + // haven't been made. We throw an error if one wasn't provided. + return models.ErrSHAOrCommitIDNotProvided{} + } + file.Options.executable = fromEntry.IsExecutable() + } + if file.Operation == "create" || file.Operation == "update" { + // For the path where this file will be created/updated, we need to make + // sure no parts of the path are existing files or links except for the last + // item in the path which is the file name, and that shouldn't exist IF it is + // a new file OR is being moved to a new path. + treePathParts := strings.Split(file.Options.treePath, "/") + subTreePath := "" + for index, part := range treePathParts { + subTreePath = path.Join(subTreePath, part) + entry, err := commit.GetTreeEntryByPath(subTreePath) + if err != nil { + if git.IsErrNotExist(err) { + // Means there is no item with that name, so we're good + break + } + return err + } + if index < len(treePathParts)-1 { + if !entry.IsDir() { + return models.ErrFilePathInvalid{ + Message: fmt.Sprintf("a file exists where you’re trying to create a subdirectory [path: %s]", subTreePath), + Path: subTreePath, + Name: part, + Type: git.EntryModeBlob, + } + } + } else if entry.IsLink() { + return models.ErrFilePathInvalid{ + Message: fmt.Sprintf("a symbolic link exists where you’re trying to create a subdirectory [path: %s]", subTreePath), + Path: subTreePath, + Name: part, + Type: git.EntryModeSymlink, + } + } else if entry.IsDir() { + return models.ErrFilePathInvalid{ + Message: fmt.Sprintf("a directory exists where you’re trying to create a file [path: %s]", subTreePath), + Path: subTreePath, + Name: part, + Type: git.EntryModeTree, + } + } else if file.Options.fromTreePath != file.Options.treePath || file.Operation == "create" { + // The entry shouldn't exist if we are creating new file or moving to a new path + return models.ErrRepoFileAlreadyExists{ + Path: file.Options.treePath, + } + } + } + } + + return nil +} + +// CreateOrUpdateFile handles creating or updating a file for ChangeRepoFiles +func CreateOrUpdateFile(ctx context.Context, t *TemporaryUploadRepository, file *ChangeRepoFile, contentStore *lfs.ContentStore, repoID int64, hasOldBranch bool) error { + // Get the two paths (might be the same if not moving) from the index if they exist + filesInIndex, err := t.LsFiles(file.TreePath, file.FromTreePath) + if err != nil { + return fmt.Errorf("UpdateRepoFile: %w", err) + } + // If is a new file (not updating) then the given path shouldn't exist + if file.Operation == "create" { + for _, indexFile := range filesInIndex { + if indexFile == file.TreePath { + return models.ErrRepoFileAlreadyExists{ + Path: file.TreePath, + } + } + } + } + + // Remove the old path from the tree + if file.Options.fromTreePath != file.Options.treePath && len(filesInIndex) > 0 { + for _, indexFile := range filesInIndex { + if indexFile == file.Options.fromTreePath { + if err := t.RemoveFilesFromIndex(file.FromTreePath); err != nil { + return err + } + } + } + } + + treeObjectContentReader := file.ContentReader + var lfsMetaObject *git_model.LFSMetaObject + if setting.LFS.StartServer && hasOldBranch { + // Check there is no way this can return multiple infos + filterAttribute, err := t.gitRepo.GitAttributeFirst("", file.Options.treePath, "filter") + if err != nil { + return err + } + + if filterAttribute == "lfs" { + // OK so we are supposed to LFS this data! + pointer, err := lfs.GeneratePointer(treeObjectContentReader) + if err != nil { + return err + } + lfsMetaObject = &git_model.LFSMetaObject{Pointer: pointer, RepositoryID: repoID} + treeObjectContentReader = strings.NewReader(pointer.StringContent()) + } + } + + // Add the object to the database + objectHash, err := t.HashObject(treeObjectContentReader) + if err != nil { + return err + } + + // Add the object to the index + if file.Options.executable { + if err := t.AddObjectToIndex("100755", objectHash, file.Options.treePath); err != nil { + return err + } + } else { + if err := t.AddObjectToIndex("100644", objectHash, file.Options.treePath); err != nil { + return err + } + } + + if lfsMetaObject != nil { + // We have an LFS object - create it + lfsMetaObject, err = git_model.NewLFSMetaObject(ctx, lfsMetaObject.RepositoryID, lfsMetaObject.Pointer) + if err != nil { + return err + } + exist, err := contentStore.Exists(lfsMetaObject.Pointer) + if err != nil { + return err + } + if !exist { + _, err := file.ContentReader.Seek(0, io.SeekStart) + if err != nil { + return err + } + if err := contentStore.Put(lfsMetaObject.Pointer, file.ContentReader); err != nil { + if _, err2 := git_model.RemoveLFSMetaObjectByOid(ctx, repoID, lfsMetaObject.Oid); err2 != nil { + return fmt.Errorf("unable to remove failed inserted LFS object %s: %v (Prev Error: %w)", lfsMetaObject.Oid, err2, err) + } + return err + } + } + } + + return nil +} + +// VerifyBranchProtection verify the branch protection for modifying the given treePath on the given branch +func VerifyBranchProtection(ctx context.Context, repo *repo_model.Repository, doer *user_model.User, branchName string, treePaths []string) error { + protectedBranch, err := git_model.GetFirstMatchProtectedBranchRule(ctx, repo.ID, branchName) + if err != nil { + return err + } + if protectedBranch != nil { + protectedBranch.Repo = repo + globUnprotected := protectedBranch.GetUnprotectedFilePatterns() + globProtected := protectedBranch.GetProtectedFilePatterns() + canUserPush := protectedBranch.CanUserPush(ctx, doer) + for _, treePath := range treePaths { + isUnprotectedFile := false + if len(globUnprotected) != 0 { + isUnprotectedFile = protectedBranch.IsUnprotectedFile(globUnprotected, treePath) + } + if !canUserPush && !isUnprotectedFile { + return models.ErrUserCannotCommit{ + UserName: doer.LowerName, + } + } + if protectedBranch.IsProtectedFile(globProtected, treePath) { + return models.ErrFilePathProtected{ + Path: treePath, + } + } + } + if protectedBranch.RequireSignedCommits { + _, _, _, err := asymkey_service.SignCRUDAction(ctx, repo.RepoPath(), doer, repo.RepoPath(), branchName) + if err != nil { + if !asymkey_service.IsErrWontSign(err) { + return err + } + return models.ErrUserCannotCommit{ + UserName: doer.LowerName, + } + } + } + } + return nil +} diff --git a/services/repository/files/upload.go b/services/repository/files/upload.go new file mode 100644 index 0000000..1330116 --- /dev/null +++ b/services/repository/files/upload.go @@ -0,0 +1,248 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package files + +import ( + "context" + "fmt" + "os" + "path" + "strings" + + git_model "code.gitea.io/gitea/models/git" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/lfs" + "code.gitea.io/gitea/modules/setting" +) + +// UploadRepoFileOptions contains the uploaded repository file options +type UploadRepoFileOptions struct { + LastCommitID string + OldBranch string + NewBranch string + TreePath string + Message string + Author *IdentityOptions + Committer *IdentityOptions + Files []string // In UUID format. + Signoff bool +} + +type uploadInfo struct { + upload *repo_model.Upload + lfsMetaObject *git_model.LFSMetaObject +} + +func cleanUpAfterFailure(ctx context.Context, infos *[]uploadInfo, t *TemporaryUploadRepository, original error) error { + for _, info := range *infos { + if info.lfsMetaObject == nil { + continue + } + if !info.lfsMetaObject.Existing { + if _, err := git_model.RemoveLFSMetaObjectByOid(ctx, t.repo.ID, info.lfsMetaObject.Oid); err != nil { + original = fmt.Errorf("%w, %v", original, err) // We wrap the original error - as this is the underlying error that required the fallback + } + } + } + return original +} + +// UploadRepoFiles uploads files to the given repository +func UploadRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *user_model.User, opts *UploadRepoFileOptions) error { + if len(opts.Files) == 0 { + return nil + } + + uploads, err := repo_model.GetUploadsByUUIDs(ctx, opts.Files) + if err != nil { + return fmt.Errorf("GetUploadsByUUIDs [uuids: %v]: %w", opts.Files, err) + } + + names := make([]string, len(uploads)) + infos := make([]uploadInfo, len(uploads)) + for i, upload := range uploads { + // Check file is not lfs locked, will return nil if lock setting not enabled + filepath := path.Join(opts.TreePath, upload.Name) + lfsLock, err := git_model.GetTreePathLock(ctx, repo.ID, filepath) + if err != nil { + return err + } + if lfsLock != nil && lfsLock.OwnerID != doer.ID { + u, err := user_model.GetUserByID(ctx, lfsLock.OwnerID) + if err != nil { + return err + } + return git_model.ErrLFSFileLocked{RepoID: repo.ID, Path: filepath, UserName: u.Name} + } + + names[i] = upload.Name + infos[i] = uploadInfo{upload: upload} + } + + t, err := NewTemporaryUploadRepository(ctx, repo) + if err != nil { + return err + } + defer t.Close() + + hasOldBranch := true + if err = t.Clone(opts.OldBranch, true); err != nil { + if !git.IsErrBranchNotExist(err) || !repo.IsEmpty { + return err + } + if err = t.Init(repo.ObjectFormatName); err != nil { + return err + } + hasOldBranch = false + opts.LastCommitID = "" + } + if hasOldBranch { + if err = t.SetDefaultIndex(); err != nil { + return err + } + } + + // Copy uploaded files into repository. + if err := copyUploadedLFSFilesIntoRepository(infos, t, opts.TreePath); err != nil { + return err + } + + // Now write the tree + treeHash, err := t.WriteTree() + if err != nil { + return err + } + + author, committer := GetAuthorAndCommitterUsers(opts.Author, opts.Committer, doer) + + // Now commit the tree + commitHash, err := t.CommitTree(opts.LastCommitID, author, committer, treeHash, opts.Message, opts.Signoff) + if err != nil { + return err + } + + // Now deal with LFS objects + for i := range infos { + if infos[i].lfsMetaObject == nil { + continue + } + infos[i].lfsMetaObject, err = git_model.NewLFSMetaObject(ctx, infos[i].lfsMetaObject.RepositoryID, infos[i].lfsMetaObject.Pointer) + if err != nil { + // OK Now we need to cleanup + return cleanUpAfterFailure(ctx, &infos, t, err) + } + // Don't move the files yet - we need to ensure that + // everything can be inserted first + } + + // OK now we can insert the data into the store - there's no way to clean up the store + // once it's in there, it's in there. + contentStore := lfs.NewContentStore() + for _, info := range infos { + if err := uploadToLFSContentStore(info, contentStore); err != nil { + return cleanUpAfterFailure(ctx, &infos, t, err) + } + } + + // Then push this tree to NewBranch + if err := t.Push(doer, commitHash, opts.NewBranch); err != nil { + return err + } + + return repo_model.DeleteUploads(ctx, uploads...) +} + +func copyUploadedLFSFilesIntoRepository(infos []uploadInfo, t *TemporaryUploadRepository, treePath string) error { + var storeInLFSFunc func(string) (bool, error) + + if setting.LFS.StartServer { + checker, err := t.gitRepo.GitAttributeChecker("", "filter") + if err != nil { + return err + } + defer checker.Close() + + storeInLFSFunc = func(name string) (bool, error) { + attrs, err := checker.CheckPath(name) + if err != nil { + return false, fmt.Errorf("could not CheckPath(%s): %w", name, err) + } + return attrs["filter"] == "lfs", nil + } + } + + // Copy uploaded files into repository. + for i, info := range infos { + storeInLFS := false + if storeInLFSFunc != nil { + var err error + storeInLFS, err = storeInLFSFunc(info.upload.Name) + if err != nil { + return err + } + } + + if err := copyUploadedLFSFileIntoRepository(&infos[i], storeInLFS, t, treePath); err != nil { + return err + } + } + return nil +} + +func copyUploadedLFSFileIntoRepository(info *uploadInfo, storeInLFS bool, t *TemporaryUploadRepository, treePath string) error { + file, err := os.Open(info.upload.LocalPath()) + if err != nil { + return err + } + defer file.Close() + + var objectHash string + if storeInLFS { + // Handle LFS + // FIXME: Inefficient! this should probably happen in models.Upload + pointer, err := lfs.GeneratePointer(file) + if err != nil { + return err + } + + info.lfsMetaObject = &git_model.LFSMetaObject{Pointer: pointer, RepositoryID: t.repo.ID} + + if objectHash, err = t.HashObject(strings.NewReader(pointer.StringContent())); err != nil { + return err + } + } else if objectHash, err = t.HashObject(file); err != nil { + return err + } + + // Add the object to the index + return t.AddObjectToIndex("100644", objectHash, path.Join(treePath, info.upload.Name)) +} + +func uploadToLFSContentStore(info uploadInfo, contentStore *lfs.ContentStore) error { + if info.lfsMetaObject == nil { + return nil + } + exist, err := contentStore.Exists(info.lfsMetaObject.Pointer) + if err != nil { + return err + } + if !exist { + file, err := os.Open(info.upload.LocalPath()) + if err != nil { + return err + } + + defer file.Close() + // FIXME: Put regenerates the hash and copies the file over. + // I guess this strictly ensures the soundness of the store but this is inefficient. + if err := contentStore.Put(info.lfsMetaObject.Pointer, file); err != nil { + // OK Now we need to cleanup + // Can't clean up the store, once uploaded there they're there. + return err + } + } + return nil +} |