diff options
author | Daniel Baumann <daniel@debian.org> | 2024-10-20 23:07:42 +0200 |
---|---|---|
committer | Daniel Baumann <daniel@debian.org> | 2024-11-09 15:38:42 +0100 |
commit | 714c83b2736d7e308bc33c49057952490eb98be2 (patch) | |
tree | 1d9ba7035798368569cd49056f4d596efc908cd8 /pkg/common | |
parent | Initial commit. (diff) | |
download | forgejo-act-debian.tar.xz forgejo-act-debian.zip |
Adding upstream version 1.21.4.HEADupstream/1.21.4upstreamdebian
Signed-off-by: Daniel Baumann <daniel@debian.org>
Diffstat (limited to 'pkg/common')
-rw-r--r-- | pkg/common/cartesian.go | 54 | ||||
-rw-r--r-- | pkg/common/cartesian_test.go | 39 | ||||
-rw-r--r-- | pkg/common/draw.go | 143 | ||||
-rw-r--r-- | pkg/common/dryrun.go | 25 | ||||
-rw-r--r-- | pkg/common/executor.go | 196 | ||||
-rw-r--r-- | pkg/common/executor_test.go | 152 | ||||
-rw-r--r-- | pkg/common/file.go | 73 | ||||
-rw-r--r-- | pkg/common/git/git.go | 418 | ||||
-rw-r--r-- | pkg/common/git/git_test.go | 249 | ||||
-rw-r--r-- | pkg/common/job_error.go | 30 | ||||
-rw-r--r-- | pkg/common/line_writer.go | 50 | ||||
-rw-r--r-- | pkg/common/line_writer_test.go | 37 | ||||
-rw-r--r-- | pkg/common/logger.go | 48 | ||||
-rw-r--r-- | pkg/common/outbound_ip.go | 75 |
14 files changed, 1589 insertions, 0 deletions
diff --git a/pkg/common/cartesian.go b/pkg/common/cartesian.go new file mode 100644 index 0000000..9cd6065 --- /dev/null +++ b/pkg/common/cartesian.go @@ -0,0 +1,54 @@ +package common + +// CartesianProduct takes map of lists and returns list of unique tuples +func CartesianProduct(mapOfLists map[string][]interface{}) []map[string]interface{} { + listNames := make([]string, 0) + lists := make([][]interface{}, 0) + for k, v := range mapOfLists { + listNames = append(listNames, k) + lists = append(lists, v) + } + + listCart := cartN(lists...) + + rtn := make([]map[string]interface{}, 0) + for _, list := range listCart { + vMap := make(map[string]interface{}) + for i, v := range list { + vMap[listNames[i]] = v + } + rtn = append(rtn, vMap) + } + return rtn +} + +func cartN(a ...[]interface{}) [][]interface{} { + c := 1 + for _, a := range a { + c *= len(a) + } + if c == 0 || len(a) == 0 { + return nil + } + p := make([][]interface{}, c) + b := make([]interface{}, c*len(a)) + n := make([]int, len(a)) + s := 0 + for i := range p { + e := s + len(a) + pi := b[s:e] + p[i] = pi + s = e + for j, n := range n { + pi[j] = a[j][n] + } + for j := len(n) - 1; j >= 0; j-- { + n[j]++ + if n[j] < len(a[j]) { + break + } + n[j] = 0 + } + } + return p +} diff --git a/pkg/common/cartesian_test.go b/pkg/common/cartesian_test.go new file mode 100644 index 0000000..c49de06 --- /dev/null +++ b/pkg/common/cartesian_test.go @@ -0,0 +1,39 @@ +package common + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCartesianProduct(t *testing.T) { + assert := assert.New(t) + input := map[string][]interface{}{ + "foo": {1, 2, 3, 4}, + "bar": {"a", "b", "c"}, + "baz": {false, true}, + } + + output := CartesianProduct(input) + assert.Len(output, 24) + + for _, v := range output { + assert.Len(v, 3) + + assert.Contains(v, "foo") + assert.Contains(v, "bar") + assert.Contains(v, "baz") + } + + input = map[string][]interface{}{ + "foo": {1, 2, 3, 4}, + "bar": {}, + "baz": {false, true}, + } + output = CartesianProduct(input) + assert.Len(output, 0) + + input = map[string][]interface{}{} + output = CartesianProduct(input) + assert.Len(output, 0) +} diff --git a/pkg/common/draw.go b/pkg/common/draw.go new file mode 100644 index 0000000..b5b21fe --- /dev/null +++ b/pkg/common/draw.go @@ -0,0 +1,143 @@ +package common + +import ( + "fmt" + "io" + "os" + "strings" +) + +// Style is a specific style +type Style int + +// Styles +const ( + StyleDoubleLine = iota + StyleSingleLine + StyleDashedLine + StyleNoLine +) + +// NewPen creates a new pen +func NewPen(style Style, color int) *Pen { + bgcolor := 49 + if os.Getenv("CLICOLOR") == "0" { + color = 0 + bgcolor = 0 + } + return &Pen{ + style: style, + color: color, + bgcolor: bgcolor, + } +} + +type styleDef struct { + cornerTL string + cornerTR string + cornerBL string + cornerBR string + lineH string + lineV string +} + +var styleDefs = []styleDef{ + {"\u2554", "\u2557", "\u255a", "\u255d", "\u2550", "\u2551"}, + {"\u256d", "\u256e", "\u2570", "\u256f", "\u2500", "\u2502"}, + {"\u250c", "\u2510", "\u2514", "\u2518", "\u254c", "\u254e"}, + {" ", " ", " ", " ", " ", " "}, +} + +// Pen struct +type Pen struct { + style Style + color int + bgcolor int +} + +// Drawing struct +type Drawing struct { + buf *strings.Builder + width int +} + +func (p *Pen) drawTopBars(buf io.Writer, labels ...string) { + style := styleDefs[p.style] + for _, label := range labels { + bar := strings.Repeat(style.lineH, len(label)+2) + fmt.Fprintf(buf, " ") + fmt.Fprintf(buf, "\x1b[%d;%dm", p.color, p.bgcolor) + fmt.Fprintf(buf, "%s%s%s", style.cornerTL, bar, style.cornerTR) + fmt.Fprintf(buf, "\x1b[%dm", 0) + } + fmt.Fprintf(buf, "\n") +} +func (p *Pen) drawBottomBars(buf io.Writer, labels ...string) { + style := styleDefs[p.style] + for _, label := range labels { + bar := strings.Repeat(style.lineH, len(label)+2) + fmt.Fprintf(buf, " ") + fmt.Fprintf(buf, "\x1b[%d;%dm", p.color, p.bgcolor) + fmt.Fprintf(buf, "%s%s%s", style.cornerBL, bar, style.cornerBR) + fmt.Fprintf(buf, "\x1b[%dm", 0) + } + fmt.Fprintf(buf, "\n") +} +func (p *Pen) drawLabels(buf io.Writer, labels ...string) { + style := styleDefs[p.style] + for _, label := range labels { + fmt.Fprintf(buf, " ") + fmt.Fprintf(buf, "\x1b[%d;%dm", p.color, p.bgcolor) + fmt.Fprintf(buf, "%s %s %s", style.lineV, label, style.lineV) + fmt.Fprintf(buf, "\x1b[%dm", 0) + } + fmt.Fprintf(buf, "\n") +} + +// DrawArrow between boxes +func (p *Pen) DrawArrow() *Drawing { + drawing := &Drawing{ + buf: new(strings.Builder), + width: 1, + } + fmt.Fprintf(drawing.buf, "\x1b[%dm", p.color) + fmt.Fprintf(drawing.buf, "\u2b07") + fmt.Fprintf(drawing.buf, "\x1b[%dm", 0) + return drawing +} + +// DrawBoxes to draw boxes +func (p *Pen) DrawBoxes(labels ...string) *Drawing { + width := 0 + for _, l := range labels { + width += len(l) + 2 + 2 + 1 + } + drawing := &Drawing{ + buf: new(strings.Builder), + width: width, + } + p.drawTopBars(drawing.buf, labels...) + p.drawLabels(drawing.buf, labels...) + p.drawBottomBars(drawing.buf, labels...) + + return drawing +} + +// Draw to writer +func (d *Drawing) Draw(writer io.Writer, centerOnWidth int) { + padSize := (centerOnWidth - d.GetWidth()) / 2 + if padSize < 0 { + padSize = 0 + } + for _, l := range strings.Split(d.buf.String(), "\n") { + if len(l) > 0 { + padding := strings.Repeat(" ", padSize) + fmt.Fprintf(writer, "%s%s\n", padding, l) + } + } +} + +// GetWidth of drawing +func (d *Drawing) GetWidth() int { + return d.width +} diff --git a/pkg/common/dryrun.go b/pkg/common/dryrun.go new file mode 100644 index 0000000..2d5a14e --- /dev/null +++ b/pkg/common/dryrun.go @@ -0,0 +1,25 @@ +package common + +import ( + "context" +) + +type dryrunContextKey string + +const dryrunContextKeyVal = dryrunContextKey("dryrun") + +// Dryrun returns true if the current context is dryrun +func Dryrun(ctx context.Context) bool { + val := ctx.Value(dryrunContextKeyVal) + if val != nil { + if dryrun, ok := val.(bool); ok { + return dryrun + } + } + return false +} + +// WithDryrun adds a value to the context for dryrun +func WithDryrun(ctx context.Context, dryrun bool) context.Context { + return context.WithValue(ctx, dryrunContextKeyVal, dryrun) +} diff --git a/pkg/common/executor.go b/pkg/common/executor.go new file mode 100644 index 0000000..a5eb079 --- /dev/null +++ b/pkg/common/executor.go @@ -0,0 +1,196 @@ +package common + +import ( + "context" + "fmt" + + log "github.com/sirupsen/logrus" +) + +// Warning that implements `error` but safe to ignore +type Warning struct { + Message string +} + +// Error the contract for error +func (w Warning) Error() string { + return w.Message +} + +// Warningf create a warning +func Warningf(format string, args ...interface{}) Warning { + w := Warning{ + Message: fmt.Sprintf(format, args...), + } + return w +} + +// Executor define contract for the steps of a workflow +type Executor func(ctx context.Context) error + +// Conditional define contract for the conditional predicate +type Conditional func(ctx context.Context) bool + +// NewInfoExecutor is an executor that logs messages +func NewInfoExecutor(format string, args ...interface{}) Executor { + return func(ctx context.Context) error { + logger := Logger(ctx) + logger.Infof(format, args...) + return nil + } +} + +// NewDebugExecutor is an executor that logs messages +func NewDebugExecutor(format string, args ...interface{}) Executor { + return func(ctx context.Context) error { + logger := Logger(ctx) + logger.Debugf(format, args...) + return nil + } +} + +// NewPipelineExecutor creates a new executor from a series of other executors +func NewPipelineExecutor(executors ...Executor) Executor { + if len(executors) == 0 { + return func(ctx context.Context) error { + return nil + } + } + var rtn Executor + for _, executor := range executors { + if rtn == nil { + rtn = executor + } else { + rtn = rtn.Then(executor) + } + } + return rtn +} + +// NewConditionalExecutor creates a new executor based on conditions +func NewConditionalExecutor(conditional Conditional, trueExecutor Executor, falseExecutor Executor) Executor { + return func(ctx context.Context) error { + if conditional(ctx) { + if trueExecutor != nil { + return trueExecutor(ctx) + } + } else { + if falseExecutor != nil { + return falseExecutor(ctx) + } + } + return nil + } +} + +// NewErrorExecutor creates a new executor that always errors out +func NewErrorExecutor(err error) Executor { + return func(ctx context.Context) error { + return err + } +} + +// NewParallelExecutor creates a new executor from a parallel of other executors +func NewParallelExecutor(parallel int, executors ...Executor) Executor { + return func(ctx context.Context) error { + work := make(chan Executor, len(executors)) + errs := make(chan error, len(executors)) + + if 1 > parallel { + log.Infof("Parallel tasks (%d) below minimum, setting to 1", parallel) + parallel = 1 + } + + for i := 0; i < parallel; i++ { + go func(work <-chan Executor, errs chan<- error) { + for executor := range work { + errs <- executor(ctx) + } + }(work, errs) + } + + for i := 0; i < len(executors); i++ { + work <- executors[i] + } + close(work) + + // Executor waits all executors to cleanup these resources. + var firstErr error + for i := 0; i < len(executors); i++ { + err := <-errs + if firstErr == nil { + firstErr = err + } + } + + if err := ctx.Err(); err != nil { + return err + } + return firstErr + } +} + +// Then runs another executor if this executor succeeds +func (e Executor) Then(then Executor) Executor { + return func(ctx context.Context) error { + err := e(ctx) + if err != nil { + switch err.(type) { + case Warning: + Logger(ctx).Warning(err.Error()) + default: + return err + } + } + if ctx.Err() != nil { + return ctx.Err() + } + return then(ctx) + } +} + +// If only runs this executor if conditional is true +func (e Executor) If(conditional Conditional) Executor { + return func(ctx context.Context) error { + if conditional(ctx) { + return e(ctx) + } + return nil + } +} + +// IfNot only runs this executor if conditional is true +func (e Executor) IfNot(conditional Conditional) Executor { + return func(ctx context.Context) error { + if !conditional(ctx) { + return e(ctx) + } + return nil + } +} + +// IfBool only runs this executor if conditional is true +func (e Executor) IfBool(conditional bool) Executor { + return e.If(func(ctx context.Context) bool { + return conditional + }) +} + +// Finally adds an executor to run after other executor +func (e Executor) Finally(finally Executor) Executor { + return func(ctx context.Context) error { + err := e(ctx) + err2 := finally(ctx) + if err2 != nil { + return fmt.Errorf("Error occurred running finally: %v (original error: %v)", err2, err) + } + return err + } +} + +// Not return an inverted conditional +func (c Conditional) Not() Conditional { + return func(ctx context.Context) bool { + return !c(ctx) + } +} diff --git a/pkg/common/executor_test.go b/pkg/common/executor_test.go new file mode 100644 index 0000000..e70c638 --- /dev/null +++ b/pkg/common/executor_test.go @@ -0,0 +1,152 @@ +package common + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestNewWorkflow(t *testing.T) { + assert := assert.New(t) + + ctx := context.Background() + + // empty + emptyWorkflow := NewPipelineExecutor() + assert.Nil(emptyWorkflow(ctx)) + + // error case + errorWorkflow := NewErrorExecutor(fmt.Errorf("test error")) + assert.NotNil(errorWorkflow(ctx)) + + // multiple success case + runcount := 0 + successWorkflow := NewPipelineExecutor( + func(ctx context.Context) error { + runcount++ + return nil + }, + func(ctx context.Context) error { + runcount++ + return nil + }) + assert.Nil(successWorkflow(ctx)) + assert.Equal(2, runcount) +} + +func TestNewConditionalExecutor(t *testing.T) { + assert := assert.New(t) + + ctx := context.Background() + + trueCount := 0 + falseCount := 0 + + err := NewConditionalExecutor(func(ctx context.Context) bool { + return false + }, func(ctx context.Context) error { + trueCount++ + return nil + }, func(ctx context.Context) error { + falseCount++ + return nil + })(ctx) + + assert.Nil(err) + assert.Equal(0, trueCount) + assert.Equal(1, falseCount) + + err = NewConditionalExecutor(func(ctx context.Context) bool { + return true + }, func(ctx context.Context) error { + trueCount++ + return nil + }, func(ctx context.Context) error { + falseCount++ + return nil + })(ctx) + + assert.Nil(err) + assert.Equal(1, trueCount) + assert.Equal(1, falseCount) +} + +func TestNewParallelExecutor(t *testing.T) { + assert := assert.New(t) + + ctx := context.Background() + + count := 0 + activeCount := 0 + maxCount := 0 + emptyWorkflow := NewPipelineExecutor(func(ctx context.Context) error { + count++ + + activeCount++ + if activeCount > maxCount { + maxCount = activeCount + } + time.Sleep(2 * time.Second) + activeCount-- + + return nil + }) + + err := NewParallelExecutor(2, emptyWorkflow, emptyWorkflow, emptyWorkflow)(ctx) + + assert.Equal(3, count, "should run all 3 executors") + assert.Equal(2, maxCount, "should run at most 2 executors in parallel") + assert.Nil(err) + + // Reset to test running the executor with 0 parallelism + count = 0 + activeCount = 0 + maxCount = 0 + + errSingle := NewParallelExecutor(0, emptyWorkflow, emptyWorkflow, emptyWorkflow)(ctx) + + assert.Equal(3, count, "should run all 3 executors") + assert.Equal(1, maxCount, "should run at most 1 executors in parallel") + assert.Nil(errSingle) +} + +func TestNewParallelExecutorFailed(t *testing.T) { + assert := assert.New(t) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + count := 0 + errorWorkflow := NewPipelineExecutor(func(ctx context.Context) error { + count++ + return fmt.Errorf("fake error") + }) + err := NewParallelExecutor(1, errorWorkflow)(ctx) + assert.Equal(1, count) + assert.ErrorIs(context.Canceled, err) +} + +func TestNewParallelExecutorCanceled(t *testing.T) { + assert := assert.New(t) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + errExpected := fmt.Errorf("fake error") + + count := 0 + successWorkflow := NewPipelineExecutor(func(ctx context.Context) error { + count++ + return nil + }) + errorWorkflow := NewPipelineExecutor(func(ctx context.Context) error { + count++ + return errExpected + }) + err := NewParallelExecutor(3, errorWorkflow, successWorkflow, successWorkflow)(ctx) + assert.Equal(3, count) + assert.Error(errExpected, err) +} diff --git a/pkg/common/file.go b/pkg/common/file.go new file mode 100644 index 0000000..09c2102 --- /dev/null +++ b/pkg/common/file.go @@ -0,0 +1,73 @@ +package common + +import ( + "fmt" + "io" + "os" +) + +// CopyFile copy file +func CopyFile(source string, dest string) (err error) { + sourcefile, err := os.Open(source) + if err != nil { + return err + } + + defer sourcefile.Close() + + destfile, err := os.Create(dest) + if err != nil { + return err + } + + defer destfile.Close() + + _, err = io.Copy(destfile, sourcefile) + if err == nil { + sourceinfo, err := os.Stat(source) + if err != nil { + _ = os.Chmod(dest, sourceinfo.Mode()) + } + } + + return +} + +// CopyDir recursive copy of directory +func CopyDir(source string, dest string) (err error) { + // get properties of source dir + sourceinfo, err := os.Stat(source) + if err != nil { + return err + } + + // create dest dir + + err = os.MkdirAll(dest, sourceinfo.Mode()) + if err != nil { + return err + } + + objects, err := os.ReadDir(source) + + for _, obj := range objects { + sourcefilepointer := source + "/" + obj.Name() + + destinationfilepointer := dest + "/" + obj.Name() + + if obj.IsDir() { + // create sub-directories - recursively + err = CopyDir(sourcefilepointer, destinationfilepointer) + if err != nil { + fmt.Println(err) + } + } else { + // perform copy + err = CopyFile(sourcefilepointer, destinationfilepointer) + if err != nil { + fmt.Println(err) + } + } + } + return err +} diff --git a/pkg/common/git/git.go b/pkg/common/git/git.go new file mode 100644 index 0000000..c7ee889 --- /dev/null +++ b/pkg/common/git/git.go @@ -0,0 +1,418 @@ +package git + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path" + "regexp" + "strings" + "sync" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/config" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/storer" + "github.com/go-git/go-git/v5/plumbing/transport/http" + "github.com/mattn/go-isatty" + log "github.com/sirupsen/logrus" + + "github.com/nektos/act/pkg/common" +) + +var ( + codeCommitHTTPRegex = regexp.MustCompile(`^https?://git-codecommit\.(.+)\.amazonaws.com/v1/repos/(.+)$`) + codeCommitSSHRegex = regexp.MustCompile(`ssh://git-codecommit\.(.+)\.amazonaws.com/v1/repos/(.+)$`) + githubHTTPRegex = regexp.MustCompile(`^https?://.*github.com.*/(.+)/(.+?)(?:.git)?$`) + githubSSHRegex = regexp.MustCompile(`github.com[:/](.+)/(.+?)(?:.git)?$`) + + cloneLock sync.Mutex + + ErrShortRef = errors.New("short SHA references are not supported") + ErrNoRepo = errors.New("unable to find git repo") +) + +type Error struct { + err error + commit string +} + +func (e *Error) Error() string { + return e.err.Error() +} + +func (e *Error) Unwrap() error { + return e.err +} + +func (e *Error) Commit() string { + return e.commit +} + +// FindGitRevision get the current git revision +func FindGitRevision(ctx context.Context, file string) (shortSha string, sha string, err error) { + logger := common.Logger(ctx) + + gitDir, err := git.PlainOpenWithOptions( + file, + &git.PlainOpenOptions{ + DetectDotGit: true, + EnableDotGitCommonDir: true, + }, + ) + + if err != nil { + logger.WithError(err).Error("path", file, "not located inside a git repository") + return "", "", err + } + + head, err := gitDir.Reference(plumbing.HEAD, true) + if err != nil { + return "", "", err + } + + if head.Hash().IsZero() { + return "", "", fmt.Errorf("HEAD sha1 could not be resolved") + } + + hash := head.Hash().String() + + logger.Debugf("Found revision: %s", hash) + return hash[:7], strings.TrimSpace(hash), nil +} + +// FindGitRef get the current git ref +func FindGitRef(ctx context.Context, file string) (string, error) { + logger := common.Logger(ctx) + + logger.Debugf("Loading revision from git directory") + _, ref, err := FindGitRevision(ctx, file) + if err != nil { + return "", err + } + + logger.Debugf("HEAD points to '%s'", ref) + + // Prefer the git library to iterate over the references and find a matching tag or branch. + var refTag = "" + var refBranch = "" + repo, err := git.PlainOpenWithOptions( + file, + &git.PlainOpenOptions{ + DetectDotGit: true, + EnableDotGitCommonDir: true, + }, + ) + + if err != nil { + return "", err + } + + iter, err := repo.References() + if err != nil { + return "", err + } + + // find the reference that matches the revision's has + err = iter.ForEach(func(r *plumbing.Reference) error { + /* tags and branches will have the same hash + * when a user checks out a tag, it is not mentioned explicitly + * in the go-git package, we must identify the revision + * then check if any tag matches that revision, + * if so then we checked out a tag + * else we look for branches and if matches, + * it means we checked out a branch + * + * If a branches matches first we must continue and check all tags (all references) + * in case we match with a tag later in the interation + */ + if r.Hash().String() == ref { + if r.Name().IsTag() { + refTag = r.Name().String() + } + if r.Name().IsBranch() { + refBranch = r.Name().String() + } + } + + // we found what we where looking for + if refTag != "" && refBranch != "" { + return storer.ErrStop + } + + return nil + }) + + if err != nil { + return "", err + } + + // order matters here see above comment. + if refTag != "" { + return refTag, nil + } + if refBranch != "" { + return refBranch, nil + } + + return "", fmt.Errorf("failed to identify reference (tag/branch) for the checked-out revision '%s'", ref) +} + +// FindGithubRepo get the repo +func FindGithubRepo(ctx context.Context, file, githubInstance, remoteName string) (string, error) { + if remoteName == "" { + remoteName = "origin" + } + + url, err := findGitRemoteURL(ctx, file, remoteName) + if err != nil { + return "", err + } + _, slug, err := findGitSlug(url, githubInstance) + return slug, err +} + +func findGitRemoteURL(_ context.Context, file, remoteName string) (string, error) { + repo, err := git.PlainOpenWithOptions( + file, + &git.PlainOpenOptions{ + DetectDotGit: true, + EnableDotGitCommonDir: true, + }, + ) + if err != nil { + return "", err + } + + remote, err := repo.Remote(remoteName) + if err != nil { + return "", err + } + + if len(remote.Config().URLs) < 1 { + return "", fmt.Errorf("remote '%s' exists but has no URL", remoteName) + } + + return remote.Config().URLs[0], nil +} + +func findGitSlug(url string, githubInstance string) (string, string, error) { + if matches := codeCommitHTTPRegex.FindStringSubmatch(url); matches != nil { + return "CodeCommit", matches[2], nil + } else if matches := codeCommitSSHRegex.FindStringSubmatch(url); matches != nil { + return "CodeCommit", matches[2], nil + } else if matches := githubHTTPRegex.FindStringSubmatch(url); matches != nil { + return "GitHub", fmt.Sprintf("%s/%s", matches[1], matches[2]), nil + } else if matches := githubSSHRegex.FindStringSubmatch(url); matches != nil { + return "GitHub", fmt.Sprintf("%s/%s", matches[1], matches[2]), nil + } else if githubInstance != "github.com" { + gheHTTPRegex := regexp.MustCompile(fmt.Sprintf(`^https?://%s/(.+)/(.+?)(?:.git)?$`, githubInstance)) + gheSSHRegex := regexp.MustCompile(fmt.Sprintf(`%s[:/](.+)/(.+?)(?:.git)?$`, githubInstance)) + if matches := gheHTTPRegex.FindStringSubmatch(url); matches != nil { + return "GitHubEnterprise", fmt.Sprintf("%s/%s", matches[1], matches[2]), nil + } else if matches := gheSSHRegex.FindStringSubmatch(url); matches != nil { + return "GitHubEnterprise", fmt.Sprintf("%s/%s", matches[1], matches[2]), nil + } + } + return "", url, nil +} + +// NewGitCloneExecutorInput the input for the NewGitCloneExecutor +type NewGitCloneExecutorInput struct { + URL string + Ref string + Dir string + Token string + OfflineMode bool + + // For Gitea + InsecureSkipTLS bool +} + +// CloneIfRequired ... +func CloneIfRequired(ctx context.Context, refName plumbing.ReferenceName, input NewGitCloneExecutorInput, logger log.FieldLogger) (*git.Repository, error) { + r, err := git.PlainOpen(input.Dir) + if err != nil { + var progressWriter io.Writer + if isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stdout.Fd()) { + if entry, ok := logger.(*log.Entry); ok { + progressWriter = entry.WriterLevel(log.DebugLevel) + } else if lgr, ok := logger.(*log.Logger); ok { + progressWriter = lgr.WriterLevel(log.DebugLevel) + } else { + log.Errorf("Unable to get writer from logger (type=%T)", logger) + progressWriter = os.Stdout + } + } + + cloneOptions := git.CloneOptions{ + URL: input.URL, + Progress: progressWriter, + + InsecureSkipTLS: input.InsecureSkipTLS, // For Gitea + } + if input.Token != "" { + cloneOptions.Auth = &http.BasicAuth{ + Username: "token", + Password: input.Token, + } + } + + r, err = git.PlainCloneContext(ctx, input.Dir, false, &cloneOptions) + if err != nil { + logger.Errorf("Unable to clone %v %s: %v", input.URL, refName, err) + return nil, err + } + + if err = os.Chmod(input.Dir, 0o755); err != nil { + return nil, err + } + } + + return r, nil +} + +func gitOptions(token string) (fetchOptions git.FetchOptions, pullOptions git.PullOptions) { + fetchOptions.RefSpecs = []config.RefSpec{"refs/*:refs/*", "HEAD:refs/heads/HEAD"} + pullOptions.Force = true + + if token != "" { + auth := &http.BasicAuth{ + Username: "token", + Password: token, + } + fetchOptions.Auth = auth + pullOptions.Auth = auth + } + + return fetchOptions, pullOptions +} + +// NewGitCloneExecutor creates an executor to clone git repos +// +//nolint:gocyclo +func NewGitCloneExecutor(input NewGitCloneExecutorInput) common.Executor { + return func(ctx context.Context) error { + logger := common.Logger(ctx) + logger.Infof(" \u2601 git clone '%s' # ref=%s", input.URL, input.Ref) + logger.Debugf(" cloning %s to %s", input.URL, input.Dir) + + cloneLock.Lock() + defer cloneLock.Unlock() + + refName := plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", input.Ref)) + r, err := CloneIfRequired(ctx, refName, input, logger) + if err != nil { + return err + } + + isOfflineMode := input.OfflineMode + + // fetch latest changes + fetchOptions, pullOptions := gitOptions(input.Token) + + if input.InsecureSkipTLS { // For Gitea + fetchOptions.InsecureSkipTLS = true + pullOptions.InsecureSkipTLS = true + } + + if !isOfflineMode { + err = r.Fetch(&fetchOptions) + if err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) { + return err + } + } + + var hash *plumbing.Hash + rev := plumbing.Revision(input.Ref) + if hash, err = r.ResolveRevision(rev); err != nil { + logger.Errorf("Unable to resolve %s: %v", input.Ref, err) + } + + if hash.String() != input.Ref && strings.HasPrefix(hash.String(), input.Ref) { + return &Error{ + err: ErrShortRef, + commit: hash.String(), + } + } + + // At this point we need to know if it's a tag or a branch + // And the easiest way to do it is duck typing + // + // If err is nil, it's a tag so let's proceed with that hash like we would if + // it was a sha + refType := "tag" + rev = plumbing.Revision(path.Join("refs", "tags", input.Ref)) + if _, err := r.Tag(input.Ref); errors.Is(err, git.ErrTagNotFound) { + rName := plumbing.ReferenceName(path.Join("refs", "remotes", "origin", input.Ref)) + if _, err := r.Reference(rName, false); errors.Is(err, plumbing.ErrReferenceNotFound) { + refType = "sha" + rev = plumbing.Revision(input.Ref) + } else { + refType = "branch" + rev = plumbing.Revision(rName) + } + } + + if hash, err = r.ResolveRevision(rev); err != nil { + logger.Errorf("Unable to resolve %s: %v", input.Ref, err) + return err + } + + var w *git.Worktree + if w, err = r.Worktree(); err != nil { + return err + } + + // If the hash resolved doesn't match the ref provided in a workflow then we're + // using a branch or tag ref, not a sha + // + // Repos on disk point to commit hashes, and need to checkout input.Ref before + // we try and pull down any changes + if hash.String() != input.Ref && refType == "branch" { + logger.Debugf("Provided ref is not a sha. Checking out branch before pulling changes") + sourceRef := plumbing.ReferenceName(path.Join("refs", "remotes", "origin", input.Ref)) + if err = w.Checkout(&git.CheckoutOptions{ + Branch: sourceRef, + Force: true, + }); err != nil { + logger.Errorf("Unable to checkout %s: %v", sourceRef, err) + return err + } + } + if !isOfflineMode { + if err = w.Pull(&pullOptions); err != nil && err != git.NoErrAlreadyUpToDate { + logger.Debugf("Unable to pull %s: %v", refName, err) + } + } + logger.Debugf("Cloned %s to %s", input.URL, input.Dir) + + if hash.String() != input.Ref && refType == "branch" { + logger.Debugf("Provided ref is not a sha. Updating branch ref after pull") + if hash, err = r.ResolveRevision(rev); err != nil { + logger.Errorf("Unable to resolve %s: %v", input.Ref, err) + return err + } + } + if err = w.Checkout(&git.CheckoutOptions{ + Hash: *hash, + Force: true, + }); err != nil { + logger.Errorf("Unable to checkout %s: %v", *hash, err) + return err + } + + if err = w.Reset(&git.ResetOptions{ + Mode: git.HardReset, + Commit: *hash, + }); err != nil { + logger.Errorf("Unable to reset to %s: %v", hash.String(), err) + return err + } + + logger.Debugf("Checked out %s", input.Ref) + return nil + } +} diff --git a/pkg/common/git/git_test.go b/pkg/common/git/git_test.go new file mode 100644 index 0000000..6ad66b6 --- /dev/null +++ b/pkg/common/git/git_test.go @@ -0,0 +1,249 @@ +package git + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "syscall" + "testing" + + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFindGitSlug(t *testing.T) { + assert := assert.New(t) + + var slugTests = []struct { + url string // input + provider string // expected result + slug string // expected result + }{ + {"https://git-codecommit.us-east-1.amazonaws.com/v1/repos/my-repo-name", "CodeCommit", "my-repo-name"}, + {"ssh://git-codecommit.us-west-2.amazonaws.com/v1/repos/my-repo", "CodeCommit", "my-repo"}, + {"git@github.com:nektos/act.git", "GitHub", "nektos/act"}, + {"git@github.com:nektos/act", "GitHub", "nektos/act"}, + {"https://github.com/nektos/act.git", "GitHub", "nektos/act"}, + {"http://github.com/nektos/act.git", "GitHub", "nektos/act"}, + {"https://github.com/nektos/act", "GitHub", "nektos/act"}, + {"http://github.com/nektos/act", "GitHub", "nektos/act"}, + {"git+ssh://git@github.com/owner/repo.git", "GitHub", "owner/repo"}, + {"http://myotherrepo.com/act.git", "", "http://myotherrepo.com/act.git"}, + } + + for _, tt := range slugTests { + provider, slug, err := findGitSlug(tt.url, "github.com") + + assert.NoError(err) + assert.Equal(tt.provider, provider) + assert.Equal(tt.slug, slug) + } +} + +func testDir(t *testing.T) string { + basedir, err := os.MkdirTemp("", "act-test") + require.NoError(t, err) + t.Cleanup(func() { _ = os.RemoveAll(basedir) }) + return basedir +} + +func cleanGitHooks(dir string) error { + hooksDir := filepath.Join(dir, ".git", "hooks") + files, err := os.ReadDir(hooksDir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + for _, f := range files { + if f.IsDir() { + continue + } + relName := filepath.Join(hooksDir, f.Name()) + if err := os.Remove(relName); err != nil { + return err + } + } + return nil +} + +func TestFindGitRemoteURL(t *testing.T) { + assert := assert.New(t) + + basedir := testDir(t) + gitConfig() + err := gitCmd("init", basedir) + assert.NoError(err) + err = cleanGitHooks(basedir) + assert.NoError(err) + + remoteURL := "https://git-codecommit.us-east-1.amazonaws.com/v1/repos/my-repo-name" + err = gitCmd("-C", basedir, "remote", "add", "origin", remoteURL) + assert.NoError(err) + + u, err := findGitRemoteURL(context.Background(), basedir, "origin") + assert.NoError(err) + assert.Equal(remoteURL, u) + + remoteURL = "git@github.com/AwesomeOwner/MyAwesomeRepo.git" + err = gitCmd("-C", basedir, "remote", "add", "upstream", remoteURL) + assert.NoError(err) + u, err = findGitRemoteURL(context.Background(), basedir, "upstream") + assert.NoError(err) + assert.Equal(remoteURL, u) +} + +func TestGitFindRef(t *testing.T) { + basedir := testDir(t) + gitConfig() + + for name, tt := range map[string]struct { + Prepare func(t *testing.T, dir string) + Assert func(t *testing.T, ref string, err error) + }{ + "new_repo": { + Prepare: func(t *testing.T, dir string) {}, + Assert: func(t *testing.T, ref string, err error) { + require.Error(t, err) + }, + }, + "new_repo_with_commit": { + Prepare: func(t *testing.T, dir string) { + require.NoError(t, gitCmd("-C", dir, "commit", "--allow-empty", "-m", "msg")) + }, + Assert: func(t *testing.T, ref string, err error) { + require.NoError(t, err) + require.Equal(t, "refs/heads/master", ref) + }, + }, + "current_head_is_tag": { + Prepare: func(t *testing.T, dir string) { + require.NoError(t, gitCmd("-C", dir, "commit", "--allow-empty", "-m", "commit msg")) + require.NoError(t, gitCmd("-C", dir, "tag", "v1.2.3")) + require.NoError(t, gitCmd("-C", dir, "checkout", "v1.2.3")) + }, + Assert: func(t *testing.T, ref string, err error) { + require.NoError(t, err) + require.Equal(t, "refs/tags/v1.2.3", ref) + }, + }, + "current_head_is_same_as_tag": { + Prepare: func(t *testing.T, dir string) { + require.NoError(t, gitCmd("-C", dir, "commit", "--allow-empty", "-m", "1.4.2 release")) + require.NoError(t, gitCmd("-C", dir, "tag", "v1.4.2")) + }, + Assert: func(t *testing.T, ref string, err error) { + require.NoError(t, err) + require.Equal(t, "refs/tags/v1.4.2", ref) + }, + }, + "current_head_is_not_tag": { + Prepare: func(t *testing.T, dir string) { + require.NoError(t, gitCmd("-C", dir, "commit", "--allow-empty", "-m", "msg")) + require.NoError(t, gitCmd("-C", dir, "tag", "v1.4.2")) + require.NoError(t, gitCmd("-C", dir, "commit", "--allow-empty", "-m", "msg2")) + }, + Assert: func(t *testing.T, ref string, err error) { + require.NoError(t, err) + require.Equal(t, "refs/heads/master", ref) + }, + }, + "current_head_is_another_branch": { + Prepare: func(t *testing.T, dir string) { + require.NoError(t, gitCmd("-C", dir, "checkout", "-b", "mybranch")) + require.NoError(t, gitCmd("-C", dir, "commit", "--allow-empty", "-m", "msg")) + }, + Assert: func(t *testing.T, ref string, err error) { + require.NoError(t, err) + require.Equal(t, "refs/heads/mybranch", ref) + }, + }, + } { + tt := tt + name := name + t.Run(name, func(t *testing.T) { + dir := filepath.Join(basedir, name) + require.NoError(t, os.MkdirAll(dir, 0o755)) + require.NoError(t, gitCmd("-C", dir, "init", "--initial-branch=master")) + require.NoError(t, cleanGitHooks(dir)) + tt.Prepare(t, dir) + ref, err := FindGitRef(context.Background(), dir) + tt.Assert(t, ref, err) + }) + } +} + +func TestGitCloneExecutor(t *testing.T) { + for name, tt := range map[string]struct { + Err error + URL, Ref string + }{ + "tag": { + Err: nil, + URL: "https://github.com/actions/checkout", + Ref: "v2", + }, + "branch": { + Err: nil, + URL: "https://github.com/anchore/scan-action", + Ref: "act-fails", + }, + "sha": { + Err: nil, + URL: "https://github.com/actions/checkout", + Ref: "5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f", // v2 + }, + "short-sha": { + Err: &Error{ErrShortRef, "5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f"}, + URL: "https://github.com/actions/checkout", + Ref: "5a4ac90", // v2 + }, + } { + t.Run(name, func(t *testing.T) { + clone := NewGitCloneExecutor(NewGitCloneExecutorInput{ + URL: tt.URL, + Ref: tt.Ref, + Dir: testDir(t), + }) + + err := clone(context.Background()) + if tt.Err != nil { + assert.Error(t, err) + assert.Equal(t, tt.Err, err) + } else { + assert.Empty(t, err) + } + }) + } +} + +func gitConfig() { + if os.Getenv("GITHUB_ACTIONS") == "true" { + var err error + if err = gitCmd("config", "--global", "user.email", "test@test.com"); err != nil { + log.Error(err) + } + if err = gitCmd("config", "--global", "user.name", "Unit Test"); err != nil { + log.Error(err) + } + } +} + +func gitCmd(args ...string) error { + cmd := exec.Command("git", args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + err := cmd.Run() + if exitError, ok := err.(*exec.ExitError); ok { + if waitStatus, ok := exitError.Sys().(syscall.WaitStatus); ok { + return fmt.Errorf("Exit error %d", waitStatus.ExitStatus()) + } + return exitError + } + return nil +} diff --git a/pkg/common/job_error.go b/pkg/common/job_error.go new file mode 100644 index 0000000..334c6ca --- /dev/null +++ b/pkg/common/job_error.go @@ -0,0 +1,30 @@ +package common + +import ( + "context" +) + +type jobErrorContextKey string + +const jobErrorContextKeyVal = jobErrorContextKey("job.error") + +// JobError returns the job error for current context if any +func JobError(ctx context.Context) error { + val := ctx.Value(jobErrorContextKeyVal) + if val != nil { + if container, ok := val.(map[string]error); ok { + return container["error"] + } + } + return nil +} + +func SetJobError(ctx context.Context, err error) { + ctx.Value(jobErrorContextKeyVal).(map[string]error)["error"] = err +} + +// WithJobErrorContainer adds a value to the context as a container for an error +func WithJobErrorContainer(ctx context.Context) context.Context { + container := map[string]error{} + return context.WithValue(ctx, jobErrorContextKeyVal, container) +} diff --git a/pkg/common/line_writer.go b/pkg/common/line_writer.go new file mode 100644 index 0000000..2035199 --- /dev/null +++ b/pkg/common/line_writer.go @@ -0,0 +1,50 @@ +package common + +import ( + "bytes" + "io" +) + +// LineHandler is a callback function for handling a line +type LineHandler func(line string) bool + +type lineWriter struct { + buffer bytes.Buffer + handlers []LineHandler +} + +// NewLineWriter creates a new instance of a line writer +func NewLineWriter(handlers ...LineHandler) io.Writer { + w := new(lineWriter) + w.handlers = handlers + return w +} + +func (lw *lineWriter) Write(p []byte) (n int, err error) { + pBuf := bytes.NewBuffer(p) + written := 0 + for { + line, err := pBuf.ReadString('\n') + w, _ := lw.buffer.WriteString(line) + written += w + if err == nil { + lw.handleLine(lw.buffer.String()) + lw.buffer.Reset() + } else if err == io.EOF { + break + } else { + return written, err + } + } + + return written, nil +} + +func (lw *lineWriter) handleLine(line string) { + for _, h := range lw.handlers { + ok := h(line) + if !ok { + break + } + } +} diff --git a/pkg/common/line_writer_test.go b/pkg/common/line_writer_test.go new file mode 100644 index 0000000..44e11ef --- /dev/null +++ b/pkg/common/line_writer_test.go @@ -0,0 +1,37 @@ +package common + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestLineWriter(t *testing.T) { + lines := make([]string, 0) + lineHandler := func(s string) bool { + lines = append(lines, s) + return true + } + + lineWriter := NewLineWriter(lineHandler) + + assert := assert.New(t) + write := func(s string) { + n, err := lineWriter.Write([]byte(s)) + assert.NoError(err) + assert.Equal(len(s), n, s) + } + + write("hello") + write(" ") + write("world!!\nextra") + write(" line\n and another\nlast") + write(" line\n") + write("no newline here...") + + assert.Len(lines, 4) + assert.Equal("hello world!!\n", lines[0]) + assert.Equal("extra line\n", lines[1]) + assert.Equal(" and another\n", lines[2]) + assert.Equal("last line\n", lines[3]) +} diff --git a/pkg/common/logger.go b/pkg/common/logger.go new file mode 100644 index 0000000..74fc96d --- /dev/null +++ b/pkg/common/logger.go @@ -0,0 +1,48 @@ +package common + +import ( + "context" + + "github.com/sirupsen/logrus" +) + +type loggerContextKey string + +const loggerContextKeyVal = loggerContextKey("logrus.FieldLogger") + +// Logger returns the appropriate logger for current context +func Logger(ctx context.Context) logrus.FieldLogger { + val := ctx.Value(loggerContextKeyVal) + if val != nil { + if logger, ok := val.(logrus.FieldLogger); ok { + return logger + } + } + return logrus.StandardLogger() +} + +// WithLogger adds a value to the context for the logger +func WithLogger(ctx context.Context, logger logrus.FieldLogger) context.Context { + return context.WithValue(ctx, loggerContextKeyVal, logger) +} + +type loggerHookKey string + +const loggerHookKeyVal = loggerHookKey("logrus.Hook") + +// LoggerHook returns the appropriate logger hook for current context +// the hook affects job logger, not global logger +func LoggerHook(ctx context.Context) logrus.Hook { + val := ctx.Value(loggerHookKeyVal) + if val != nil { + if hook, ok := val.(logrus.Hook); ok { + return hook + } + } + return nil +} + +// WithLoggerHook adds a value to the context for the logger hook +func WithLoggerHook(ctx context.Context, hook logrus.Hook) context.Context { + return context.WithValue(ctx, loggerHookKeyVal, hook) +} diff --git a/pkg/common/outbound_ip.go b/pkg/common/outbound_ip.go new file mode 100644 index 0000000..66e15e5 --- /dev/null +++ b/pkg/common/outbound_ip.go @@ -0,0 +1,75 @@ +package common + +import ( + "net" + "sort" + "strings" +) + +// GetOutboundIP returns an outbound IP address of this machine. +// It tries to access the internet and returns the local IP address of the connection. +// If the machine cannot access the internet, it returns a preferred IP address from network interfaces. +// It returns nil if no IP address is found. +func GetOutboundIP() net.IP { + // See https://stackoverflow.com/a/37382208 + conn, err := net.Dial("udp", "8.8.8.8:80") + if err == nil { + defer conn.Close() + return conn.LocalAddr().(*net.UDPAddr).IP + } + + // So the machine cannot access the internet. Pick an IP address from network interfaces. + if ifs, err := net.Interfaces(); err == nil { + type IP struct { + net.IP + net.Interface + } + var ips []IP + for _, i := range ifs { + if addrs, err := i.Addrs(); err == nil { + for _, addr := range addrs { + var ip net.IP + switch v := addr.(type) { + case *net.IPNet: + ip = v.IP + case *net.IPAddr: + ip = v.IP + } + if ip.IsGlobalUnicast() { + ips = append(ips, IP{ip, i}) + } + } + } + } + if len(ips) > 1 { + sort.Slice(ips, func(i, j int) bool { + ifi := ips[i].Interface + ifj := ips[j].Interface + + // ethernet is preferred + if vi, vj := strings.HasPrefix(ifi.Name, "e"), strings.HasPrefix(ifj.Name, "e"); vi != vj { + return vi + } + + ipi := ips[i].IP + ipj := ips[j].IP + + // IPv4 is preferred + if vi, vj := ipi.To4() != nil, ipj.To4() != nil; vi != vj { + return vi + } + + // en0 is preferred to en1 + if ifi.Name != ifj.Name { + return ifi.Name < ifj.Name + } + + // fallback + return ipi.String() < ipj.String() + }) + return ips[0].IP + } + } + + return nil +} |