diff options
Diffstat (limited to 'modules/httplib')
| -rw-r--r-- | modules/httplib/request.go | 206 | ||||
| -rw-r--r-- | modules/httplib/serve.go | 237 | ||||
| -rw-r--r-- | modules/httplib/serve_test.go | 109 | ||||
| -rw-r--r-- | modules/httplib/url.go | 27 | ||||
| -rw-r--r-- | modules/httplib/url_test.go | 123 |
5 files changed, 702 insertions, 0 deletions
diff --git a/modules/httplib/request.go b/modules/httplib/request.go new file mode 100644 index 0000000..880d7ad --- /dev/null +++ b/modules/httplib/request.go @@ -0,0 +1,206 @@ +// Copyright 2013 The Beego Authors. All rights reserved. +// Copyright 2014 The Gogs Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package httplib + +import ( + "bytes" + "context" + "crypto/tls" + "fmt" + "io" + "net" + "net/http" + "net/url" + "strings" + "time" +) + +var defaultSetting = Settings{"GiteaServer", 60 * time.Second, 60 * time.Second, nil, nil} + +// newRequest returns *Request with specific method +func newRequest(url, method string) *Request { + var resp http.Response + req := http.Request{ + Method: method, + Header: make(http.Header), + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + } + return &Request{url, &req, map[string]string{}, defaultSetting, &resp, nil} +} + +// NewRequest returns *Request with specific method +func NewRequest(url, method string) *Request { + return newRequest(url, method) +} + +// Settings is the default settings for http client +type Settings struct { + UserAgent string + ConnectTimeout time.Duration + ReadWriteTimeout time.Duration + TLSClientConfig *tls.Config + Transport http.RoundTripper +} + +// Request provides more useful methods for requesting one url than http.Request. +type Request struct { + url string + req *http.Request + params map[string]string + setting Settings + resp *http.Response + body []byte +} + +// SetContext sets the request's Context +func (r *Request) SetContext(ctx context.Context) *Request { + r.req = r.req.WithContext(ctx) + return r +} + +// SetTimeout sets connect time out and read-write time out for BeegoRequest. +func (r *Request) SetTimeout(connectTimeout, readWriteTimeout time.Duration) *Request { + r.setting.ConnectTimeout = connectTimeout + r.setting.ReadWriteTimeout = readWriteTimeout + return r +} + +func (r *Request) SetReadWriteTimeout(readWriteTimeout time.Duration) *Request { + r.setting.ReadWriteTimeout = readWriteTimeout + return r +} + +// SetTLSClientConfig sets tls connection configurations if visiting https url. +func (r *Request) SetTLSClientConfig(config *tls.Config) *Request { + r.setting.TLSClientConfig = config + return r +} + +// Header add header item string in request. +func (r *Request) Header(key, value string) *Request { + r.req.Header.Set(key, value) + return r +} + +// SetTransport sets transport to +func (r *Request) SetTransport(transport http.RoundTripper) *Request { + r.setting.Transport = transport + return r +} + +// Param adds query param in to request. +// params build query string as ?key1=value1&key2=value2... +func (r *Request) Param(key, value string) *Request { + r.params[key] = value + return r +} + +// Body adds request raw body. +// it supports string and []byte. +func (r *Request) Body(data any) *Request { + switch t := data.(type) { + case string: + bf := bytes.NewBufferString(t) + r.req.Body = io.NopCloser(bf) + r.req.ContentLength = int64(len(t)) + case []byte: + bf := bytes.NewBuffer(t) + r.req.Body = io.NopCloser(bf) + r.req.ContentLength = int64(len(t)) + } + return r +} + +func (r *Request) getResponse() (*http.Response, error) { + if r.resp.StatusCode != 0 { + return r.resp, nil + } + + var paramBody string + if len(r.params) > 0 { + var buf bytes.Buffer + for k, v := range r.params { + buf.WriteString(url.QueryEscape(k)) + buf.WriteByte('=') + buf.WriteString(url.QueryEscape(v)) + buf.WriteByte('&') + } + paramBody = buf.String() + paramBody = paramBody[0 : len(paramBody)-1] + } + + if r.req.Method == "GET" && len(paramBody) > 0 { + if strings.Contains(r.url, "?") { + r.url += "&" + paramBody + } else { + r.url = r.url + "?" + paramBody + } + } else if r.req.Method == "POST" && r.req.Body == nil && len(paramBody) > 0 { + r.Header("Content-Type", "application/x-www-form-urlencoded") + r.Body(paramBody) + } + + var err error + r.req.URL, err = url.Parse(r.url) + if err != nil { + return nil, err + } + + trans := r.setting.Transport + if trans == nil { + // create default transport + trans = &http.Transport{ + TLSClientConfig: r.setting.TLSClientConfig, + Proxy: http.ProxyFromEnvironment, + DialContext: TimeoutDialer(r.setting.ConnectTimeout), + } + } else if t, ok := trans.(*http.Transport); ok { + if t.TLSClientConfig == nil { + t.TLSClientConfig = r.setting.TLSClientConfig + } + if t.DialContext == nil { + t.DialContext = TimeoutDialer(r.setting.ConnectTimeout) + } + } + + client := &http.Client{ + Transport: trans, + Timeout: r.setting.ReadWriteTimeout, + } + + if len(r.setting.UserAgent) > 0 && len(r.req.Header.Get("User-Agent")) == 0 { + r.req.Header.Set("User-Agent", r.setting.UserAgent) + } + + resp, err := client.Do(r.req) + if err != nil { + return nil, err + } + r.resp = resp + return resp, nil +} + +// Response executes request client gets response manually. +func (r *Request) Response() (*http.Response, error) { + return r.getResponse() +} + +// TimeoutDialer returns functions of connection dialer with timeout settings for http.Transport Dial field. +func TimeoutDialer(cTimeout time.Duration) func(ctx context.Context, net, addr string) (c net.Conn, err error) { + return func(ctx context.Context, netw, addr string) (net.Conn, error) { + d := net.Dialer{Timeout: cTimeout} + conn, err := d.DialContext(ctx, netw, addr) + if err != nil { + return nil, err + } + return conn, nil + } +} + +func (r *Request) GoString() string { + return fmt.Sprintf("%s %s", r.req.Method, r.url) +} diff --git a/modules/httplib/serve.go b/modules/httplib/serve.go new file mode 100644 index 0000000..2e3e6a7 --- /dev/null +++ b/modules/httplib/serve.go @@ -0,0 +1,237 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package httplib + +import ( + "bytes" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "path" + "path/filepath" + "strconv" + "strings" + "time" + + charsetModule "code.gitea.io/gitea/modules/charset" + "code.gitea.io/gitea/modules/container" + "code.gitea.io/gitea/modules/httpcache" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/typesniffer" + "code.gitea.io/gitea/modules/util" + + "github.com/klauspost/compress/gzhttp" +) + +type ServeHeaderOptions struct { + ContentType string // defaults to "application/octet-stream" + ContentTypeCharset string + ContentLength *int64 + Disposition string // defaults to "attachment" + Filename string + CacheDuration time.Duration // defaults to 5 minutes + LastModified time.Time +} + +// ServeSetHeaders sets necessary content serve headers +func ServeSetHeaders(w http.ResponseWriter, opts *ServeHeaderOptions) { + header := w.Header() + + skipCompressionExts := container.SetOf(".gz", ".bz2", ".zip", ".xz", ".zst", ".deb", ".apk", ".jar", ".png", ".jpg", ".webp") + if skipCompressionExts.Contains(strings.ToLower(path.Ext(opts.Filename))) { + w.Header().Add(gzhttp.HeaderNoCompression, "1") + } + + contentType := typesniffer.ApplicationOctetStream + if opts.ContentType != "" { + if opts.ContentTypeCharset != "" { + contentType = opts.ContentType + "; charset=" + strings.ToLower(opts.ContentTypeCharset) + } else { + contentType = opts.ContentType + } + } + header.Set("Content-Type", contentType) + header.Set("X-Content-Type-Options", "nosniff") + + if opts.ContentLength != nil { + header.Set("Content-Length", strconv.FormatInt(*opts.ContentLength, 10)) + } + + if opts.Filename != "" { + disposition := opts.Disposition + if disposition == "" { + disposition = "attachment" + } + + backslashEscapedName := strings.ReplaceAll(strings.ReplaceAll(opts.Filename, `\`, `\\`), `"`, `\"`) // \ -> \\, " -> \" + header.Set("Content-Disposition", fmt.Sprintf(`%s; filename="%s"; filename*=UTF-8''%s`, disposition, backslashEscapedName, url.PathEscape(opts.Filename))) + header.Set("Access-Control-Expose-Headers", "Content-Disposition") + } + + duration := opts.CacheDuration + if duration == 0 { + duration = 5 * time.Minute + } + httpcache.SetCacheControlInHeader(header, duration) + + if !opts.LastModified.IsZero() { + // http.TimeFormat required a UTC time, refer to https://pkg.go.dev/net/http#TimeFormat + header.Set("Last-Modified", opts.LastModified.UTC().Format(http.TimeFormat)) + } +} + +// ServeData download file from io.Reader +func setServeHeadersByFile(r *http.Request, w http.ResponseWriter, filePath string, mineBuf []byte) { + // do not set "Content-Length", because the length could only be set by callers, and it needs to support range requests + opts := &ServeHeaderOptions{ + Filename: path.Base(filePath), + } + + sniffedType := typesniffer.DetectContentType(mineBuf) + + // the "render" parameter came from year 2016: 638dd24c, it doesn't have clear meaning, so I think it could be removed later + isPlain := sniffedType.IsText() || r.FormValue("render") != "" + + if setting.MimeTypeMap.Enabled { + fileExtension := strings.ToLower(filepath.Ext(filePath)) + opts.ContentType = setting.MimeTypeMap.Map[fileExtension] + } + + if opts.ContentType == "" { + if sniffedType.IsBrowsableBinaryType() { + opts.ContentType = sniffedType.GetMimeType() + } else if isPlain { + opts.ContentType = "text/plain" + } else { + opts.ContentType = typesniffer.ApplicationOctetStream + } + } + + if isPlain { + charset, err := charsetModule.DetectEncoding(mineBuf) + if err != nil { + log.Error("Detect raw file %s charset failed: %v, using by default utf-8", filePath, err) + charset = "utf-8" + } + opts.ContentTypeCharset = strings.ToLower(charset) + } + + isSVG := sniffedType.IsSvgImage() + + // serve types that can present a security risk with CSP + if isSVG { + w.Header().Set("Content-Security-Policy", "default-src 'none'; style-src 'unsafe-inline'; sandbox") + } else if sniffedType.IsPDF() { + // no sandbox attribute for pdf as it breaks rendering in at least safari. this + // should generally be safe as scripts inside PDF can not escape the PDF document + // see https://bugs.chromium.org/p/chromium/issues/detail?id=413851 for more discussion + w.Header().Set("Content-Security-Policy", "default-src 'none'; style-src 'unsafe-inline'") + } + + opts.Disposition = "inline" + if isSVG && !setting.UI.SVG.Enabled { + opts.Disposition = "attachment" + } + + ServeSetHeaders(w, opts) +} + +const mimeDetectionBufferLen = 1024 + +func ServeContentByReader(r *http.Request, w http.ResponseWriter, filePath string, size int64, reader io.Reader) { + buf := make([]byte, mimeDetectionBufferLen) + n, err := util.ReadAtMost(reader, buf) + if err != nil { + http.Error(w, "serve content: unable to pre-read", http.StatusRequestedRangeNotSatisfiable) + return + } + if n >= 0 { + buf = buf[:n] + } + setServeHeadersByFile(r, w, filePath, buf) + + // reset the reader to the beginning + reader = io.MultiReader(bytes.NewReader(buf), reader) + + rangeHeader := r.Header.Get("Range") + + // if no size or no supported range, serve as 200 (complete response) + if size <= 0 || !strings.HasPrefix(rangeHeader, "bytes=") { + if size >= 0 { + w.Header().Set("Content-Length", strconv.FormatInt(size, 10)) + } + _, _ = io.Copy(w, reader) // just like http.ServeContent, not necessary to handle the error + return + } + + // do our best to support the minimal "Range" request (no support for multiple range: "Range: bytes=0-50, 100-150") + // + // GET /... + // Range: bytes=0-1023 + // + // HTTP/1.1 206 Partial Content + // Content-Range: bytes 0-1023/146515 + // Content-Length: 1024 + + _, rangeParts, _ := strings.Cut(rangeHeader, "=") + rangeBytesStart, rangeBytesEnd, found := strings.Cut(rangeParts, "-") + start, err := strconv.ParseInt(rangeBytesStart, 10, 64) + if start < 0 || start >= size { + err = errors.New("invalid start range") + } + if err != nil { + http.Error(w, err.Error(), http.StatusRequestedRangeNotSatisfiable) + return + } + end, err := strconv.ParseInt(rangeBytesEnd, 10, 64) + if rangeBytesEnd == "" && found { + err = nil + end = size - 1 + } + if end >= size { + end = size - 1 + } + if end < start { + err = errors.New("invalid end range") + } + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + partialLength := end - start + 1 + w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, size)) + w.Header().Set("Content-Length", strconv.FormatInt(partialLength, 10)) + if _, err = io.CopyN(io.Discard, reader, start); err != nil { + http.Error(w, "serve content: unable to skip", http.StatusInternalServerError) + return + } + + w.WriteHeader(http.StatusPartialContent) + _, _ = io.CopyN(w, reader, partialLength) // just like http.ServeContent, not necessary to handle the error +} + +func ServeContentByReadSeeker(r *http.Request, w http.ResponseWriter, filePath string, modTime *time.Time, reader io.ReadSeeker) { + buf := make([]byte, mimeDetectionBufferLen) + n, err := util.ReadAtMost(reader, buf) + if err != nil { + http.Error(w, "serve content: unable to read", http.StatusInternalServerError) + return + } + if _, err = reader.Seek(0, io.SeekStart); err != nil { + http.Error(w, "serve content: unable to seek", http.StatusInternalServerError) + return + } + if n >= 0 { + buf = buf[:n] + } + setServeHeadersByFile(r, w, filePath, buf) + if modTime == nil { + modTime = &time.Time{} + } + http.ServeContent(w, r, path.Base(filePath), *modTime, reader) +} diff --git a/modules/httplib/serve_test.go b/modules/httplib/serve_test.go new file mode 100644 index 0000000..fe609e1 --- /dev/null +++ b/modules/httplib/serve_test.go @@ -0,0 +1,109 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package httplib + +import ( + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "os" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestServeContentByReader(t *testing.T) { + data := "0123456789abcdef" + + test := func(t *testing.T, expectedStatusCode int, expectedContent string) { + _, rangeStr, _ := strings.Cut(t.Name(), "_range_") + r := &http.Request{Header: http.Header{}, Form: url.Values{}} + if rangeStr != "" { + r.Header.Set("Range", fmt.Sprintf("bytes=%s", rangeStr)) + } + reader := strings.NewReader(data) + w := httptest.NewRecorder() + ServeContentByReader(r, w, "test", int64(len(data)), reader) + assert.Equal(t, expectedStatusCode, w.Code) + if expectedStatusCode == http.StatusPartialContent || expectedStatusCode == http.StatusOK { + assert.Equal(t, fmt.Sprint(len(expectedContent)), w.Header().Get("Content-Length")) + assert.Equal(t, expectedContent, w.Body.String()) + } + } + + t.Run("_range_", func(t *testing.T) { + test(t, http.StatusOK, data) + }) + t.Run("_range_0-", func(t *testing.T) { + test(t, http.StatusPartialContent, data) + }) + t.Run("_range_0-15", func(t *testing.T) { + test(t, http.StatusPartialContent, data) + }) + t.Run("_range_1-", func(t *testing.T) { + test(t, http.StatusPartialContent, data[1:]) + }) + t.Run("_range_1-3", func(t *testing.T) { + test(t, http.StatusPartialContent, data[1:3+1]) + }) + t.Run("_range_16-", func(t *testing.T) { + test(t, http.StatusRequestedRangeNotSatisfiable, "") + }) + t.Run("_range_1-99999", func(t *testing.T) { + test(t, http.StatusPartialContent, data[1:]) + }) +} + +func TestServeContentByReadSeeker(t *testing.T) { + data := "0123456789abcdef" + tmpFile := t.TempDir() + "/test" + err := os.WriteFile(tmpFile, []byte(data), 0o644) + require.NoError(t, err) + + test := func(t *testing.T, expectedStatusCode int, expectedContent string) { + _, rangeStr, _ := strings.Cut(t.Name(), "_range_") + r := &http.Request{Header: http.Header{}, Form: url.Values{}} + if rangeStr != "" { + r.Header.Set("Range", fmt.Sprintf("bytes=%s", rangeStr)) + } + + seekReader, err := os.OpenFile(tmpFile, os.O_RDONLY, 0o644) + require.NoError(t, err) + + defer seekReader.Close() + + w := httptest.NewRecorder() + ServeContentByReadSeeker(r, w, "test", nil, seekReader) + assert.Equal(t, expectedStatusCode, w.Code) + if expectedStatusCode == http.StatusPartialContent || expectedStatusCode == http.StatusOK { + assert.Equal(t, fmt.Sprint(len(expectedContent)), w.Header().Get("Content-Length")) + assert.Equal(t, expectedContent, w.Body.String()) + } + } + + t.Run("_range_", func(t *testing.T) { + test(t, http.StatusOK, data) + }) + t.Run("_range_0-", func(t *testing.T) { + test(t, http.StatusPartialContent, data) + }) + t.Run("_range_0-15", func(t *testing.T) { + test(t, http.StatusPartialContent, data) + }) + t.Run("_range_1-", func(t *testing.T) { + test(t, http.StatusPartialContent, data[1:]) + }) + t.Run("_range_1-3", func(t *testing.T) { + test(t, http.StatusPartialContent, data[1:3+1]) + }) + t.Run("_range_16-", func(t *testing.T) { + test(t, http.StatusRequestedRangeNotSatisfiable, "") + }) + t.Run("_range_1-99999", func(t *testing.T) { + test(t, http.StatusPartialContent, data[1:]) + }) +} diff --git a/modules/httplib/url.go b/modules/httplib/url.go new file mode 100644 index 0000000..14b9589 --- /dev/null +++ b/modules/httplib/url.go @@ -0,0 +1,27 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package httplib + +import ( + "net/url" + "strings" + + "code.gitea.io/gitea/modules/setting" +) + +// IsRiskyRedirectURL returns true if the URL is considered risky for redirects +func IsRiskyRedirectURL(s string) bool { + // Unfortunately browsers consider a redirect Location with preceding "//", "\\", "/\" and "\/" as meaning redirect to "http(s)://REST_OF_PATH" + // Therefore we should ignore these redirect locations to prevent open redirects + if len(s) > 1 && (s[0] == '/' || s[0] == '\\') && (s[1] == '/' || s[1] == '\\') { + return true + } + + u, err := url.Parse(s) + if err != nil || ((u.Scheme != "" || u.Host != "") && !strings.HasPrefix(strings.ToLower(s), strings.ToLower(setting.AppURL))) { + return true + } + + return false +} diff --git a/modules/httplib/url_test.go b/modules/httplib/url_test.go new file mode 100644 index 0000000..2842edd --- /dev/null +++ b/modules/httplib/url_test.go @@ -0,0 +1,123 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package httplib + +import ( + "testing" + + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/test" + + "github.com/stretchr/testify/assert" +) + +func TestIsRiskyRedirectURL(t *testing.T) { + defer test.MockVariableValue(&setting.AppURL, "http://localhost:3000/sub/")() + defer test.MockVariableValue(&setting.AppSubURL, "/sub")() + + tests := []struct { + input string + want bool + }{ + {"", false}, + {"foo", false}, + {"./", false}, + {"?key=val", false}, + {"/sub/", false}, + {"http://localhost:3000/sub/", false}, + {"/sub/foo", false}, + {"http://localhost:3000/sub/foo", false}, + {"http://localhost:3000/sub/test?param=false", false}, + // FIXME: should probably be true (would requires resolving references using setting.appURL.ResolveReference(u)) + {"/sub/../", false}, + {"http://localhost:3000/sub/../", false}, + {"/sUb/", false}, + {"http://localhost:3000/sUb/foo", false}, + {"/sub", false}, + {"/foo?k=%20#abc", false}, + {"/", false}, + {"a/", false}, + {"test?param=false", false}, + {"/hey/hey/hey#3244", false}, + + {"//", true}, + {"\\\\", true}, + {"/\\", true}, + {"\\/", true}, + {"mail:a@b.com", true}, + {"https://test.com", true}, + {"http://localhost:3000/foo", true}, + {"http://localhost:3000/sub", true}, + {"http://localhost:3000/sub?key=val", true}, + {"https://example.com/", true}, + {"//example.com", true}, + {"http://example.com", true}, + {"http://localhost:3000/test?param=false", true}, + {"//localhost:3000/test?param=false", true}, + {"://missing protocol scheme", true}, + // FIXME: should probably be false + {"//localhost:3000/sub/test?param=false", true}, + } + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + assert.Equal(t, tt.want, IsRiskyRedirectURL(tt.input)) + }) + } +} + +func TestIsRiskyRedirectURLWithoutSubURL(t *testing.T) { + defer test.MockVariableValue(&setting.AppURL, "https://next.forgejo.org/")() + defer test.MockVariableValue(&setting.AppSubURL, "")() + + tests := []struct { + input string + want bool + }{ + {"", false}, + {"foo", false}, + {"./", false}, + {"?key=val", false}, + {"/sub/", false}, + {"https://next.forgejo.org/sub/", false}, + {"/sub/foo", false}, + {"https://next.forgejo.org/sub/foo", false}, + {"https://next.forgejo.org/sub/test?param=false", false}, + {"https://next.forgejo.org/sub/../", false}, + {"/sub/../", false}, + {"/sUb/", false}, + {"https://next.forgejo.org/sUb/foo", false}, + {"/sub", false}, + {"/foo?k=%20#abc", false}, + {"/", false}, + {"a/", false}, + {"test?param=false", false}, + {"/hey/hey/hey#3244", false}, + {"https://next.forgejo.org/test?param=false", false}, + {"https://next.forgejo.org/foo", false}, + {"https://next.forgejo.org/sub", false}, + {"https://next.forgejo.org/sub?key=val", false}, + + {"//", true}, + {"\\\\", true}, + {"/\\", true}, + {"\\/", true}, + {"mail:a@b.com", true}, + {"https://test.com", true}, + {"https://example.com/", true}, + {"//example.com", true}, + {"http://example.com", true}, + {"://missing protocol scheme", true}, + {"https://forgejo.org", true}, + {"https://example.org?url=https://next.forgejo.org", true}, + // FIXME: should probably be false + {"https://next.forgejo.org", true}, + {"//next.forgejo.org/test?param=false", true}, + {"//next.forgejo.org/sub/test?param=false", true}, + } + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + assert.Equal(t, tt.want, IsRiskyRedirectURL(tt.input)) + }) + } +} |