From dd136858f1ea40ad3c94191d647487fa4f31926c Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 18 Oct 2024 20:33:49 +0200 Subject: Adding upstream version 9.0.0. Signed-off-by: Daniel Baumann --- modules/hostmatcher/hostmatcher.go | 161 ++++++++++++++++++++++++++++++++ modules/hostmatcher/hostmatcher_test.go | 161 ++++++++++++++++++++++++++++++++ modules/hostmatcher/http.go | 65 +++++++++++++ 3 files changed, 387 insertions(+) create mode 100644 modules/hostmatcher/hostmatcher.go create mode 100644 modules/hostmatcher/hostmatcher_test.go create mode 100644 modules/hostmatcher/http.go (limited to 'modules/hostmatcher') diff --git a/modules/hostmatcher/hostmatcher.go b/modules/hostmatcher/hostmatcher.go new file mode 100644 index 0000000..1069310 --- /dev/null +++ b/modules/hostmatcher/hostmatcher.go @@ -0,0 +1,161 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package hostmatcher + +import ( + "net" + "path/filepath" + "strings" +) + +// HostMatchList is used to check if a host or IP is in a list. +type HostMatchList struct { + SettingKeyHint string + SettingValue string + + // builtins networks + builtins []string + // patterns for host names (with wildcard support) + patterns []string + // ipNets is the CIDR network list + ipNets []*net.IPNet +} + +// MatchBuiltinExternal A valid non-private unicast IP, all hosts on public internet are matched +const MatchBuiltinExternal = "external" + +// MatchBuiltinPrivate RFC 1918 (10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16) and RFC 4193 (FC00::/7). Also called LAN/Intranet. +const MatchBuiltinPrivate = "private" + +// MatchBuiltinLoopback 127.0.0.0/8 for IPv4 and ::1/128 for IPv6, localhost is included. +const MatchBuiltinLoopback = "loopback" + +func isBuiltin(s string) bool { + return s == MatchBuiltinExternal || s == MatchBuiltinPrivate || s == MatchBuiltinLoopback +} + +// ParseHostMatchList parses the host list HostMatchList +func ParseHostMatchList(settingKeyHint, hostList string) *HostMatchList { + hl := &HostMatchList{SettingKeyHint: settingKeyHint, SettingValue: hostList} + for _, s := range strings.Split(hostList, ",") { + s = strings.ToLower(strings.TrimSpace(s)) + if s == "" { + continue + } + _, ipNet, err := net.ParseCIDR(s) + if err == nil { + hl.ipNets = append(hl.ipNets, ipNet) + } else if isBuiltin(s) { + hl.builtins = append(hl.builtins, s) + } else { + hl.patterns = append(hl.patterns, s) + } + } + return hl +} + +// ParseSimpleMatchList parse a simple matchlist (no built-in networks, no CIDR support, only wildcard pattern match) +func ParseSimpleMatchList(settingKeyHint, matchList string) *HostMatchList { + hl := &HostMatchList{ + SettingKeyHint: settingKeyHint, + SettingValue: matchList, + } + for _, s := range strings.Split(matchList, ",") { + s = strings.ToLower(strings.TrimSpace(s)) + if s == "" { + continue + } + // we keep the same result as old `matchlist`, so no builtin/CIDR support here, we only match wildcard patterns + hl.patterns = append(hl.patterns, s) + } + return hl +} + +// AppendBuiltin appends more builtins to match +func (hl *HostMatchList) AppendBuiltin(builtin string) { + hl.builtins = append(hl.builtins, builtin) +} + +// AppendPattern appends more pattern to match +func (hl *HostMatchList) AppendPattern(pattern string) { + hl.patterns = append(hl.patterns, pattern) +} + +// IsEmpty checks if the checklist is empty +func (hl *HostMatchList) IsEmpty() bool { + return hl == nil || (len(hl.builtins) == 0 && len(hl.patterns) == 0 && len(hl.ipNets) == 0) +} + +func (hl *HostMatchList) checkPattern(host string) bool { + host = strings.ToLower(strings.TrimSpace(host)) + for _, pattern := range hl.patterns { + if matched, _ := filepath.Match(pattern, host); matched { + return true + } + } + return false +} + +func (hl *HostMatchList) checkIP(ip net.IP) bool { + for _, pattern := range hl.patterns { + if pattern == "*" { + return true + } + } + for _, builtin := range hl.builtins { + switch builtin { + case MatchBuiltinExternal: + if ip.IsGlobalUnicast() && !ip.IsPrivate() { + return true + } + case MatchBuiltinPrivate: + if ip.IsPrivate() { + return true + } + case MatchBuiltinLoopback: + if ip.IsLoopback() { + return true + } + } + } + for _, ipNet := range hl.ipNets { + if ipNet.Contains(ip) { + return true + } + } + return false +} + +// MatchHostName checks if the host matches an allow/deny(block) list +func (hl *HostMatchList) MatchHostName(host string) bool { + if hl == nil { + return false + } + + hostname, _, err := net.SplitHostPort(host) + if err != nil { + hostname = host + } + if hl.checkPattern(hostname) { + return true + } + if ip := net.ParseIP(hostname); ip != nil { + return hl.checkIP(ip) + } + return false +} + +// MatchIPAddr checks if the IP matches an allow/deny(block) list, it's safe to pass `nil` to `ip` +func (hl *HostMatchList) MatchIPAddr(ip net.IP) bool { + if hl == nil { + return false + } + host := ip.String() // nil-safe, we will get "" if ip is nil + return hl.checkPattern(host) || hl.checkIP(ip) +} + +// MatchHostOrIP checks if the host or IP matches an allow/deny(block) list +func (hl *HostMatchList) MatchHostOrIP(host string, ip net.IP) bool { + return hl.MatchHostName(host) || hl.MatchIPAddr(ip) +} diff --git a/modules/hostmatcher/hostmatcher_test.go b/modules/hostmatcher/hostmatcher_test.go new file mode 100644 index 0000000..c781847 --- /dev/null +++ b/modules/hostmatcher/hostmatcher_test.go @@ -0,0 +1,161 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package hostmatcher + +import ( + "net" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestHostOrIPMatchesList(t *testing.T) { + type tc struct { + host string + ip net.IP + expected bool + } + + // for IPv6: "::1" is loopback, "fd00::/8" is private + + hl := ParseHostMatchList("", "private, External, *.myDomain.com, 169.254.1.0/24") + + test := func(cases []tc) { + for _, c := range cases { + assert.Equalf(t, c.expected, hl.MatchHostOrIP(c.host, c.ip), "case domain=%s, ip=%v, expected=%v", c.host, c.ip, c.expected) + } + } + + cases := []tc{ + {"", net.IPv4zero, false}, + {"", net.IPv6zero, false}, + + {"", net.ParseIP("127.0.0.1"), false}, + {"127.0.0.1", nil, false}, + {"", net.ParseIP("::1"), false}, + + {"", net.ParseIP("10.0.1.1"), true}, + {"10.0.1.1", nil, true}, + {"10.0.1.1:8080", nil, true}, + {"", net.ParseIP("192.168.1.1"), true}, + {"192.168.1.1", nil, true}, + {"", net.ParseIP("fd00::1"), true}, + {"fd00::1", nil, true}, + + {"", net.ParseIP("8.8.8.8"), true}, + {"", net.ParseIP("1001::1"), true}, + + {"mydomain.com", net.IPv4zero, false}, + {"sub.mydomain.com", net.IPv4zero, true}, + {"sub.mydomain.com:8080", net.IPv4zero, true}, + + {"", net.ParseIP("169.254.1.1"), true}, + {"169.254.1.1", nil, true}, + {"", net.ParseIP("169.254.2.2"), false}, + {"169.254.2.2", nil, false}, + } + test(cases) + + hl = ParseHostMatchList("", "loopback") + cases = []tc{ + {"", net.IPv4zero, false}, + {"", net.ParseIP("127.0.0.1"), true}, + {"", net.ParseIP("10.0.1.1"), false}, + {"", net.ParseIP("192.168.1.1"), false}, + {"", net.ParseIP("8.8.8.8"), false}, + + {"", net.ParseIP("::1"), true}, + {"", net.ParseIP("fd00::1"), false}, + {"", net.ParseIP("1000::1"), false}, + + {"mydomain.com", net.IPv4zero, false}, + } + test(cases) + + hl = ParseHostMatchList("", "private") + cases = []tc{ + {"", net.IPv4zero, false}, + {"", net.ParseIP("127.0.0.1"), false}, + {"", net.ParseIP("10.0.1.1"), true}, + {"", net.ParseIP("192.168.1.1"), true}, + {"", net.ParseIP("8.8.8.8"), false}, + + {"", net.ParseIP("::1"), false}, + {"", net.ParseIP("fd00::1"), true}, + {"", net.ParseIP("1000::1"), false}, + + {"mydomain.com", net.IPv4zero, false}, + } + test(cases) + + hl = ParseHostMatchList("", "external") + cases = []tc{ + {"", net.IPv4zero, false}, + {"", net.ParseIP("127.0.0.1"), false}, + {"", net.ParseIP("10.0.1.1"), false}, + {"", net.ParseIP("192.168.1.1"), false}, + {"", net.ParseIP("8.8.8.8"), true}, + + {"", net.ParseIP("::1"), false}, + {"", net.ParseIP("fd00::1"), false}, + {"", net.ParseIP("1000::1"), true}, + + {"mydomain.com", net.IPv4zero, false}, + } + test(cases) + + hl = ParseHostMatchList("", "*") + cases = []tc{ + {"", net.IPv4zero, true}, + {"", net.ParseIP("127.0.0.1"), true}, + {"", net.ParseIP("10.0.1.1"), true}, + {"", net.ParseIP("192.168.1.1"), true}, + {"", net.ParseIP("8.8.8.8"), true}, + + {"", net.ParseIP("::1"), true}, + {"", net.ParseIP("fd00::1"), true}, + {"", net.ParseIP("1000::1"), true}, + + {"mydomain.com", net.IPv4zero, true}, + } + test(cases) + + // built-in network names can be escaped (warping the first char with `[]`) to be used as a real host name + // this mechanism is reversed for internal usage only (maybe for some rare cases), it's not supposed to be used by end users + // a real user should never use loopback/private/external as their host names + hl = ParseHostMatchList("", "loopback, [p]rivate") + cases = []tc{ + {"loopback", nil, false}, + {"", net.ParseIP("127.0.0.1"), true}, + {"private", nil, true}, + {"", net.ParseIP("192.168.1.1"), false}, + } + test(cases) + + hl = ParseSimpleMatchList("", "loopback, *.domain.com") + cases = []tc{ + {"loopback", nil, true}, + {"", net.ParseIP("127.0.0.1"), false}, + {"sub.domain.com", nil, true}, + {"other.com", nil, false}, + {"", net.ParseIP("1.1.1.1"), false}, + } + test(cases) + + hl = ParseSimpleMatchList("", "external") + cases = []tc{ + {"", net.ParseIP("192.168.1.1"), false}, + {"", net.ParseIP("1.1.1.1"), false}, + {"external", nil, true}, + } + test(cases) + + hl = ParseSimpleMatchList("", "") + cases = []tc{ + {"", net.ParseIP("192.168.1.1"), false}, + {"", net.ParseIP("1.1.1.1"), false}, + {"external", nil, false}, + } + test(cases) +} diff --git a/modules/hostmatcher/http.go b/modules/hostmatcher/http.go new file mode 100644 index 0000000..8828902 --- /dev/null +++ b/modules/hostmatcher/http.go @@ -0,0 +1,65 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package hostmatcher + +import ( + "context" + "fmt" + "net" + "net/url" + "syscall" + "time" +) + +// NewDialContext returns a DialContext for Transport, the DialContext will do allow/block list check +func NewDialContext(usage string, allowList, blockList *HostMatchList, proxy *url.URL) func(ctx context.Context, network, addr string) (net.Conn, error) { + // How Go HTTP Client works with redirection: + // transport.RoundTrip URL=http://domain.com, Host=domain.com + // transport.DialContext addrOrHost=domain.com:80 + // dialer.Control tcp4:11.22.33.44:80 + // transport.RoundTrip URL=http://www.domain.com/, Host=(empty here, in the direction, HTTP client doesn't fill the Host field) + // transport.DialContext addrOrHost=domain.com:80 + // dialer.Control tcp4:11.22.33.44:80 + return func(ctx context.Context, network, addrOrHost string) (net.Conn, error) { + dialer := net.Dialer{ + // default values comes from http.DefaultTransport + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + + Control: func(network, ipAddr string, c syscall.RawConn) error { + host, port, err := net.SplitHostPort(addrOrHost) + if err != nil { + return err + } + if proxy != nil { + // Always allow the host of the proxy, but only on the specified port. + if host == proxy.Hostname() && port == proxy.Port() { + return nil + } + } + + // in Control func, the addr was already resolved to IP:PORT format, there is no cost to do ResolveTCPAddr here + tcpAddr, err := net.ResolveTCPAddr(network, ipAddr) + if err != nil { + return fmt.Errorf("%s can only call HTTP servers via TCP, deny '%s(%s:%s)', err=%w", usage, host, network, ipAddr, err) + } + + var blockedError error + if blockList.MatchHostOrIP(host, tcpAddr.IP) { + blockedError = fmt.Errorf("%s can not call blocked HTTP servers (check your %s setting), deny '%s(%s)'", usage, blockList.SettingKeyHint, host, ipAddr) + } + + // if we have an allow-list, check the allow-list first + if !allowList.IsEmpty() { + if !allowList.MatchHostOrIP(host, tcpAddr.IP) { + return fmt.Errorf("%s can only call allowed HTTP servers (check your %s setting), deny '%s(%s)'", usage, allowList.SettingKeyHint, host, ipAddr) + } + } + // otherwise, we always follow the blocked list + return blockedError + }, + } + return dialer.DialContext(ctx, network, addrOrHost) + } +} -- cgit v1.2.3