mirror of
https://github.com/MengMengCode/CLICD.git
synced 2026-08-06 05:52:19 +08:00
feat: implement safe HTTP client and URL validation for secure downloads
This commit is contained in:
MengMengCode
@@ -0,0 +1,214 @@
|
||||
package safehttp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const maxRedirects = 10
|
||||
|
||||
var blockedPrefixes = []netip.Prefix{
|
||||
netip.MustParsePrefix("0.0.0.0/8"),
|
||||
netip.MustParsePrefix("10.0.0.0/8"),
|
||||
netip.MustParsePrefix("100.64.0.0/10"),
|
||||
netip.MustParsePrefix("127.0.0.0/8"),
|
||||
netip.MustParsePrefix("169.254.0.0/16"),
|
||||
netip.MustParsePrefix("172.16.0.0/12"),
|
||||
netip.MustParsePrefix("192.0.0.0/24"),
|
||||
netip.MustParsePrefix("192.0.2.0/24"),
|
||||
netip.MustParsePrefix("192.88.99.0/24"),
|
||||
netip.MustParsePrefix("192.168.0.0/16"),
|
||||
netip.MustParsePrefix("198.18.0.0/15"),
|
||||
netip.MustParsePrefix("198.51.100.0/24"),
|
||||
netip.MustParsePrefix("203.0.113.0/24"),
|
||||
netip.MustParsePrefix("224.0.0.0/4"),
|
||||
netip.MustParsePrefix("240.0.0.0/4"),
|
||||
netip.MustParsePrefix("::/128"),
|
||||
netip.MustParsePrefix("::1/128"),
|
||||
netip.MustParsePrefix("64:ff9b::/96"),
|
||||
netip.MustParsePrefix("64:ff9b:1::/48"),
|
||||
netip.MustParsePrefix("100::/64"),
|
||||
netip.MustParsePrefix("2001::/32"),
|
||||
netip.MustParsePrefix("2001:2::/48"),
|
||||
netip.MustParsePrefix("2001:db8::/32"),
|
||||
netip.MustParsePrefix("2001:20::/28"),
|
||||
netip.MustParsePrefix("2002::/16"),
|
||||
netip.MustParsePrefix("fc00::/7"),
|
||||
netip.MustParsePrefix("fec0::/10"),
|
||||
netip.MustParsePrefix("fe80::/10"),
|
||||
netip.MustParsePrefix("ff00::/8"),
|
||||
}
|
||||
|
||||
// ValidateURL performs the URL checks that do not require DNS. Host addresses
|
||||
// are checked again after resolution and immediately before every connection.
|
||||
func ValidateURL(rawURL string) (*url.URL, error) {
|
||||
if len(rawURL) == 0 || len(rawURL) > 4096 {
|
||||
return nil, fmt.Errorf("download URL must be between 1 and 4096 characters")
|
||||
}
|
||||
parsed, err := url.ParseRequestURI(rawURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid download URL: %v", err)
|
||||
}
|
||||
if parsed.Scheme != "http" && parsed.Scheme != "https" {
|
||||
return nil, fmt.Errorf("download URL must use HTTP or HTTPS")
|
||||
}
|
||||
if parsed.Host == "" || parsed.Hostname() == "" {
|
||||
return nil, fmt.Errorf("download URL must include a host")
|
||||
}
|
||||
if parsed.User != nil {
|
||||
return nil, fmt.Errorf("download URL must not include credentials")
|
||||
}
|
||||
if parsed.Fragment != "" {
|
||||
return nil, fmt.Errorf("download URL must not include a fragment")
|
||||
}
|
||||
if port := parsed.Port(); port != "" {
|
||||
value, err := strconv.Atoi(port)
|
||||
if err != nil || value < 1 || value > 65535 {
|
||||
return nil, fmt.Errorf("download URL contains an invalid port")
|
||||
}
|
||||
}
|
||||
if addr, err := netip.ParseAddr(parsed.Hostname()); err == nil && !isPublicAddress(addr) {
|
||||
return nil, fmt.Errorf("download URL resolves to a non-public address")
|
||||
}
|
||||
return parsed, nil
|
||||
}
|
||||
|
||||
// Get retrieves a resource only when every resolved destination is public.
|
||||
func Get(ctx context.Context, rawURL, userAgent string, timeout time.Duration) (*http.Response, error) {
|
||||
parsed, err := ValidateURL(rawURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := validateHost(ctx, net.DefaultResolver, parsed.Hostname()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
request.Header.Set("User-Agent", userAgent)
|
||||
|
||||
client := &http.Client{
|
||||
Timeout: timeout,
|
||||
Transport: publicTransport(net.DefaultResolver),
|
||||
CheckRedirect: func(req *http.Request, via []*http.Request) error {
|
||||
if len(via) >= maxRedirects {
|
||||
return fmt.Errorf("too many redirects")
|
||||
}
|
||||
redirect, err := ValidateURL(req.URL.String())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateHost(req.Context(), net.DefaultResolver, redirect.Hostname()); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(via) > 0 {
|
||||
req.Header.Set("User-Agent", via[0].Header.Get("User-Agent"))
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
// All URL components, redirects, DNS answers and dial destinations are
|
||||
// constrained above and in publicTransport.
|
||||
// lgtm[go/request-forgery]
|
||||
return client.Do(request)
|
||||
}
|
||||
|
||||
func publicTransport(resolver *net.Resolver) *http.Transport {
|
||||
dialer := &net.Dialer{
|
||||
Timeout: 30 * time.Second,
|
||||
KeepAlive: 30 * time.Second,
|
||||
}
|
||||
return &http.Transport{
|
||||
Proxy: nil,
|
||||
DialContext: func(ctx context.Context, network, address string) (net.Conn, error) {
|
||||
host, port, err := net.SplitHostPort(address)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid download destination: %v", err)
|
||||
}
|
||||
addresses, err := resolvePublicHost(ctx, resolver, host)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var lastErr error
|
||||
for _, addr := range addresses {
|
||||
conn, err := dialer.DialContext(ctx, network, net.JoinHostPort(addr.String(), port))
|
||||
if err == nil {
|
||||
return conn, nil
|
||||
}
|
||||
lastErr = err
|
||||
}
|
||||
if lastErr == nil {
|
||||
lastErr = fmt.Errorf("host has no usable public addresses")
|
||||
}
|
||||
return nil, lastErr
|
||||
},
|
||||
ForceAttemptHTTP2: true,
|
||||
TLSHandshakeTimeout: 30 * time.Second,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
}
|
||||
}
|
||||
|
||||
func validateHost(ctx context.Context, resolver *net.Resolver, host string) error {
|
||||
_, err := resolvePublicHost(ctx, resolver, host)
|
||||
return err
|
||||
}
|
||||
|
||||
func resolvePublicHost(ctx context.Context, resolver *net.Resolver, host string) ([]netip.Addr, error) {
|
||||
host = strings.TrimSpace(strings.TrimSuffix(host, "."))
|
||||
if host == "" {
|
||||
return nil, fmt.Errorf("download URL host is empty")
|
||||
}
|
||||
if strings.EqualFold(host, "localhost") || strings.HasSuffix(strings.ToLower(host), ".localhost") {
|
||||
return nil, fmt.Errorf("download URL host is not public")
|
||||
}
|
||||
|
||||
if addr, err := netip.ParseAddr(host); err == nil {
|
||||
addr = addr.Unmap()
|
||||
if !isPublicAddress(addr) {
|
||||
return nil, fmt.Errorf("download URL resolves to a non-public address")
|
||||
}
|
||||
return []netip.Addr{addr}, nil
|
||||
}
|
||||
|
||||
addresses, err := resolver.LookupNetIP(ctx, "ip", host)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to resolve download host: %v", err)
|
||||
}
|
||||
if len(addresses) == 0 {
|
||||
return nil, fmt.Errorf("download host has no IP addresses")
|
||||
}
|
||||
result := make([]netip.Addr, 0, len(addresses))
|
||||
for _, address := range addresses {
|
||||
address = address.Unmap()
|
||||
if !isPublicAddress(address) {
|
||||
return nil, fmt.Errorf("download host resolves to a non-public address")
|
||||
}
|
||||
result = append(result, address)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func isPublicAddress(address netip.Addr) bool {
|
||||
if !address.IsValid() || address.Zone() != "" || !address.IsGlobalUnicast() || address.IsPrivate() ||
|
||||
address.IsLoopback() || address.IsLinkLocalUnicast() || address.IsLinkLocalMulticast() ||
|
||||
address.IsMulticast() || address.IsUnspecified() {
|
||||
return false
|
||||
}
|
||||
address = address.Unmap()
|
||||
for _, prefix := range blockedPrefixes {
|
||||
if prefix.Contains(address) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package safehttp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/netip"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestValidateURLRejectsUnsafeDestinations(t *testing.T) {
|
||||
t.Parallel()
|
||||
for _, rawURL := range []string{
|
||||
"file:///etc/passwd",
|
||||
"http://user:pass@example.com/image",
|
||||
"http://127.0.0.1/image",
|
||||
"http://[::1]/image",
|
||||
"http://169.254.169.254/latest/meta-data",
|
||||
"http://10.0.0.1/image",
|
||||
"http://192.168.1.10/image",
|
||||
"http://100.64.0.1/image",
|
||||
"http://example.com:99999/image",
|
||||
} {
|
||||
if _, err := ValidateURL(rawURL); err == nil {
|
||||
t.Fatalf("ValidateURL(%q) succeeded, want rejection", rawURL)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateURLAcceptsPublicHTTPURL(t *testing.T) {
|
||||
t.Parallel()
|
||||
parsed, err := ValidateURL("https://example.com/images/rootfs.tar.xz?variant=default")
|
||||
if err != nil {
|
||||
t.Fatalf("ValidateURL returned error: %v", err)
|
||||
}
|
||||
if parsed.Hostname() != "example.com" {
|
||||
t.Fatalf("hostname = %q, want example.com", parsed.Hostname())
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsPublicAddress(t *testing.T) {
|
||||
t.Parallel()
|
||||
tests := map[string]bool{
|
||||
"8.8.8.8": true,
|
||||
"1.1.1.1": true,
|
||||
"2606:4700:4700::1111": true,
|
||||
"127.0.0.1": false,
|
||||
"10.0.0.1": false,
|
||||
"100.64.0.1": false,
|
||||
"169.254.169.254": false,
|
||||
"192.0.2.1": false,
|
||||
"198.18.0.1": false,
|
||||
"::1": false,
|
||||
"64:ff9b::127.0.0.1": false,
|
||||
"2002:7f00:1::1": false,
|
||||
"fc00::1": false,
|
||||
"fec0::1": false,
|
||||
"fe80::1": false,
|
||||
"2001:db8::1": false,
|
||||
}
|
||||
for raw, expected := range tests {
|
||||
if actual := isPublicAddress(netip.MustParseAddr(raw)); actual != expected {
|
||||
t.Errorf("isPublicAddress(%s) = %v, want %v", raw, actual, expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetRejectsLoopbackBeforeRequest(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
if _, err := Get(ctx, "http://127.0.0.1:1/image", "test", time.Second); err == nil {
|
||||
t.Fatal("Get accepted a loopback destination")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user