diff options
| author | Ian Lance Taylor <iant@golang.org> | 2019-09-06 18:12:46 +0000 |
|---|---|---|
| committer | Ian Lance Taylor <ian@gcc.gnu.org> | 2019-09-06 18:12:46 +0000 |
| commit | aa8901e9bb0399d2c16f988ba2fe46eb0c0c5d13 (patch) | |
| tree | 7e63b06d1eec92beec6997c9d3ab47a5d6a835be /libgo/go/net | |
| parent | 920ea3b8ba3164b61ac9490dfdfceb6936eda6dd (diff) | |
libgo: update to Go 1.13beta1 release
Reviewed-on: https://go-review.googlesource.com/c/gofrontend/+/193497
From-SVN: r275473
Diffstat (limited to 'libgo/go/net')
89 files changed, 2400 insertions, 580 deletions
diff --git a/libgo/go/net/cgo_unix.go b/libgo/go/net/cgo_unix.go index 342ed9e0281..b8871650966 100644 --- a/libgo/go/net/cgo_unix.go +++ b/libgo/go/net/cgo_unix.go @@ -14,10 +14,16 @@ package net #include <netdb.h> #include <unistd.h> #include <string.h> + +// If nothing else defined EAI_OVERFLOW, make sure it has a value. +#ifndef EAI_OVERFLOW +#define EAI_OVERFLOW -12 +#endif */ import ( "context" + "os" "syscall" "unsafe" ) @@ -51,6 +57,16 @@ func (eai addrinfoErrno) Error() string { return bytePtrToString(libc_gai_stre func (eai addrinfoErrno) Temporary() bool { return eai == syscall.EAI_AGAIN } func (eai addrinfoErrno) Timeout() bool { return false } +func (eai addrinfoErrno) Is(target error) bool { + switch target { + case os.ErrTemporary: + return eai.Temporary() + case os.ErrTimeout: + return eai.Timeout() + } + return false +} + type portLookupResult struct { port int err error @@ -125,6 +141,7 @@ func cgoLookupServicePort(hints *syscall.Addrinfo, network, service string) (por gerrno := libc_getaddrinfo(nil, s, hints, &res) syscall.Exitsyscall() if gerrno != 0 { + isTemporary := false switch gerrno { case syscall.EAI_SYSTEM: errno := syscall.GetErrno() @@ -134,8 +151,9 @@ func cgoLookupServicePort(hints *syscall.Addrinfo, network, service string) (por err = errno default: err = addrinfoErrno(gerrno) + isTemporary = addrinfoErrno(gerrno).Temporary() } - return 0, &DNSError{Err: err.Error(), Name: network + "/" + service} + return 0, &DNSError{Err: err.Error(), Name: network + "/" + service, IsTemporary: isTemporary} } defer libc_freeaddrinfo(res) @@ -180,6 +198,8 @@ func cgoLookupIPCNAME(network, name string) (addrs []IPAddr, cname string, err e gerrno := libc_getaddrinfo(h, nil, &hints, &res) syscall.Exitsyscall() if gerrno != 0 { + isErrorNoSuchHost := false + isTemporary := false switch gerrno { case syscall.EAI_SYSTEM: errno := syscall.GetErrno() @@ -196,10 +216,13 @@ func cgoLookupIPCNAME(network, name string) (addrs []IPAddr, cname string, err e err = errno case syscall.EAI_NONAME: err = errNoSuchHost + isErrorNoSuchHost = true default: err = addrinfoErrno(gerrno) + isTemporary = addrinfoErrno(gerrno).Temporary() } - return nil, "", &DNSError{Err: err.Error(), Name: name} + + return nil, "", &DNSError{Err: err.Error(), Name: name, IsNotFound: isErrorNoSuchHost, IsTemporary: isTemporary} } defer libc_freeaddrinfo(res) @@ -320,6 +343,7 @@ func cgoLookupAddrPTR(addr string, sa *syscall.RawSockaddr, salen syscall.Sockle } } if gerrno != 0 { + isTemporary := false switch gerrno { case syscall.EAI_SYSTEM: if err == nil { // see golang.org/issue/6232 @@ -327,8 +351,9 @@ func cgoLookupAddrPTR(addr string, sa *syscall.RawSockaddr, salen syscall.Sockle } default: err = addrinfoErrno(gerrno) + isTemporary = addrinfoErrno(gerrno).Temporary() } - return nil, &DNSError{Err: err.Error(), Name: addr} + return nil, &DNSError{Err: err.Error(), Name: addr, IsTemporary: isTemporary} } for i := 0; i < len(b); i++ { if b[i] == 0 { diff --git a/libgo/go/net/dial.go b/libgo/go/net/dial.go index 1dd8690739e..4d55a95ddf6 100644 --- a/libgo/go/net/dial.go +++ b/libgo/go/net/dial.go @@ -12,6 +12,12 @@ import ( "time" ) +// defaultTCPKeepAlive is a default constant value for TCPKeepAlive times +// See golang.org/issue/31510 +const ( + defaultTCPKeepAlive = 15 * time.Second +) + // A Dialer contains options for connecting to an address. // // The zero value for each field is equivalent to dialing @@ -63,12 +69,13 @@ type Dialer struct { // A negative value disables Fast Fallback support. FallbackDelay time.Duration - // KeepAlive specifies the keep-alive period for an active - // network connection. - // If zero, keep-alives are enabled if supported by the protocol - // and operating system. Network protocols or operating systems - // that do not support keep-alives ignore this field. - // If negative, keep-alives are disabled. + // KeepAlive specifies the interval between keep-alive + // probes for an active network connection. + // If zero, keep-alive probes are sent with a default value + // (currently 15 seconds), if supported by the protocol and operating + // system. Network protocols or operating systems that do + // not support keep-alives ignore this field. + // If negative, keep-alive probes are disabled. KeepAlive time.Duration // Resolver optionally specifies an alternate resolver to use. @@ -76,7 +83,7 @@ type Dialer struct { // Cancel is an optional channel whose closure indicates that // the dial should be canceled. Not all types of dials support - // cancelation. + // cancellation. // // Deprecated: Use DialContext instead. Cancel <-chan struct{} @@ -424,7 +431,7 @@ func (d *Dialer) DialContext(ctx context.Context, network, address string) (Conn setKeepAlive(tc.fd, true) ka := d.KeepAlive if d.KeepAlive == 0 { - ka = 15 * time.Second + ka = defaultTCPKeepAlive } setKeepAlivePeriod(tc.fd, ka) testHookSetKeepAlive(ka) @@ -596,6 +603,14 @@ type ListenConfig struct { // necessarily the ones passed to Listen. For example, passing "tcp" to // Listen will cause the Control function to be called with "tcp4" or "tcp6". Control func(network, address string, c syscall.RawConn) error + + // KeepAlive specifies the keep-alive period for network + // connections accepted by this listener. + // If zero, keep-alives are enabled if supported by the protocol + // and operating system. Network protocols or operating systems + // that do not support keep-alives ignore this field. + // If negative, keep-alives are disabled. + KeepAlive time.Duration } // Listen announces on the local network address. diff --git a/libgo/go/net/dial_test.go b/libgo/go/net/dial_test.go index 3a2c59a2d1d..1bf96fd3ce3 100644 --- a/libgo/go/net/dial_test.go +++ b/libgo/go/net/dial_test.go @@ -463,7 +463,7 @@ func TestDialParallelSpuriousConnection(t *testing.T) { origTestHookDialTCP := testHookDialTCP defer func() { testHookDialTCP = origTestHookDialTCP }() testHookDialTCP = func(ctx context.Context, net string, laddr, raddr *TCPAddr) (*TCPConn, error) { - // Sleep long enough for Happy Eyeballs to kick in, and inhibit cancelation. + // Sleep long enough for Happy Eyeballs to kick in, and inhibit cancellation. // This forces dialParallel to juggle two successful connections. time.Sleep(fallbackDelay * 2) @@ -865,7 +865,7 @@ func TestCancelAfterDial(t *testing.T) { d := &Dialer{Cancel: cancel} c, err := d.Dial("tcp", ln.Addr().String()) - // Immediately after dialing, request cancelation and sleep. + // Immediately after dialing, request cancellation and sleep. // Before Issue 15078 was fixed, this would cause subsequent operations // to fail with an i/o timeout roughly 50% of the time. close(cancel) @@ -973,11 +973,11 @@ func TestDialerControl(t *testing.T) { } // mustHaveExternalNetwork is like testenv.MustHaveExternalNetwork -// except that it won't skip testing on non-iOS builders. +// except that it won't skip testing on non-mobile builders. func mustHaveExternalNetwork(t *testing.T) { t.Helper() - ios := runtime.GOOS == "darwin" && (runtime.GOARCH == "arm" || runtime.GOARCH == "arm64") - if testenv.Builder() == "" || ios { + mobile := runtime.GOOS == "android" || runtime.GOOS == "darwin" && (runtime.GOARCH == "arm" || runtime.GOARCH == "arm64") + if testenv.Builder() == "" || mobile { testenv.MustHaveExternalNetwork(t) } } diff --git a/libgo/go/net/dnsclient.go b/libgo/go/net/dnsclient.go index 4fdf60ff4e3..b5bb3a4d11d 100644 --- a/libgo/go/net/dnsclient.go +++ b/libgo/go/net/dnsclient.go @@ -8,7 +8,7 @@ import ( "math/rand" "sort" - "internal/x/net/dns/dnsmessage" + "golang.org/x/net/dns/dnsmessage" ) // reverseaddr returns the in-addr.arpa. or ip6.arpa. hostname of the IP diff --git a/libgo/go/net/dnsclient_unix.go b/libgo/go/net/dnsclient_unix.go index 94dbe95afa3..b3284b8cd76 100644 --- a/libgo/go/net/dnsclient_unix.go +++ b/libgo/go/net/dnsclient_unix.go @@ -23,7 +23,13 @@ import ( "sync" "time" - "internal/x/net/dns/dnsmessage" + "golang.org/x/net/dns/dnsmessage" +) + +const ( + // to be used as a useTCP parameter to exchange + useTCPOnly = true + useUDPOrTCP = false ) var ( @@ -131,13 +137,19 @@ func dnsStreamRoundTrip(c Conn, id uint16, query dnsmessage.Question, b []byte) } // exchange sends a query on the connection and hopes for a response. -func (r *Resolver) exchange(ctx context.Context, server string, q dnsmessage.Question, timeout time.Duration) (dnsmessage.Parser, dnsmessage.Header, error) { +func (r *Resolver) exchange(ctx context.Context, server string, q dnsmessage.Question, timeout time.Duration, useTCP bool) (dnsmessage.Parser, dnsmessage.Header, error) { q.Class = dnsmessage.ClassINET id, udpReq, tcpReq, err := newRequest(q) if err != nil { return dnsmessage.Parser{}, dnsmessage.Header{}, errCannotMarshalDNSMessage } - for _, network := range []string{"udp", "tcp"} { + var networks []string + if useTCP { + networks = []string{"tcp"} + } else { + networks = []string{"udp", "tcp"} + } + for _, network := range networks { ctx, cancel := context.WithDeadline(ctx, time.Now().Add(timeout)) defer cancel() @@ -171,7 +183,7 @@ func (r *Resolver) exchange(ctx context.Context, server string, q dnsmessage.Que } // checkHeader performs basic sanity checks on the header. -func checkHeader(p *dnsmessage.Parser, h dnsmessage.Header, name, server string) error { +func checkHeader(p *dnsmessage.Parser, h dnsmessage.Header) error { if h.RCode == dnsmessage.RCodeNameError { return errNoSuchHost } @@ -202,7 +214,7 @@ func checkHeader(p *dnsmessage.Parser, h dnsmessage.Header, name, server string) return nil } -func skipToAnswer(p *dnsmessage.Parser, qtype dnsmessage.Type, name, server string) error { +func skipToAnswer(p *dnsmessage.Parser, qtype dnsmessage.Type) error { for { h, err := p.AnswerHeader() if err == dnsmessage.ErrSectionDone { @@ -241,7 +253,7 @@ func (r *Resolver) tryOneName(ctx context.Context, cfg *dnsConfig, name string, for j := uint32(0); j < sLen; j++ { server := cfg.servers[(serverOffset+j)%sLen] - p, h, err := r.exchange(ctx, server, q, cfg.timeout) + p, h, err := r.exchange(ctx, server, q, cfg.timeout, cfg.useTCP) if err != nil { dnsErr := &DNSError{ Err: err.Error(), @@ -260,7 +272,7 @@ func (r *Resolver) tryOneName(ctx context.Context, cfg *dnsConfig, name string, continue } - if err := checkHeader(&p, h, name, server); err != nil { + if err := checkHeader(&p, h); err != nil { dnsErr := &DNSError{ Err: err.Error(), Name: name, @@ -272,17 +284,15 @@ func (r *Resolver) tryOneName(ctx context.Context, cfg *dnsConfig, name string, if err == errNoSuchHost { // The name does not exist, so trying // another server won't help. - // - // TODO: indicate this in a more - // obvious way, such as a field on - // DNSError? + + dnsErr.IsNotFound = true return p, server, dnsErr } lastErr = dnsErr continue } - err = skipToAnswer(&p, qtype, name, server) + err = skipToAnswer(&p, qtype) if err == nil { return p, server, nil } @@ -294,9 +304,8 @@ func (r *Resolver) tryOneName(ctx context.Context, cfg *dnsConfig, name string, if err == errNoSuchHost { // The name does not exist, so trying another // server won't help. - // - // TODO: indicate this in a more obvious way, - // such as a field on DNSError? + + lastErr.(*DNSError).IsNotFound = true return p, server, lastErr } } @@ -386,7 +395,7 @@ func (r *Resolver) lookup(ctx context.Context, name string, qtype dnsmessage.Typ // Other lookups might allow broader name syntax // (for example Multicast DNS allows UTF-8; see RFC 6762). // For consistency with libc resolvers, report no such host. - return dnsmessage.Parser{}, "", &DNSError{Err: errNoSuchHost.Error(), Name: name} + return dnsmessage.Parser{}, "", &DNSError{Err: errNoSuchHost.Error(), Name: name, IsNotFound: true} } resolvConf.tryUpdate("/etc/resolv.conf") resolvConf.mu.RLock() @@ -563,40 +572,58 @@ func (r *Resolver) goLookupIPCNAMEOrder(ctx context.Context, name string, order } if !isDomainName(name) { // See comment in func lookup above about use of errNoSuchHost. - return nil, dnsmessage.Name{}, &DNSError{Err: errNoSuchHost.Error(), Name: name} + return nil, dnsmessage.Name{}, &DNSError{Err: errNoSuchHost.Error(), Name: name, IsNotFound: true} } resolvConf.tryUpdate("/etc/resolv.conf") resolvConf.mu.RLock() conf := resolvConf.dnsConfig resolvConf.mu.RUnlock() - type racer struct { + type result struct { p dnsmessage.Parser server string error } - lane := make(chan racer, 1) + lane := make(chan result, 1) qtypes := [...]dnsmessage.Type{dnsmessage.TypeA, dnsmessage.TypeAAAA} - var lastErr error - for _, fqdn := range conf.nameList(name) { - for _, qtype := range qtypes { + var queryFn func(fqdn string, qtype dnsmessage.Type) + var responseFn func(fqdn string, qtype dnsmessage.Type) result + if conf.singleRequest { + queryFn = func(fqdn string, qtype dnsmessage.Type) {} + responseFn = func(fqdn string, qtype dnsmessage.Type) result { + dnsWaitGroup.Add(1) + defer dnsWaitGroup.Done() + p, server, err := r.tryOneName(ctx, conf, fqdn, qtype) + return result{p, server, err} + } + } else { + queryFn = func(fqdn string, qtype dnsmessage.Type) { dnsWaitGroup.Add(1) go func(qtype dnsmessage.Type) { p, server, err := r.tryOneName(ctx, conf, fqdn, qtype) - lane <- racer{p, server, err} + lane <- result{p, server, err} dnsWaitGroup.Done() }(qtype) } + responseFn = func(fqdn string, qtype dnsmessage.Type) result { + return <-lane + } + } + var lastErr error + for _, fqdn := range conf.nameList(name) { + for _, qtype := range qtypes { + queryFn(fqdn, qtype) + } hitStrictError := false - for range qtypes { - racer := <-lane - if racer.error != nil { - if nerr, ok := racer.error.(Error); ok && nerr.Temporary() && r.strictErrors() { + for _, qtype := range qtypes { + result := responseFn(fqdn, qtype) + if result.error != nil { + if nerr, ok := result.error.(Error); ok && nerr.Temporary() && r.strictErrors() { // This error will abort the nameList loop. hitStrictError = true - lastErr = racer.error + lastErr = result.error } else if lastErr == nil || fqdn == name+"." { // Prefer error for original name. - lastErr = racer.error + lastErr = result.error } continue } @@ -618,12 +645,12 @@ func (r *Resolver) goLookupIPCNAMEOrder(ctx context.Context, name string, order loop: for { - h, err := racer.p.AnswerHeader() + h, err := result.p.AnswerHeader() if err != nil && err != dnsmessage.ErrSectionDone { lastErr = &DNSError{ Err: "cannot marshal DNS message", Name: name, - Server: racer.server, + Server: result.server, } } if err != nil { @@ -631,35 +658,35 @@ func (r *Resolver) goLookupIPCNAMEOrder(ctx context.Context, name string, order } switch h.Type { case dnsmessage.TypeA: - a, err := racer.p.AResource() + a, err := result.p.AResource() if err != nil { lastErr = &DNSError{ Err: "cannot marshal DNS message", Name: name, - Server: racer.server, + Server: result.server, } break loop } addrs = append(addrs, IPAddr{IP: IP(a.A[:])}) case dnsmessage.TypeAAAA: - aaaa, err := racer.p.AAAAResource() + aaaa, err := result.p.AAAAResource() if err != nil { lastErr = &DNSError{ Err: "cannot marshal DNS message", Name: name, - Server: racer.server, + Server: result.server, } break loop } addrs = append(addrs, IPAddr{IP: IP(aaaa.AAAA[:])}) default: - if err := racer.p.SkipAnswer(); err != nil { + if err := result.p.SkipAnswer(); err != nil { lastErr = &DNSError{ Err: "cannot marshal DNS message", Name: name, - Server: racer.server, + Server: result.server, } break loop } diff --git a/libgo/go/net/dnsclient_unix_test.go b/libgo/go/net/dnsclient_unix_test.go index 93ecc6cf69c..b51d60861f8 100644 --- a/libgo/go/net/dnsclient_unix_test.go +++ b/libgo/go/net/dnsclient_unix_test.go @@ -17,10 +17,11 @@ import ( "reflect" "strings" "sync" + "sync/atomic" "testing" "time" - "internal/x/net/dns/dnsmessage" + "golang.org/x/net/dns/dnsmessage" ) var goResolver = Resolver{PreferGo: true} @@ -29,7 +30,7 @@ var goResolver = Resolver{PreferGo: true} var TestAddr = [4]byte{0xc0, 0x00, 0x02, 0x01} // Test address from 2001:db8::/32 block, reserved by RFC 3849 for documentation. -var VarTestAddr6 = [16]byte{0x20, 0x01, 0x0d, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1} +var TestAddr6 = [16]byte{0x20, 0x01, 0x0d, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1} func mustNewName(name string) dnsmessage.Name { nn, err := dnsmessage.NewName(name) @@ -80,7 +81,7 @@ func TestDNSTransportFallback(t *testing.T) { for _, tt := range dnsTransportFallbackTests { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, h, err := r.exchange(ctx, tt.server, tt.question, time.Second) + _, h, err := r.exchange(ctx, tt.server, tt.question, time.Second, useUDPOrTCP) if err != nil { t.Error(err) continue @@ -136,7 +137,7 @@ func TestSpecialDomainName(t *testing.T) { for _, tt := range specialDomainNameTests { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, h, err := r.exchange(ctx, server, tt.question, 3*time.Second) + _, h, err := r.exchange(ctx, server, tt.question, 3*time.Second, useUDPOrTCP) if err != nil { t.Error(err) continue @@ -522,7 +523,7 @@ func TestGoLookupIPWithResolverConfig(t *testing.T) { Length: 16, }, Body: &dnsmessage.AAAAResource{ - AAAA: VarTestAddr6, + AAAA: TestAddr6, }, }) } @@ -588,6 +589,8 @@ func TestGoLookupIPOrderFallbackToFile(t *testing.T) { if err != nil { t.Fatal(err) } + defer conf.teardown() + if err := conf.writeAndUpdate([]string{}); err != nil { t.Fatal(err) } @@ -619,7 +622,6 @@ func TestGoLookupIPOrderFallbackToFile(t *testing.T) { t.Errorf("%s: address doesn't match expectation. got %v, want %v", name, got, want) } } - defer conf.teardown() } // Issue 12712. @@ -665,7 +667,7 @@ func TestErrorForOriginalNameWhenSearching(t *testing.T) { wantErr *DNSError }{ {true, &DNSError{Name: fqdn, Err: "server misbehaving", IsTemporary: true}}, - {false, &DNSError{Name: fqdn, Err: errNoSuchHost.Error()}}, + {false, &DNSError{Name: fqdn, Err: errNoSuchHost.Error(), IsNotFound: true}}, } for _, tt := range cases { r := Resolver{PreferGo: true, StrictErrors: tt.strictErrors, Dial: fake.DialContext} @@ -1137,9 +1139,10 @@ func TestStrictErrorsLookupIP(t *testing.T) { } makeNxDomain := func() error { return &DNSError{ - Err: errNoSuchHost.Error(), - Name: name, - Server: server, + Err: errNoSuchHost.Error(), + Name: name, + Server: server, + IsNotFound: true, } } @@ -1298,7 +1301,7 @@ func TestStrictErrorsLookupIP(t *testing.T) { Length: 16, }, Body: &dnsmessage.AAAAResource{ - AAAA: VarTestAddr6, + AAAA: TestAddr6, }, }, } @@ -1471,6 +1474,32 @@ func TestIssue8434(t *testing.T) { } } +func TestIssueNoSuchHostExists(t *testing.T) { + err := lookupWithFake(fakeDNSServer{ + rh: func(n, _ string, q dnsmessage.Message, _ time.Time) (dnsmessage.Message, error) { + return dnsmessage.Message{ + Header: dnsmessage.Header{ + ID: q.ID, + Response: true, + RCode: dnsmessage.RCodeNameError, + }, + Questions: q.Questions, + }, nil + }, + }, "golang.org.", dnsmessage.TypeALL) + if err == nil { + t.Fatal("expected an error") + } + if _, ok := err.(Error); !ok { + t.Fatalf("err = %#v; wanted something supporting net.Error", err) + } + if de, ok := err.(*DNSError); !ok { + t.Fatalf("err = %#v; wanted a *net.DNSError", err) + } else if !de.IsNotFound { + t.Fatalf("IsNotFound = false for err = %#v; want IsNotFound == true", err) + } +} + // TestNoSuchHost verifies that tryOneName works correctly when the domain does // not exist. // @@ -1540,6 +1569,9 @@ func TestNoSuchHost(t *testing.T) { if de.Err != errNoSuchHost.Error() { t.Fatalf("Err = %#v; wanted %q", de.Err, errNoSuchHost.Error()) } + if !de.IsNotFound { + t.Fatalf("IsNotFound = %v wanted true", de.IsNotFound) + } }) } } @@ -1563,7 +1595,7 @@ func TestDNSDialTCP(t *testing.T) { } r := Resolver{PreferGo: true, Dial: fake.DialContext} ctx := context.Background() - _, _, err := r.exchange(ctx, "0.0.0.0", mustQuestion("com.", dnsmessage.TypeALL, dnsmessage.ClassINET), time.Second) + _, _, err := r.exchange(ctx, "0.0.0.0", mustQuestion("com.", dnsmessage.TypeALL, dnsmessage.ClassINET), time.Second, useUDPOrTCP) if err != nil { t.Fatal("exhange failed:", err) } @@ -1621,3 +1653,103 @@ func TestTXTRecordTwoStrings(t *testing.T) { t.Errorf("txt[1], got %q, want %q", txt[1], want) } } + +// Issue 29644: support single-request resolv.conf option in pure Go resolver. +// The A and AAAA queries will be sent sequentially, not in parallel. +func TestSingleRequestLookup(t *testing.T) { + defer dnsWaitGroup.Wait() + var ( + firstcalled int32 + ipv4 int32 = 1 + ipv6 int32 = 2 + ) + fake := fakeDNSServer{rh: func(n, s string, q dnsmessage.Message, _ time.Time) (dnsmessage.Message, error) { + r := dnsmessage.Message{ + Header: dnsmessage.Header{ + ID: q.ID, + Response: true, + }, + Questions: q.Questions, + } + for _, question := range q.Questions { + switch question.Type { + case dnsmessage.TypeA: + if question.Name.String() == "slowipv4.example.net." { + time.Sleep(10 * time.Millisecond) + } + if !atomic.CompareAndSwapInt32(&firstcalled, 0, ipv4) { + t.Errorf("the A query was received after the AAAA query !") + } + r.Answers = append(r.Answers, dnsmessage.Resource{ + Header: dnsmessage.ResourceHeader{ + Name: q.Questions[0].Name, + Type: dnsmessage.TypeA, + Class: dnsmessage.ClassINET, + Length: 4, + }, + Body: &dnsmessage.AResource{ + A: TestAddr, + }, + }) + case dnsmessage.TypeAAAA: + atomic.CompareAndSwapInt32(&firstcalled, 0, ipv6) + r.Answers = append(r.Answers, dnsmessage.Resource{ + Header: dnsmessage.ResourceHeader{ + Name: q.Questions[0].Name, + Type: dnsmessage.TypeAAAA, + Class: dnsmessage.ClassINET, + Length: 16, + }, + Body: &dnsmessage.AAAAResource{ + AAAA: TestAddr6, + }, + }) + } + } + return r, nil + }} + r := Resolver{PreferGo: true, Dial: fake.DialContext} + + conf, err := newResolvConfTest() + if err != nil { + t.Fatal(err) + } + defer conf.teardown() + if err := conf.writeAndUpdate([]string{"options single-request"}); err != nil { + t.Fatal(err) + } + for _, name := range []string{"hostname.example.net", "slowipv4.example.net"} { + firstcalled = 0 + _, err := r.LookupIPAddr(context.Background(), name) + if err != nil { + t.Error(err) + } + } +} + +// Issue 29358. Add configuration knob to force TCP-only DNS requests in the pure Go resolver. +func TestDNSUseTCP(t *testing.T) { + fake := fakeDNSServer{ + rh: func(n, _ string, q dnsmessage.Message, _ time.Time) (dnsmessage.Message, error) { + r := dnsmessage.Message{ + Header: dnsmessage.Header{ + ID: q.Header.ID, + Response: true, + RCode: dnsmessage.RCodeSuccess, + }, + Questions: q.Questions, + } + if n == "udp" { + t.Fatal("udp protocol was used instead of tcp") + } + return r, nil + }, + } + r := Resolver{PreferGo: true, Dial: fake.DialContext} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + _, _, err := r.exchange(ctx, "0.0.0.0", mustQuestion("com.", dnsmessage.TypeALL, dnsmessage.ClassINET), time.Second, useTCPOnly) + if err != nil { + t.Fatal("exchange failed:", err) + } +} diff --git a/libgo/go/net/dnsconfig_unix.go b/libgo/go/net/dnsconfig_unix.go index 58ae4a35b75..b76b32fb253 100644 --- a/libgo/go/net/dnsconfig_unix.go +++ b/libgo/go/net/dnsconfig_unix.go @@ -21,17 +21,19 @@ var ( ) type dnsConfig struct { - servers []string // server addresses (in host:port form) to use - search []string // rooted suffixes to append to local name - ndots int // number of dots in name to trigger absolute lookup - timeout time.Duration // wait before giving up on a query, including retries - attempts int // lost packets before giving up on server - rotate bool // round robin among servers - unknownOpt bool // anything unknown was encountered - lookup []string // OpenBSD top-level database "lookup" order - err error // any error that occurs during open of resolv.conf - mtime time.Time // time of resolv.conf modification - soffset uint32 // used by serverOffset + servers []string // server addresses (in host:port form) to use + search []string // rooted suffixes to append to local name + ndots int // number of dots in name to trigger absolute lookup + timeout time.Duration // wait before giving up on a query, including retries + attempts int // lost packets before giving up on server + rotate bool // round robin among servers + unknownOpt bool // anything unknown was encountered + lookup []string // OpenBSD top-level database "lookup" order + err error // any error that occurs during open of resolv.conf + mtime time.Time // time of resolv.conf modification + soffset uint32 // used by serverOffset + singleRequest bool // use sequential A and AAAA queries instead of parallel queries + useTCP bool // force usage of TCP for DNS resolutions } // See resolv.conf(5) on a Linux machine. @@ -115,6 +117,21 @@ func dnsReadConfig(filename string) *dnsConfig { conf.attempts = n case s == "rotate": conf.rotate = true + case s == "single-request" || s == "single-request-reopen": + // Linux option: + // http://man7.org/linux/man-pages/man5/resolv.conf.5.html + // "By default, glibc performs IPv4 and IPv6 lookups in parallel [...] + // This option disables the behavior and makes glibc + // perform the IPv6 and IPv4 requests sequentially." + conf.singleRequest = true + case s == "use-vc" || s == "usevc" || s == "tcp": + // Linux (use-vc), FreeBSD (usevc) and OpenBSD (tcp) option: + // http://man7.org/linux/man-pages/man5/resolv.conf.5.html + // "Sets RES_USEVC in _res.options. + // This option forces the use of TCP for DNS resolutions." + // https://www.freebsd.org/cgi/man.cgi?query=resolv.conf&sektion=5&manpath=freebsd-release-ports + // https://man.openbsd.org/resolv.conf.5 + conf.useTCP = true default: conf.unknownOpt = true } diff --git a/libgo/go/net/dnsconfig_unix_test.go b/libgo/go/net/dnsconfig_unix_test.go index 4a41b48e574..2fca3293501 100644 --- a/libgo/go/net/dnsconfig_unix_test.go +++ b/libgo/go/net/dnsconfig_unix_test.go @@ -102,6 +102,61 @@ var dnsReadConfigTests = []struct { search: []string{"c.symbolic-datum-552.internal."}, }, }, + { + name: "testdata/single-request-resolv.conf", + want: &dnsConfig{ + servers: defaultNS, + ndots: 1, + singleRequest: true, + timeout: 5 * time.Second, + attempts: 2, + search: []string{"domain.local."}, + }, + }, + { + name: "testdata/single-request-reopen-resolv.conf", + want: &dnsConfig{ + servers: defaultNS, + ndots: 1, + singleRequest: true, + timeout: 5 * time.Second, + attempts: 2, + search: []string{"domain.local."}, + }, + }, + { + name: "testdata/linux-use-vc-resolv.conf", + want: &dnsConfig{ + servers: defaultNS, + ndots: 1, + useTCP: true, + timeout: 5 * time.Second, + attempts: 2, + search: []string{"domain.local."}, + }, + }, + { + name: "testdata/freebsd-usevc-resolv.conf", + want: &dnsConfig{ + servers: defaultNS, + ndots: 1, + useTCP: true, + timeout: 5 * time.Second, + attempts: 2, + search: []string{"domain.local."}, + }, + }, + { + name: "testdata/openbsd-tcp-resolv.conf", + want: &dnsConfig{ + servers: defaultNS, + ndots: 1, + useTCP: true, + timeout: 5 * time.Second, + attempts: 2, + search: []string{"domain.local."}, + }, + }, } func TestDNSReadConfig(t *testing.T) { diff --git a/libgo/go/net/error_test.go b/libgo/go/net/error_test.go index 2819986c0cd..c4fee5aa5e5 100644 --- a/libgo/go/net/error_test.go +++ b/libgo/go/net/error_test.go @@ -185,7 +185,7 @@ func TestDialError(t *testing.T) { func TestProtocolDialError(t *testing.T) { switch runtime.GOOS { - case "nacl", "solaris": + case "nacl", "solaris", "illumos": t.Skipf("not supported on %s", runtime.GOOS) } @@ -436,7 +436,7 @@ second: goto third } switch nestedErr { - case poll.ErrNetClosing, poll.ErrTimeout: + case poll.ErrNetClosing, poll.ErrTimeout, poll.ErrNotPollable: return nil } return fmt.Errorf("unexpected type on 2nd nested level: %T", nestedErr) @@ -627,7 +627,7 @@ second: goto third } switch nestedErr { - case poll.ErrNetClosing, poll.ErrTimeout: + case poll.ErrNetClosing, poll.ErrTimeout, poll.ErrNotPollable: return nil } return fmt.Errorf("unexpected type on 2nd nested level: %T", nestedErr) diff --git a/libgo/go/net/fd_unix.go b/libgo/go/net/fd_unix.go index 42fe8804e5a..286d3f1c92a 100644 --- a/libgo/go/net/fd_unix.go +++ b/libgo/go/net/fd_unix.go @@ -81,12 +81,12 @@ func (fd *netFD) connect(ctx context.Context, la, ra syscall.Sockaddr) (rsa sysc runtime.KeepAlive(fd) return nil, nil case syscall.EINVAL: - // On Solaris we can see EINVAL if the socket has - // already been accepted and closed by the server. - // Treat this as a successful connection--writes to - // the socket will see EOF. For details and a test - // case in C see https://golang.org/issue/6828. - if runtime.GOOS == "solaris" { + // On Solaris and illumos we can see EINVAL if the socket has + // already been accepted and closed by the server. Treat this + // as a successful connection--writes to the socket will see + // EOF. For details and a test case in C see + // https://golang.org/issue/6828. + if runtime.GOOS == "solaris" || runtime.GOOS == "illumos" { return nil, nil } fallthrough diff --git a/libgo/go/net/file_plan9.go b/libgo/go/net/file_plan9.go index d16e5a166c3..dfb23d2e842 100644 --- a/libgo/go/net/file_plan9.go +++ b/libgo/go/net/file_plan9.go @@ -127,7 +127,7 @@ func fileListener(f *os.File) (Listener, error) { return nil, errors.New("file does not represent a listener") } - return &TCPListener{fd}, nil + return &TCPListener{fd: fd}, nil } func filePacketConn(f *os.File) (PacketConn, error) { diff --git a/libgo/go/net/file_unix.go b/libgo/go/net/file_unix.go index 46ea8e25d84..25caafb9fa8 100644 --- a/libgo/go/net/file_unix.go +++ b/libgo/go/net/file_unix.go @@ -93,7 +93,7 @@ func fileListener(f *os.File) (Listener, error) { } switch laddr := fd.laddr.(type) { case *TCPAddr: - return &TCPListener{fd}, nil + return &TCPListener{fd: fd}, nil case *UnixAddr: return &UnixListener{fd: fd, path: laddr.Name, unlink: false}, nil } diff --git a/libgo/go/net/http/cgi/child.go b/libgo/go/net/http/cgi/child.go index 10325c2eb5a..cb140f8f2f6 100644 --- a/libgo/go/net/http/cgi/child.go +++ b/libgo/go/net/http/cgi/child.go @@ -102,7 +102,7 @@ func RequestFromMap(params map[string]string) (*http.Request, error) { } // There's apparently a de-facto standard for this. - // https://docstore.mik.ua/orelly/linux/cgi/ch03_02.htm#ch03-35636 + // https://web.archive.org/web/20170105004655/http://docstore.mik.ua/orelly/linux/cgi/ch03_02.htm#ch03-35636 if s := params["HTTPS"]; s == "on" || s == "ON" || s == "1" { r.TLS = &tls.ConnectionState{HandshakeComplete: true} } diff --git a/libgo/go/net/http/client.go b/libgo/go/net/http/client.go index 921f86bd92d..65a9d51cc6b 100644 --- a/libgo/go/net/http/client.go +++ b/libgo/go/net/http/client.go @@ -100,7 +100,7 @@ type Client struct { // For compatibility, the Client will also use the deprecated // CancelRequest method on Transport if found. New // RoundTripper implementations should use the Request's Context - // for cancelation instead of implementing CancelRequest. + // for cancellation instead of implementing CancelRequest. Timeout time.Duration } @@ -238,7 +238,7 @@ func send(ireq *Request, rt RoundTripper, deadline time.Time) (resp *Response, d username := u.Username() password, _ := u.Password() forkReq() - req.Header = ireq.Header.clone() + req.Header = ireq.Header.Clone() req.Header.Set("Authorization", "Basic "+basicAuth(username, password)) } @@ -643,7 +643,7 @@ func (c *Client) do(req *Request) (retres *Response, reterr error) { reqBodyClosed = true if !deadline.IsZero() && didTimeout() { err = &httpError{ - // TODO: early in cycle: s/Client.Timeout exceeded/timeout or context cancelation/ + // TODO: early in cycle: s/Client.Timeout exceeded/timeout or context cancellation/ err: err.Error() + " (Client.Timeout exceeded while awaiting headers)", timeout: true, } @@ -668,7 +668,7 @@ func (c *Client) makeHeadersCopier(ireq *Request) func(*Request) { // The headers to copy are from the very initial request. // We use a closured callback to keep a reference to these original headers. var ( - ireqhdr = ireq.Header.clone() + ireqhdr = ireq.Header.Clone() icookies map[string][]*Cookie ) if c.Jar != nil && ireq.Header.Get("Cookie") != "" { @@ -870,7 +870,7 @@ func (b *cancelTimerBody) Read(p []byte) (n int, err error) { } if b.reqDidTimeout() { err = &httpError{ - // TODO: early in cycle: s/Client.Timeout exceeded/timeout or context cancelation/ + // TODO: early in cycle: s/Client.Timeout exceeded/timeout or context cancellation/ err: err.Error() + " (Client.Timeout exceeded while reading body)", timeout: true, } @@ -926,10 +926,9 @@ func isDomainOrSubdomain(sub, parent string) bool { } func stripPassword(u *url.URL) string { - pass, passSet := u.User.Password() + _, passSet := u.User.Password() if passSet { - return strings.Replace(u.String(), pass+"@", "***@", 1) + return strings.Replace(u.String(), u.User.String()+"@", u.User.Username()+":***@", 1) } - return u.String() } diff --git a/libgo/go/net/http/client_test.go b/libgo/go/net/http/client_test.go index 1c59ce74352..de490bc6074 100644 --- a/libgo/go/net/http/client_test.go +++ b/libgo/go/net/http/client_test.go @@ -317,8 +317,7 @@ func TestClientRedirectContext(t *testing.T) { return errors.New("redirected request's context never expired after root request canceled") } } - req, _ := NewRequest("GET", ts.URL, nil) - req = req.WithContext(ctx) + req, _ := NewRequestWithContext(ctx, "GET", ts.URL, nil) _, err := c.Do(req) ue, ok := err.(*url.Error) if !ok { @@ -1185,6 +1184,11 @@ func TestStripPasswordFromError(t *testing.T) { in: "http://user:password@dummy.faketld/password", out: "Get http://user:***@dummy.faketld/password: dummy impl", }, + { + desc: "Strip escaped password", + in: "http://user:pa%2Fssword@dummy.faketld/", + out: "Get http://user:***@dummy.faketld/: dummy impl", + }, } for _, tC := range testCases { t.Run(tC.desc, func(t *testing.T) { @@ -1288,7 +1292,7 @@ func testClientTimeout_Headers(t *testing.T, h2 bool) { donec := make(chan bool, 1) cst := newClientServerTest(t, h2, HandlerFunc(func(w ResponseWriter, r *Request) { <-donec - })) + }), optQuietLog) defer cst.close() // Note that we use a channel send here and not a close. // The race detector doesn't know that we're waiting for a timeout diff --git a/libgo/go/net/http/clientserver_test.go b/libgo/go/net/http/clientserver_test.go index 4da218b2568..ee48fb05ccd 100644 --- a/libgo/go/net/http/clientserver_test.go +++ b/libgo/go/net/http/clientserver_test.go @@ -560,7 +560,7 @@ func testCancelRequestMidBody(t *testing.T, h2 bool) { if all != "Hello" { t.Errorf("Read %q (%q + %q); want Hello", all, firstRead, rest) } - if !reflect.DeepEqual(err, ExportErrRequestCanceled) { + if err != ExportErrRequestCanceled { t.Errorf("ReadAll error = %v; want %v", err, ExportErrRequestCanceled) } } diff --git a/libgo/go/net/http/clone.go b/libgo/go/net/http/clone.go new file mode 100644 index 00000000000..5f2784d2808 --- /dev/null +++ b/libgo/go/net/http/clone.go @@ -0,0 +1,64 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +import ( + "mime/multipart" + "net/textproto" + "net/url" +) + +func cloneURLValues(v url.Values) url.Values { + if v == nil { + return nil + } + // http.Header and url.Values have the same representation, so temporarily + // treat it like http.Header, which does have a clone: + return url.Values(Header(v).Clone()) +} + +func cloneURL(u *url.URL) *url.URL { + if u == nil { + return nil + } + u2 := new(url.URL) + *u2 = *u + if u.User != nil { + u2.User = new(url.Userinfo) + *u2.User = *u.User + } + return u2 +} + +func cloneMultipartForm(f *multipart.Form) *multipart.Form { + if f == nil { + return nil + } + f2 := &multipart.Form{ + Value: (map[string][]string)(Header(f.Value).Clone()), + } + if f.File != nil { + m := make(map[string][]*multipart.FileHeader) + for k, vv := range f.File { + vv2 := make([]*multipart.FileHeader, len(vv)) + for i, v := range vv { + vv2[i] = cloneMultipartFileHeader(v) + } + m[k] = vv2 + } + f2.File = m + } + return f2 +} + +func cloneMultipartFileHeader(fh *multipart.FileHeader) *multipart.FileHeader { + if fh == nil { + return nil + } + fh2 := new(multipart.FileHeader) + *fh2 = *fh + fh2.Header = textproto.MIMEHeader(Header(fh.Header).Clone()) + return fh2 +} diff --git a/libgo/go/net/http/cookie.go b/libgo/go/net/http/cookie.go index 63f62214db8..91ff544e79f 100644 --- a/libgo/go/net/http/cookie.go +++ b/libgo/go/net/http/cookie.go @@ -48,6 +48,7 @@ const ( SameSiteDefaultMode SameSite = iota + 1 SameSiteLaxMode SameSiteStrictMode + SameSiteNoneMode ) // readSetCookies parses all "Set-Cookie" values from @@ -105,6 +106,8 @@ func readSetCookies(h Header) []*Cookie { c.SameSite = SameSiteLaxMode case "strict": c.SameSite = SameSiteStrictMode + case "none": + c.SameSite = SameSiteNoneMode default: c.SameSite = SameSiteDefaultMode } @@ -168,8 +171,12 @@ func (c *Cookie) String() string { if c == nil || !isCookieNameValid(c.Name) { return "" } + // extraCookieLength derived from typical length of cookie attributes + // see RFC 6265 Sec 4.1. + const extraCookieLength = 110 var b strings.Builder - b.WriteString(sanitizeCookieName(c.Name)) + b.Grow(len(c.Name) + len(c.Value) + len(c.Domain) + len(c.Path) + extraCookieLength) + b.WriteString(c.Name) b.WriteRune('=') b.WriteString(sanitizeCookieValue(c.Value)) @@ -213,6 +220,8 @@ func (c *Cookie) String() string { switch c.SameSite { case SameSiteDefaultMode: b.WriteString("; SameSite") + case SameSiteNoneMode: + b.WriteString("; SameSite=None") case SameSiteLaxMode: b.WriteString("; SameSite=Lax") case SameSiteStrictMode: @@ -226,25 +235,28 @@ func (c *Cookie) String() string { // // if filter isn't empty, only cookies of that name are returned func readCookies(h Header, filter string) []*Cookie { - lines, ok := h["Cookie"] - if !ok { + lines := h["Cookie"] + if len(lines) == 0 { return []*Cookie{} } - cookies := []*Cookie{} + cookies := make([]*Cookie, 0, len(lines)+strings.Count(lines[0], ";")) for _, line := range lines { - parts := strings.Split(strings.TrimSpace(line), ";") - if len(parts) == 1 && parts[0] == "" { - continue - } - // Per-line attributes - for i := 0; i < len(parts); i++ { - parts[i] = strings.TrimSpace(parts[i]) - if len(parts[i]) == 0 { + line = strings.TrimSpace(line) + + var part string + for len(line) > 0 { // continue since we have rest + if splitIndex := strings.Index(line, ";"); splitIndex > 0 { + part, line = line[:splitIndex], line[splitIndex+1:] + } else { + part, line = line, "" + } + part = strings.TrimSpace(part) + if len(part) == 0 { continue } - name, val := parts[i], "" - if j := strings.Index(name, "="); j >= 0 { + name, val := part, "" + if j := strings.Index(part, "="); j >= 0 { name, val = name[:j], name[j+1:] } if !isCookieNameValid(name) { diff --git a/libgo/go/net/http/cookie_test.go b/libgo/go/net/http/cookie_test.go index 022adaa90de..9e8196ebce0 100644 --- a/libgo/go/net/http/cookie_test.go +++ b/libgo/go/net/http/cookie_test.go @@ -77,6 +77,10 @@ var writeSetCookiesTests = []struct { &Cookie{Name: "cookie-14", Value: "samesite-strict", SameSite: SameSiteStrictMode}, "cookie-14=samesite-strict; SameSite=Strict", }, + { + &Cookie{Name: "cookie-15", Value: "samesite-none", SameSite: SameSiteNoneMode}, + "cookie-15=samesite-none; SameSite=None", + }, // The "special" cookies have values containing commas or spaces which // are disallowed by RFC 6265 but are common in the wild. { @@ -127,6 +131,22 @@ var writeSetCookiesTests = []struct { &Cookie{Name: "\t"}, ``, }, + { + &Cookie{Name: "\r"}, + ``, + }, + { + &Cookie{Name: "a\nb", Value: "v"}, + ``, + }, + { + &Cookie{Name: "a\nb", Value: "v"}, + ``, + }, + { + &Cookie{Name: "a\rb", Value: "v"}, + ``, + }, } func TestWriteSetCookies(t *testing.T) { @@ -280,6 +300,15 @@ var readSetCookiesTests = []struct { Raw: "samesitestrict=foo; SameSite=Strict", }}, }, + { + Header{"Set-Cookie": {"samesitenone=foo; SameSite=None"}}, + []*Cookie{{ + Name: "samesitenone", + Value: "foo", + SameSite: SameSiteNoneMode, + Raw: "samesitenone=foo; SameSite=None", + }}, + }, // Make sure we can properly read back the Set-Cookie headers we create // for values containing spaces or commas: { @@ -385,6 +414,19 @@ var readCookiesTests = []struct { {Name: "c2", Value: "v2"}, }, }, + { + Header{"Cookie": {`Cookie-1="v$1"; c2=v2;`}}, + "", + []*Cookie{ + {Name: "Cookie-1", Value: "v$1"}, + {Name: "c2", Value: "v2"}, + }, + }, + { + Header{"Cookie": {``}}, + "", + []*Cookie{}, + }, } func TestReadCookies(t *testing.T) { diff --git a/libgo/go/net/http/export_test.go b/libgo/go/net/http/export_test.go index b6965c239e9..f0dfa8cd336 100644 --- a/libgo/go/net/http/export_test.go +++ b/libgo/go/net/http/export_test.go @@ -33,6 +33,7 @@ var ( ExportHttp2ConfigureServer = http2ConfigureServer Export_shouldCopyHeaderOnRedirect = shouldCopyHeaderOnRedirect Export_writeStatusLine = writeStatusLine + Export_is408Message = is408Message ) const MaxWriteWaitBeforeConnReuse = maxWriteWaitBeforeConnReuse @@ -244,3 +245,18 @@ func ExportSetH2GoawayTimeout(d time.Duration) (restore func()) { } func (r *Request) ExportIsReplayable() bool { return r.isReplayable() } + +// ExportCloseTransportConnsAbruptly closes all idle connections from +// tr in an abrupt way, just reaching into the underlying Conns and +// closing them, without telling the Transport or its persistConns +// that it's doing so. This is to simulate the server closing connections +// on the Transport. +func ExportCloseTransportConnsAbruptly(tr *Transport) { + tr.idleMu.Lock() + for _, pcs := range tr.idleConn { + for _, pc := range pcs { + pc.conn.Close() + } + } + tr.idleMu.Unlock() +} diff --git a/libgo/go/net/http/fs.go b/libgo/go/net/http/fs.go index db44d6b0296..41d46dced2a 100644 --- a/libgo/go/net/http/fs.go +++ b/libgo/go/net/http/fs.go @@ -63,6 +63,8 @@ func mapDirOpenError(originalErr error, name string) error { return originalErr } +// Open implements FileSystem using os.Open, opening files for reading rooted +// and relative to the directory d. func (d Dir) Open(name string) (File, error) { if filepath.Separator != '/' && strings.ContainsRune(name, filepath.Separator) { return nil, errors.New("http: invalid character in file path") diff --git a/libgo/go/net/http/h2_bundle.go b/libgo/go/net/http/h2_bundle.go index a848d68fd9c..2efa0ef125b 100644 --- a/libgo/go/net/http/h2_bundle.go +++ b/libgo/go/net/http/h2_bundle.go @@ -1,5 +1,5 @@ // Code generated by golang.org/x/tools/cmd/bundle. DO NOT EDIT. -//go:generate bundle -o h2_bundle.go -prefix http2 -underscore golang.org/x/net/http2 +//go:generate bundle -o h2_bundle.go -prefix http2 golang.org/x/net/http2 // Package http2 implements the HTTP/2 protocol. // @@ -42,11 +42,12 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" - "internal/x/net/http/httpguts" - "internal/x/net/http2/hpack" - "internal/x/net/idna" + "golang.org/x/net/http/httpguts" + "golang.org/x/net/http2/hpack" + "golang.org/x/net/idna" ) // A list of the possible cipher suite ids. Taken from @@ -1885,7 +1886,7 @@ func (f *http2Framer) WriteData(streamID uint32, endStream bool, data []byte) er return f.WriteDataPadded(streamID, endStream, data, nil) } -// WriteData writes a DATA frame with optional padding. +// WriteDataPadded writes a DATA frame with optional padding. // // If pad is nil, the padding bit is not sent. // The length of pad must not exceed 255 bytes. @@ -3831,7 +3832,20 @@ func http2ConfigureServer(s *Server, conf *http2Server) error { if http2testHookOnConn != nil { http2testHookOnConn() } + // The TLSNextProto interface predates contexts, so + // the net/http package passes down its per-connection + // base context via an exported but unadvertised + // method on the Handler. This is for internal + // net/http<=>http2 use only. + var ctx context.Context + type baseContexter interface { + BaseContext() context.Context + } + if bc, ok := h.(baseContexter); ok { + ctx = bc.BaseContext() + } conf.ServeConn(c, &http2ServeConnOpts{ + Context: ctx, Handler: h, BaseConfig: hs, }) @@ -3842,6 +3856,10 @@ func http2ConfigureServer(s *Server, conf *http2Server) error { // ServeConnOpts are options for the Server.ServeConn method. type http2ServeConnOpts struct { + // Context is the base context to use. + // If nil, context.Background is used. + Context context.Context + // BaseConfig optionally sets the base configuration // for values. If nil, defaults are used. BaseConfig *Server @@ -3852,6 +3870,13 @@ type http2ServeConnOpts struct { Handler Handler } +func (o *http2ServeConnOpts) context() context.Context { + if o.Context != nil { + return o.Context + } + return context.Background() +} + func (o *http2ServeConnOpts) baseConfig() *Server { if o != nil && o.BaseConfig != nil { return o.BaseConfig @@ -3997,7 +4022,7 @@ func (s *http2Server) ServeConn(c net.Conn, opts *http2ServeConnOpts) { } func http2serverConnBaseContext(c net.Conn, opts *http2ServeConnOpts) (ctx context.Context, cancel func()) { - ctx, cancel = context.WithCancel(context.Background()) + ctx, cancel = context.WithCancel(opts.context()) ctx = context.WithValue(ctx, LocalAddrContextKey, c.LocalAddr()) if hs := opts.baseConfig(); hs != nil { ctx = context.WithValue(ctx, ServerContextKey, hs) @@ -5870,7 +5895,16 @@ type http2chunkWriter struct{ rws *http2responseWriterState } func (cw http2chunkWriter) Write(p []byte) (n int, err error) { return cw.rws.writeChunk(p) } -func (rws *http2responseWriterState) hasTrailers() bool { return len(rws.trailers) != 0 } +func (rws *http2responseWriterState) hasTrailers() bool { return len(rws.trailers) > 0 } + +func (rws *http2responseWriterState) hasNonemptyTrailers() bool { + for _, trailer := range rws.trailers { + if _, ok := rws.handlerHeader[trailer]; ok { + return true + } + } + return false +} // declareTrailer is called for each Trailer header when the // response header is written. It notes that a header will need to be @@ -5970,7 +6004,10 @@ func (rws *http2responseWriterState) writeChunk(p []byte) (n int, err error) { rws.promoteUndeclaredTrailers() } - endStream := rws.handlerDone && !rws.hasTrailers() + // only send trailers if they have actually been defined by the + // server handler. + hasNonemptyTrailers := rws.hasNonemptyTrailers() + endStream := rws.handlerDone && !hasNonemptyTrailers if len(p) > 0 || endStream { // only send a 0 byte DATA frame if we're ending the stream. if err := rws.conn.writeDataFromHandler(rws.stream, p, endStream); err != nil { @@ -5979,7 +6016,7 @@ func (rws *http2responseWriterState) writeChunk(p []byte) (n int, err error) { } } - if rws.handlerDone && rws.hasTrailers() { + if rws.handlerDone && hasNonemptyTrailers { err = rws.conn.writeHeaders(rws.stream, &http2writeResHeaders{ streamID: rws.stream.id, h: rws.handlerHeader, @@ -6621,6 +6658,7 @@ type http2ClientConn struct { t *http2Transport tconn net.Conn // usually *tls.Conn, except specialized impls tlsState *tls.ConnectionState // nil only for specialized impls + reused uint32 // whether conn is being reused; atomic singleUse bool // whether being used for a single http.Request // readLoop goroutine fields: @@ -6863,7 +6901,8 @@ func (t *http2Transport) RoundTripOpt(req *Request, opt http2RoundTripOpt) (*Res t.vlogf("http2: Transport failed to get client conn for %s: %v", addr, err) return nil, err } - http2traceGotConn(req, cc) + reused := !atomic.CompareAndSwapUint32(&cc.reused, 0, 1) + http2traceGotConn(req, cc, reused) res, gotErrAfterReqBodyWrite, err := cc.roundTrip(req) if err != nil && retry <= 6 { if req, err = http2shouldRetryRequest(req, err, gotErrAfterReqBodyWrite); err == nil { @@ -7849,7 +7888,11 @@ func (cc *http2ClientConn) encodeHeaders(req *Request, addGzipHeader bool, trail // followed by the query production (see Sections 3.3 and 3.4 of // [RFC3986]). f(":authority", host) - f(":method", req.Method) + m := req.Method + if m == "" { + m = MethodGet + } + f(":method", m) if req.Method != "CONNECT" { f(":path", path) f(":scheme", req.URL.Scheme) @@ -8993,15 +9036,15 @@ func http2traceGetConn(req *Request, hostPort string) { trace.GetConn(hostPort) } -func http2traceGotConn(req *Request, cc *http2ClientConn) { +func http2traceGotConn(req *Request, cc *http2ClientConn, reused bool) { trace := httptrace.ContextClientTrace(req.Context()) if trace == nil || trace.GotConn == nil { return } ci := httptrace.GotConnInfo{Conn: cc.tconn} + ci.Reused = reused cc.mu.Lock() - ci.Reused = cc.nextStreamID > 1 - ci.WasIdle = len(cc.streams) == 0 && ci.Reused + ci.WasIdle = len(cc.streams) == 0 && reused if ci.WasIdle && !cc.lastActive.IsZero() { ci.IdleTime = time.Now().Sub(cc.lastActive) } diff --git a/libgo/go/net/http/header.go b/libgo/go/net/http/header.go index b699e7ef8ff..1e1ed981ecc 100644 --- a/libgo/go/net/http/header.go +++ b/libgo/go/net/http/header.go @@ -78,12 +78,19 @@ func (h Header) write(w io.Writer, trace *httptrace.ClientTrace) error { return h.writeSubset(w, nil, trace) } -func (h Header) clone() Header { +// Clone returns a copy of h. +func (h Header) Clone() Header { + // Find total number of values. + nv := 0 + for _, vv := range h { + nv += len(vv) + } + sv := make([]string, nv) // shared backing array for headers' values h2 := make(Header, len(h)) for k, vv := range h { - vv2 := make([]string, len(vv)) - copy(vv2, vv) - h2[k] = vv2 + n := copy(sv, vv) + h2[k] = sv[:n:n] + sv = sv[n:] } return h2 } diff --git a/libgo/go/net/http/http.go b/libgo/go/net/http/http.go index e5d59e14120..3510fe604d0 100644 --- a/libgo/go/net/http/http.go +++ b/libgo/go/net/http/http.go @@ -11,7 +11,7 @@ import ( "time" "unicode/utf8" - "internal/x/net/http/httpguts" + "golang.org/x/net/http/httpguts" ) // maxInt64 is the effective "infinite" value for the Server and @@ -19,7 +19,7 @@ import ( const maxInt64 = 1<<63 - 1 // aLongTimeAgo is a non-zero time, far in the past, used for -// immediate cancelation of network operations. +// immediate cancellation of network operations. var aLongTimeAgo = time.Unix(1, 0) // TODO(bradfitz): move common stuff here. The other files have accumulated diff --git a/libgo/go/net/http/httptest/recorder.go b/libgo/go/net/http/httptest/recorder.go index f2c3c0757ba..d0bc0fade98 100644 --- a/libgo/go/net/http/httptest/recorder.go +++ b/libgo/go/net/http/httptest/recorder.go @@ -12,7 +12,7 @@ import ( "strconv" "strings" - "internal/x/net/http/httpguts" + "golang.org/x/net/http/httpguts" ) // ResponseRecorder is an implementation of http.ResponseWriter that @@ -59,7 +59,10 @@ func NewRecorder() *ResponseRecorder { // an explicit DefaultRemoteAddr isn't set on ResponseRecorder. const DefaultRemoteAddr = "1.2.3.4" -// Header returns the response headers. +// Header implements http.ResponseWriter. It returns the response +// headers to mutate within a handler. To test the headers that were +// written after a handler completes, use the Result method and see +// the returned Response value's Header. func (rw *ResponseRecorder) Header() http.Header { m := rw.HeaderMap if m == nil { @@ -98,7 +101,8 @@ func (rw *ResponseRecorder) writeHeader(b []byte, str string) { rw.WriteHeader(200) } -// Write always succeeds and writes to rw.Body, if not nil. +// Write implements http.ResponseWriter. The data in buf is written to +// rw.Body, if not nil. func (rw *ResponseRecorder) Write(buf []byte) (int, error) { rw.writeHeader(buf, "") if rw.Body != nil { @@ -107,7 +111,8 @@ func (rw *ResponseRecorder) Write(buf []byte) (int, error) { return len(buf), nil } -// WriteString always succeeds and writes to rw.Body, if not nil. +// WriteString implements io.StringWriter. The data in str is written +// to rw.Body, if not nil. func (rw *ResponseRecorder) WriteString(str string) (int, error) { rw.writeHeader(nil, str) if rw.Body != nil { @@ -116,8 +121,7 @@ func (rw *ResponseRecorder) WriteString(str string) (int, error) { return len(str), nil } -// WriteHeader sets rw.Code. After it is called, changing rw.Header -// will not affect rw.HeaderMap. +// WriteHeader implements http.ResponseWriter. func (rw *ResponseRecorder) WriteHeader(code int) { if rw.wroteHeader { return @@ -127,20 +131,11 @@ func (rw *ResponseRecorder) WriteHeader(code int) { if rw.HeaderMap == nil { rw.HeaderMap = make(http.Header) } - rw.snapHeader = cloneHeader(rw.HeaderMap) + rw.snapHeader = rw.HeaderMap.Clone() } -func cloneHeader(h http.Header) http.Header { - h2 := make(http.Header, len(h)) - for k, vv := range h { - vv2 := make([]string, len(vv)) - copy(vv2, vv) - h2[k] = vv2 - } - return h2 -} - -// Flush sets rw.Flushed to true. +// Flush implements http.Flusher. To test whether Flush was +// called, see rw.Flushed. func (rw *ResponseRecorder) Flush() { if !rw.wroteHeader { rw.WriteHeader(200) @@ -168,7 +163,7 @@ func (rw *ResponseRecorder) Result() *http.Response { return rw.result } if rw.snapHeader == nil { - rw.snapHeader = cloneHeader(rw.HeaderMap) + rw.snapHeader = rw.HeaderMap.Clone() } res := &http.Response{ Proto: "HTTP/1.1", diff --git a/libgo/go/net/http/httputil/dump_test.go b/libgo/go/net/http/httputil/dump_test.go index 63312dd8856..97954ca88d0 100644 --- a/libgo/go/net/http/httputil/dump_test.go +++ b/libgo/go/net/http/httputil/dump_test.go @@ -18,7 +18,10 @@ import ( ) type dumpTest struct { - Req http.Request + // Either Req or GetReq can be set/nil but not both. + Req *http.Request + GetReq func() *http.Request + Body interface{} // optional []byte or func() io.ReadCloser to populate Req.Body WantDump string @@ -29,7 +32,7 @@ type dumpTest struct { var dumpTests = []dumpTest{ // HTTP/1.1 => chunked coding; body; empty trailer { - Req: http.Request{ + Req: &http.Request{ Method: "GET", URL: &url.URL{ Scheme: "http", @@ -52,7 +55,7 @@ var dumpTests = []dumpTest{ // Verify that DumpRequest preserves the HTTP version number, doesn't add a Host, // and doesn't add a User-Agent. { - Req: http.Request{ + Req: &http.Request{ Method: "GET", URL: mustParseURL("/foo"), ProtoMajor: 1, @@ -67,7 +70,7 @@ var dumpTests = []dumpTest{ }, { - Req: *mustNewRequest("GET", "http://example.com/foo", nil), + Req: mustNewRequest("GET", "http://example.com/foo", nil), WantDumpOut: "GET /foo HTTP/1.1\r\n" + "Host: example.com\r\n" + @@ -79,8 +82,7 @@ var dumpTests = []dumpTest{ // with a bytes.Buffer and hang with all goroutines not // runnable. { - Req: *mustNewRequest("GET", "https://example.com/foo", nil), - + Req: mustNewRequest("GET", "https://example.com/foo", nil), WantDumpOut: "GET /foo HTTP/1.1\r\n" + "Host: example.com\r\n" + "User-Agent: Go-http-client/1.1\r\n" + @@ -89,7 +91,7 @@ var dumpTests = []dumpTest{ // Request with Body, but Dump requested without it. { - Req: http.Request{ + Req: &http.Request{ Method: "POST", URL: &url.URL{ Scheme: "http", @@ -114,7 +116,7 @@ var dumpTests = []dumpTest{ // Request with Body > 8196 (default buffer size) { - Req: http.Request{ + Req: &http.Request{ Method: "POST", URL: &url.URL{ Scheme: "http", @@ -145,8 +147,10 @@ var dumpTests = []dumpTest{ }, { - Req: *mustReadRequest("GET http://foo.com/ HTTP/1.1\r\n" + - "User-Agent: blah\r\n\r\n"), + GetReq: func() *http.Request { + return mustReadRequest("GET http://foo.com/ HTTP/1.1\r\n" + + "User-Agent: blah\r\n\r\n") + }, NoBody: true, WantDump: "GET http://foo.com/ HTTP/1.1\r\n" + "User-Agent: blah\r\n\r\n", @@ -154,22 +158,25 @@ var dumpTests = []dumpTest{ // Issue #7215. DumpRequest should return the "Content-Length" when set { - Req: *mustReadRequest("POST /v2/api/?login HTTP/1.1\r\n" + - "Host: passport.myhost.com\r\n" + - "Content-Length: 3\r\n" + - "\r\nkey1=name1&key2=name2"), + GetReq: func() *http.Request { + return mustReadRequest("POST /v2/api/?login HTTP/1.1\r\n" + + "Host: passport.myhost.com\r\n" + + "Content-Length: 3\r\n" + + "\r\nkey1=name1&key2=name2") + }, WantDump: "POST /v2/api/?login HTTP/1.1\r\n" + "Host: passport.myhost.com\r\n" + "Content-Length: 3\r\n" + "\r\nkey", }, - // Issue #7215. DumpRequest should return the "Content-Length" in ReadRequest { - Req: *mustReadRequest("POST /v2/api/?login HTTP/1.1\r\n" + - "Host: passport.myhost.com\r\n" + - "Content-Length: 0\r\n" + - "\r\nkey1=name1&key2=name2"), + GetReq: func() *http.Request { + return mustReadRequest("POST /v2/api/?login HTTP/1.1\r\n" + + "Host: passport.myhost.com\r\n" + + "Content-Length: 0\r\n" + + "\r\nkey1=name1&key2=name2") + }, WantDump: "POST /v2/api/?login HTTP/1.1\r\n" + "Host: passport.myhost.com\r\n" + "Content-Length: 0\r\n\r\n", @@ -177,9 +184,11 @@ var dumpTests = []dumpTest{ // Issue #7215. DumpRequest should not return the "Content-Length" if unset { - Req: *mustReadRequest("POST /v2/api/?login HTTP/1.1\r\n" + - "Host: passport.myhost.com\r\n" + - "\r\nkey1=name1&key2=name2"), + GetReq: func() *http.Request { + return mustReadRequest("POST /v2/api/?login HTTP/1.1\r\n" + + "Host: passport.myhost.com\r\n" + + "\r\nkey1=name1&key2=name2") + }, WantDump: "POST /v2/api/?login HTTP/1.1\r\n" + "Host: passport.myhost.com\r\n\r\n", }, @@ -187,8 +196,7 @@ var dumpTests = []dumpTest{ // Issue 18506: make drainBody recognize NoBody. Otherwise // this was turning into a chunked request. { - Req: *mustNewRequest("POST", "http://example.com/foo", http.NoBody), - + Req: mustNewRequest("POST", "http://example.com/foo", http.NoBody), WantDumpOut: "POST /foo HTTP/1.1\r\n" + "Host: example.com\r\n" + "User-Agent: Go-http-client/1.1\r\n" + @@ -200,28 +208,40 @@ var dumpTests = []dumpTest{ func TestDumpRequest(t *testing.T) { numg0 := runtime.NumGoroutine() for i, tt := range dumpTests { - setBody := func() { - if tt.Body == nil { - return + if tt.Req != nil && tt.GetReq != nil || tt.Req == nil && tt.GetReq == nil { + t.Errorf("#%d: either .Req(%p) or .GetReq(%p) can be set/nil but not both", i, tt.Req, tt.GetReq) + continue + } + + freshReq := func(ti dumpTest) *http.Request { + req := ti.Req + if req == nil { + req = ti.GetReq() } - switch b := tt.Body.(type) { + + if req.Header == nil { + req.Header = make(http.Header) + } + + if ti.Body == nil { + return req + } + switch b := ti.Body.(type) { case []byte: - tt.Req.Body = ioutil.NopCloser(bytes.NewReader(b)) + req.Body = ioutil.NopCloser(bytes.NewReader(b)) case func() io.ReadCloser: - tt.Req.Body = b() + req.Body = b() default: - t.Fatalf("Test %d: unsupported Body of %T", i, tt.Body) + t.Fatalf("Test %d: unsupported Body of %T", i, ti.Body) } - } - if tt.Req.Header == nil { - tt.Req.Header = make(http.Header) + return req } if tt.WantDump != "" { - setBody() - dump, err := DumpRequest(&tt.Req, !tt.NoBody) + req := freshReq(tt) + dump, err := DumpRequest(req, !tt.NoBody) if err != nil { - t.Errorf("DumpRequest #%d: %s", i, err) + t.Errorf("DumpRequest #%d: %s\nWantDump:\n%s", i, err, tt.WantDump) continue } if string(dump) != tt.WantDump { @@ -231,8 +251,8 @@ func TestDumpRequest(t *testing.T) { } if tt.WantDumpOut != "" { - setBody() - dump, err := DumpRequestOut(&tt.Req, !tt.NoBody) + req := freshReq(tt) + dump, err := DumpRequestOut(req, !tt.NoBody) if err != nil { t.Errorf("DumpRequestOut #%d: %s", i, err) continue diff --git a/libgo/go/net/http/httputil/persist.go b/libgo/go/net/http/httputil/persist.go index cbedf25ad1b..84b116df8cf 100644 --- a/libgo/go/net/http/httputil/persist.go +++ b/libgo/go/net/http/httputil/persist.go @@ -292,8 +292,8 @@ func (cc *ClientConn) Close() error { } // Write writes a request. An ErrPersistEOF error is returned if the connection -// has been closed in an HTTP keepalive sense. If req.Close equals true, the -// keepalive connection is logically closed after this request and the opposing +// has been closed in an HTTP keep-alive sense. If req.Close equals true, the +// keep-alive connection is logically closed after this request and the opposing // server is informed. An ErrUnexpectedEOF indicates the remote closed the // underlying TCP connection, which is usually considered as graceful close. func (cc *ClientConn) Write(req *http.Request) error { diff --git a/libgo/go/net/http/httputil/reverseproxy.go b/libgo/go/net/http/httputil/reverseproxy.go index 4b165d65a6a..1d7b0efa11b 100644 --- a/libgo/go/net/http/httputil/reverseproxy.go +++ b/libgo/go/net/http/httputil/reverseproxy.go @@ -18,7 +18,7 @@ import ( "sync" "time" - "internal/x/net/http/httpguts" + "golang.org/x/net/http/httpguts" ) // ReverseProxy is an HTTP Handler that takes an incoming request and @@ -51,8 +51,7 @@ type ReverseProxy struct { // ErrorLog specifies an optional logger for errors // that occur when attempting to proxy the request. - // If nil, logging goes to os.Stderr via the log package's - // standard logger. + // If nil, logging is done via the log package's standard logger. ErrorLog *log.Logger // BufferPool optionally specifies a buffer pool to @@ -132,16 +131,6 @@ func copyHeader(dst, src http.Header) { } } -func cloneHeader(h http.Header) http.Header { - h2 := make(http.Header, len(h)) - for k, vv := range h { - vv2 := make([]string, len(vv)) - copy(vv2, vv) - h2[k] = vv2 - } - return h2 -} - // Hop-by-hop headers. These are removed when sent to the backend. // As of RFC 7230, hop-by-hop headers are required to appear in the // Connection header field. These are the headers defined by the @@ -206,13 +195,11 @@ func (p *ReverseProxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) { }() } - outreq := req.WithContext(ctx) // includes shallow copies of maps, but okay + outreq := req.Clone(ctx) if req.ContentLength == 0 { outreq.Body = nil // Issue 16036: nil Body for http.Transport retries } - outreq.Header = cloneHeader(req.Header) - p.Director(outreq) outreq.Close = false @@ -357,10 +344,10 @@ func shouldPanicOnCopyError(req *http.Request) bool { // removeConnectionHeaders removes hop-by-hop headers listed in the "Connection" header of h. // See RFC 7230, section 6.1 func removeConnectionHeaders(h http.Header) { - if c := h.Get("Connection"); c != "" { - for _, f := range strings.Split(c, ",") { - if f = strings.TrimSpace(f); f != "" { - h.Del(f) + for _, f := range h["Connection"] { + for _, sf := range strings.Split(f, ",") { + if sf = strings.TrimSpace(sf); sf != "" { + h.Del(sf) } } } diff --git a/libgo/go/net/http/httputil/reverseproxy_test.go b/libgo/go/net/http/httputil/reverseproxy_test.go index 367ba73ae24..e8cb8149387 100644 --- a/libgo/go/net/http/httputil/reverseproxy_test.go +++ b/libgo/go/net/http/httputil/reverseproxy_test.go @@ -20,6 +20,7 @@ import ( "net/url" "os" "reflect" + "sort" "strconv" "strings" "sync" @@ -160,13 +161,17 @@ func TestReverseProxyStripHeadersPresentInConnection(t *testing.T) { const someConnHeader = "X-Some-Conn-Header" backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if c := r.Header.Get("Connection"); c != "" { + t.Errorf("handler got header %q = %q; want empty", "Connection", c) + } if c := r.Header.Get(fakeConnectionToken); c != "" { t.Errorf("handler got header %q = %q; want empty", fakeConnectionToken, c) } if c := r.Header.Get(someConnHeader); c != "" { t.Errorf("handler got header %q = %q; want empty", someConnHeader, c) } - w.Header().Set("Connection", someConnHeader+", "+fakeConnectionToken) + w.Header().Add("Connection", "Upgrade, "+fakeConnectionToken) + w.Header().Add("Connection", someConnHeader) w.Header().Set(someConnHeader, "should be deleted") w.Header().Set(fakeConnectionToken, "should be deleted") io.WriteString(w, backendResponse) @@ -179,15 +184,34 @@ func TestReverseProxyStripHeadersPresentInConnection(t *testing.T) { proxyHandler := NewSingleHostReverseProxy(backendURL) frontend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { proxyHandler.ServeHTTP(w, r) - if c := r.Header.Get(someConnHeader); c != "original value" { - t.Errorf("handler modified header %q = %q; want %q", someConnHeader, c, "original value") + if c := r.Header.Get(someConnHeader); c != "should be deleted" { + t.Errorf("handler modified header %q = %q; want %q", someConnHeader, c, "should be deleted") + } + if c := r.Header.Get(fakeConnectionToken); c != "should be deleted" { + t.Errorf("handler modified header %q = %q; want %q", fakeConnectionToken, c, "should be deleted") + } + c := r.Header["Connection"] + var cf []string + for _, f := range c { + for _, sf := range strings.Split(f, ",") { + if sf = strings.TrimSpace(sf); sf != "" { + cf = append(cf, sf) + } + } + } + sort.Strings(cf) + expectedValues := []string{"Upgrade", someConnHeader, fakeConnectionToken} + sort.Strings(expectedValues) + if !reflect.DeepEqual(cf, expectedValues) { + t.Errorf("handler modified header %q = %q; want %q", "Connection", cf, expectedValues) } })) defer frontend.Close() getReq, _ := http.NewRequest("GET", frontend.URL, nil) - getReq.Header.Set("Connection", someConnHeader+", "+fakeConnectionToken) - getReq.Header.Set(someConnHeader, "original value") + getReq.Header.Add("Connection", "Upgrade, "+fakeConnectionToken) + getReq.Header.Add("Connection", someConnHeader) + getReq.Header.Set(someConnHeader, "should be deleted") getReq.Header.Set(fakeConnectionToken, "should be deleted") res, err := frontend.Client().Do(getReq) if err != nil { @@ -201,6 +225,9 @@ func TestReverseProxyStripHeadersPresentInConnection(t *testing.T) { if got, want := string(bodyBytes), backendResponse; got != want { t.Errorf("got body %q; want %q", got, want) } + if c := res.Header.Get("Connection"); c != "" { + t.Errorf("handler got header %q = %q; want empty", "Connection", c) + } if c := res.Header.Get(someConnHeader); c != "" { t.Errorf("handler got header %q = %q; want empty", someConnHeader, c) } diff --git a/libgo/go/net/http/internal/testcert.go b/libgo/go/net/http/internal/testcert.go index 407890920fa..2284a836fb7 100644 --- a/libgo/go/net/http/internal/testcert.go +++ b/libgo/go/net/http/internal/testcert.go @@ -4,6 +4,8 @@ package internal +import "strings" + // LocalhostCert is a PEM-encoded TLS cert with SAN IPs // "127.0.0.1" and "[::1]", expiring at Jan 29 16:00:00 2084 GMT. // generated from src/crypto/tls: @@ -24,7 +26,7 @@ fblo6RBxUQ== -----END CERTIFICATE-----`) // LocalhostKey is the private key for localhostCert. -var LocalhostKey = []byte(`-----BEGIN RSA PRIVATE KEY----- +var LocalhostKey = []byte(testingKey(`-----BEGIN RSA TESTING KEY----- MIICXgIBAAKBgQDuLnQAI3mDgey3VBzWnB2L39JUU4txjeVE6myuDqkM/uGlfjb9 SjY1bIw4iA5sBBZzHi3z0h1YV8QPuxEbi4nW91IJm2gsvvZhIrCHS3l6afab4pZB l2+XsDulrKBxKKtD1rGxlG4LjncdabFn9gvLZad2bSysqz/qTAUStTvqJQIDAQAB @@ -38,4 +40,6 @@ fQtuUE9txblTu14q3N7gHRZB4ZMhFYyDy8CKrN2cPg/Fvyt0Xlp/DoCzjA0CQQDU y2ptGsuSmgUtWj3NM9xuwYPm+Z/F84K6+ARYiZ6PYj013sovGKUFfYAqVXVlxtIX qyUBnu3X9ps8ZfjLZO7BAkEAlT4R5Yl6cGhaJQYZHOde3JEMhNRcVFMO8dJDaFeo f9Oeos0UUothgiDktdQHxdNEwLjQf7lJJBzV+5OtwswCWA== ------END RSA PRIVATE KEY-----`) +-----END RSA TESTING KEY-----`)) + +func testingKey(s string) string { return strings.ReplaceAll(s, "TESTING KEY", "PRIVATE KEY") } diff --git a/libgo/go/net/http/request.go b/libgo/go/net/http/request.go index dcad2b6fab3..fa63175c20c 100644 --- a/libgo/go/net/http/request.go +++ b/libgo/go/net/http/request.go @@ -26,7 +26,7 @@ import ( "strings" "sync" - "internal/x/net/idna" + "golang.org/x/net/idna" ) const ( @@ -304,7 +304,7 @@ type Request struct { // // For server requests, this field is not applicable. // - // Deprecated: Use the Context and WithContext methods + // Deprecated: Set the Request's context with NewRequestWithContext // instead. If a Request's Cancel field and context are both // set, it is undefined whether Cancel is respected. Cancel <-chan struct{} @@ -327,7 +327,7 @@ type Request struct { // The returned context is always non-nil; it defaults to the // background context. // -// For outgoing client requests, the context controls cancelation. +// For outgoing client requests, the context controls cancellation. // // For incoming server requests, the context is canceled when the // client's connection closes, the request is canceled (with HTTP/2), @@ -345,6 +345,11 @@ func (r *Request) Context() context.Context { // For outgoing client request, the context controls the entire // lifetime of a request and its response: obtaining a connection, // sending the request, and reading the response headers and body. +// +// To create a new request with a context, use NewRequestWithContext. +// To change the context of a request (such as an incoming) you then +// also want to modify to send back out, use Request.Clone. Between +// those two uses, it's rare to need WithContext. func (r *Request) WithContext(ctx context.Context) *Request { if ctx == nil { panic("nil context") @@ -352,16 +357,38 @@ func (r *Request) WithContext(ctx context.Context) *Request { r2 := new(Request) *r2 = *r r2.ctx = ctx + r2.URL = cloneURL(r.URL) // legacy behavior; TODO: try to remove. Issue 23544 + return r2 +} - // Deep copy the URL because it isn't - // a map and the URL is mutable by users - // of WithContext. - if r.URL != nil { - r2URL := new(url.URL) - *r2URL = *r.URL - r2.URL = r2URL +// Clone returns a deep copy of r with its context changed to ctx. +// The provided ctx must be non-nil. +// +// For an outgoing client request, the context controls the entire +// lifetime of a request and its response: obtaining a connection, +// sending the request, and reading the response headers and body. +func (r *Request) Clone(ctx context.Context) *Request { + if ctx == nil { + panic("nil context") } - + r2 := new(Request) + *r2 = *r + r2.ctx = ctx + r2.URL = cloneURL(r.URL) + if r.Header != nil { + r2.Header = r.Header.Clone() + } + if r.Trailer != nil { + r2.Trailer = r.Trailer.Clone() + } + if s := r.TransferEncoding; s != nil { + s2 := make([]string, len(s)) + copy(s2, s) + r2.TransferEncoding = s + } + r2.Form = cloneURLValues(r.Form) + r2.PostForm = cloneURLValues(r.PostForm) + r2.MultipartForm = cloneMultipartForm(r.MultipartForm) return r2 } @@ -781,25 +808,34 @@ func validMethod(method string) bool { return len(method) > 0 && strings.IndexFunc(method, isNotToken) == -1 } -// NewRequest returns a new Request given a method, URL, and optional body. +// NewRequest wraps NewRequestWithContext using the background context. +func NewRequest(method, url string, body io.Reader) (*Request, error) { + return NewRequestWithContext(context.Background(), method, url, body) +} + +// NewRequestWithContext returns a new Request given a method, URL, and +// optional body. // // If the provided body is also an io.Closer, the returned // Request.Body is set to body and will be closed by the Client // methods Do, Post, and PostForm, and Transport.RoundTrip. // -// NewRequest returns a Request suitable for use with Client.Do or -// Transport.RoundTrip. To create a request for use with testing a -// Server Handler, either use the NewRequest function in the +// NewRequestWithContext returns a Request suitable for use with +// Client.Do or Transport.RoundTrip. To create a request for use with +// testing a Server Handler, either use the NewRequest function in the // net/http/httptest package, use ReadRequest, or manually update the -// Request fields. See the Request type's documentation for the -// difference between inbound and outbound request fields. +// Request fields. For an outgoing client request, the context +// controls the entire lifetime of a request and its response: +// obtaining a connection, sending the request, and reading the +// response headers and body. See the Request type's documentation for +// the difference between inbound and outbound request fields. // // If body is of type *bytes.Buffer, *bytes.Reader, or // *strings.Reader, the returned request's ContentLength is set to its // exact value (instead of -1), GetBody is populated (so 307 and 308 // redirects can replay the body), and Body is set to NoBody if the // ContentLength is 0. -func NewRequest(method, url string, body io.Reader) (*Request, error) { +func NewRequestWithContext(ctx context.Context, method, url string, body io.Reader) (*Request, error) { if method == "" { // We document that "" means "GET" for Request.Method, and people have // relied on that from NewRequest, so keep that working. @@ -809,6 +845,9 @@ func NewRequest(method, url string, body io.Reader) (*Request, error) { if !validMethod(method) { return nil, fmt.Errorf("net/http: invalid method %q", method) } + if ctx == nil { + return nil, errors.New("net/http: nil Context") + } u, err := parseURL(url) // Just url.Parse (url is shadowed for godoc). if err != nil { return nil, err @@ -820,6 +859,7 @@ func NewRequest(method, url string, body io.Reader) (*Request, error) { // The host's colon:port should be normalized. See Issue 14836. u.Host = removeEmptyPort(u.Host) req := &Request{ + ctx: ctx, Method: method, URL: u, Proto: "HTTP/1.1", @@ -912,6 +952,10 @@ func parseBasicAuth(auth string) (username, password string, ok bool) { // // With HTTP Basic Authentication the provided username and password // are not encrypted. +// +// Some protocols may impose additional requirements on pre-escaping the +// username and password. For instance, when used with OAuth2, both arguments +// must be URL encoded first with url.QueryEscape. func (r *Request) SetBasicAuth(username, password string) { r.Header.Set("Authorization", "Basic "+basicAuth(username, password)) } diff --git a/libgo/go/net/http/request_test.go b/libgo/go/net/http/request_test.go index e8005571df9..b072f958024 100644 --- a/libgo/go/net/http/request_test.go +++ b/libgo/go/net/http/request_test.go @@ -8,12 +8,14 @@ import ( "bufio" "bytes" "context" + "crypto/rand" "encoding/base64" "fmt" "io" "io/ioutil" "mime/multipart" . "net/http" + "net/http/httptest" "net/url" "os" "reflect" @@ -133,30 +135,31 @@ func TestParseFormInitializeOnError(t *testing.T) { } func TestMultipartReader(t *testing.T) { - req := &Request{ - Method: "POST", - Header: Header{"Content-Type": {`multipart/form-data; boundary="foo123"`}}, - Body: ioutil.NopCloser(new(bytes.Buffer)), - } - multipart, err := req.MultipartReader() - if multipart == nil { - t.Errorf("expected multipart; error: %v", err) - } - - req = &Request{ - Method: "POST", - Header: Header{"Content-Type": {`multipart/mixed; boundary="foo123"`}}, - Body: ioutil.NopCloser(new(bytes.Buffer)), - } - multipart, err = req.MultipartReader() - if multipart == nil { - t.Errorf("expected multipart; error: %v", err) + tests := []struct { + shouldError bool + contentType string + }{ + {false, `multipart/form-data; boundary="foo123"`}, + {false, `multipart/mixed; boundary="foo123"`}, + {true, `text/plain`}, } - req.Header = Header{"Content-Type": {"text/plain"}} - multipart, err = req.MultipartReader() - if multipart != nil { - t.Error("unexpected multipart for text/plain") + for i, test := range tests { + req := &Request{ + Method: "POST", + Header: Header{"Content-Type": {test.contentType}}, + Body: ioutil.NopCloser(new(bytes.Buffer)), + } + multipart, err := req.MultipartReader() + if test.shouldError { + if err == nil || multipart != nil { + t.Errorf("test %d: unexpectedly got nil-error (%v) or non-nil-multipart (%v)", i, err, multipart) + } + continue + } + if err != nil || multipart == nil { + t.Errorf("test %d: unexpectedly got error (%v) or nil-multipart (%v)", i, err, multipart) + } } } @@ -1046,3 +1049,92 @@ func BenchmarkReadRequestWrk(b *testing.B) { Host: localhost:8080 `) } + +const ( + withTLS = true + noTLS = false +) + +func BenchmarkFileAndServer_1KB(b *testing.B) { + benchmarkFileAndServer(b, 1<<10) +} + +func BenchmarkFileAndServer_16MB(b *testing.B) { + benchmarkFileAndServer(b, 1<<24) +} + +func BenchmarkFileAndServer_64MB(b *testing.B) { + benchmarkFileAndServer(b, 1<<26) +} + +func benchmarkFileAndServer(b *testing.B, n int64) { + f, err := ioutil.TempFile(os.TempDir(), "go-bench-http-file-and-server") + if err != nil { + b.Fatalf("Failed to create temp file: %v", err) + } + + defer func() { + f.Close() + os.RemoveAll(f.Name()) + }() + + if _, err := io.CopyN(f, rand.Reader, n); err != nil { + b.Fatalf("Failed to copy %d bytes: %v", n, err) + } + + b.Run("NoTLS", func(b *testing.B) { + runFileAndServerBenchmarks(b, noTLS, f, n) + }) + + b.Run("TLS", func(b *testing.B) { + runFileAndServerBenchmarks(b, withTLS, f, n) + }) +} + +func runFileAndServerBenchmarks(b *testing.B, tlsOption bool, f *os.File, n int64) { + handler := HandlerFunc(func(rw ResponseWriter, req *Request) { + defer req.Body.Close() + nc, err := io.Copy(ioutil.Discard, req.Body) + if err != nil { + panic(err) + } + + if nc != n { + panic(fmt.Errorf("Copied %d Wanted %d bytes", nc, n)) + } + }) + + var cst *httptest.Server + if tlsOption == withTLS { + cst = httptest.NewTLSServer(handler) + } else { + cst = httptest.NewServer(handler) + } + + defer cst.Close() + b.ResetTimer() + for i := 0; i < b.N; i++ { + // Perform some setup. + b.StopTimer() + if _, err := f.Seek(0, 0); err != nil { + b.Fatalf("Failed to seek back to file: %v", err) + } + + b.StartTimer() + req, err := NewRequest("PUT", cst.URL, ioutil.NopCloser(f)) + if err != nil { + b.Fatal(err) + } + + req.ContentLength = n + // Prevent mime sniffing by setting the Content-Type. + req.Header.Set("Content-Type", "application/octet-stream") + res, err := cst.Client().Do(req) + if err != nil { + b.Fatalf("Failed to make request to backend: %v", err) + } + + res.Body.Close() + b.SetBytes(n) + } +} diff --git a/libgo/go/net/http/response.go b/libgo/go/net/http/response.go index f906ce829b4..2065a250156 100644 --- a/libgo/go/net/http/response.go +++ b/libgo/go/net/http/response.go @@ -12,12 +12,13 @@ import ( "crypto/tls" "errors" "fmt" - "internal/x/net/http/httpguts" "io" "net/textproto" "net/url" "strconv" "strings" + + "golang.org/x/net/http/httpguts" ) var respExcludeHeader = map[string]bool{ @@ -66,7 +67,7 @@ type Response struct { // with a "chunked" Transfer-Encoding. // // As of Go 1.12, the Body will be also implement io.Writer - // on a successful "101 Switching Protocols" responses, + // on a successful "101 Switching Protocols" response, // as used by WebSockets and HTTP/2's "h2c" mode. Body io.ReadCloser diff --git a/libgo/go/net/http/response_test.go b/libgo/go/net/http/response_test.go index c46f13f7988..ee7f0d0b708 100644 --- a/libgo/go/net/http/response_test.go +++ b/libgo/go/net/http/response_test.go @@ -10,7 +10,7 @@ import ( "compress/gzip" "crypto/rand" "fmt" - "go/ast" + "go/token" "io" "io/ioutil" "net/http/internal" @@ -736,7 +736,7 @@ func diff(t *testing.T, prefix string, have, want interface{}) { } for i := 0; i < hv.NumField(); i++ { name := hv.Type().Field(i).Name - if !ast.IsExported(name) { + if !token.IsExported(name) { continue } hf := hv.Field(i).Interface() diff --git a/libgo/go/net/http/roundtrip_js.go b/libgo/go/net/http/roundtrip_js.go index 1e38b908d38..6331351a838 100644 --- a/libgo/go/net/http/roundtrip_js.go +++ b/libgo/go/net/http/roundtrip_js.go @@ -11,12 +11,12 @@ import ( "fmt" "io" "io/ioutil" - "os" "strconv" - "strings" "syscall/js" ) +var uint8Array = js.Global().Get("Uint8Array") + // jsFetchMode is a Request.Header map key that, if present, // signals that the map entry is actually an option to the Fetch API mode setting. // Valid values are: "cors", "no-cors", "same-origin", "navigate" @@ -33,9 +33,19 @@ const jsFetchMode = "js.fetch:mode" // Reference: https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch#Parameters const jsFetchCreds = "js.fetch:credentials" +// jsFetchRedirect is a Request.Header map key that, if present, +// signals that the map entry is actually an option to the Fetch API redirect setting. +// Valid values are: "follow", "error", "manual" +// The default is "follow". +// +// Reference: https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch#Parameters +const jsFetchRedirect = "js.fetch:redirect" + +var useFakeNetwork = js.Global().Get("fetch") == js.Undefined() + // RoundTrip implements the RoundTripper interface using the WHATWG Fetch API. func (t *Transport) RoundTrip(req *Request) (*Response, error) { - if useFakeNetwork() { + if useFakeNetwork { return t.roundTrip(req) } @@ -60,6 +70,10 @@ func (t *Transport) RoundTrip(req *Request) (*Response, error) { opt.Set("mode", h) req.Header.Del(jsFetchMode) } + if h := req.Header.Get(jsFetchRedirect); h != "" { + opt.Set("redirect", h) + req.Header.Del(jsFetchRedirect) + } if ac != js.Undefined() { opt.Set("signal", ac.Get("signal")) } @@ -84,9 +98,9 @@ func (t *Transport) RoundTrip(req *Request) (*Response, error) { return nil, err } req.Body.Close() - a := js.TypedArrayOf(body) - defer a.Release() - opt.Set("body", a) + buf := uint8Array.New(len(body)) + js.CopyBytesToJS(buf, body) + opt.Set("body", buf) } respPromise := js.Global().Call("fetch", req.URL.String(), opt) var ( @@ -126,10 +140,11 @@ func (t *Transport) RoundTrip(req *Request) (*Response, error) { body = &arrayReader{arrayPromise: result.Call("arrayBuffer")} } + code := result.Get("status").Int() select { case respCh <- &Response{ - Status: result.Get("status").String() + " " + StatusText(result.Get("status").Int()), - StatusCode: result.Get("status").Int(), + Status: fmt.Sprintf("%d %s", code, StatusText(code)), + StatusCode: code, Header: header, ContentLength: contentLength, Body: body, @@ -167,12 +182,6 @@ func (t *Transport) RoundTrip(req *Request) (*Response, error) { var errClosed = errors.New("net/http: reader is closed") -// useFakeNetwork is used to determine whether the request is made -// by a test and should be made to use the fake in-memory network. -func useFakeNetwork() bool { - return len(os.Args) > 0 && strings.HasSuffix(os.Args[0], ".test") -} - // streamReader implements an io.ReadCloser wrapper for ReadableStream. // See https://fetch.spec.whatwg.org/#readablestream for more information. type streamReader struct { @@ -197,9 +206,7 @@ func (r *streamReader) Read(p []byte) (n int, err error) { return nil } value := make([]byte, result.Get("value").Get("byteLength").Int()) - a := js.TypedArrayOf(value) - a.Call("set", result.Get("value")) - a.Release() + js.CopyBytesToGo(value, result.Get("value")) bCh <- value return nil }) @@ -260,11 +267,9 @@ func (r *arrayReader) Read(p []byte) (n int, err error) { ) success := js.FuncOf(func(this js.Value, args []js.Value) interface{} { // Wrap the input ArrayBuffer with a Uint8Array - uint8arrayWrapper := js.Global().Get("Uint8Array").New(args[0]) + uint8arrayWrapper := uint8Array.New(args[0]) value := make([]byte, uint8arrayWrapper.Get("byteLength").Int()) - a := js.TypedArrayOf(value) - a.Call("set", uint8arrayWrapper) - a.Release() + js.CopyBytesToGo(value, uint8arrayWrapper) bCh <- value return nil }) diff --git a/libgo/go/net/http/serve_test.go b/libgo/go/net/http/serve_test.go index 6eb0088a963..e7ed15c3aa4 100644 --- a/libgo/go/net/http/serve_test.go +++ b/libgo/go/net/http/serve_test.go @@ -4273,7 +4273,7 @@ func testServerEmptyBodyRace(t *testing.T, h2 bool) { var n int32 cst := newClientServerTest(t, h2, HandlerFunc(func(rw ResponseWriter, req *Request) { atomic.AddInt32(&n, 1) - })) + }), optQuietLog) defer cst.close() var wg sync.WaitGroup const reqs = 20 @@ -4697,6 +4697,10 @@ func TestServerHandlersCanHandleH2PRI(t *testing.T) { defer afterTest(t) ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { conn, br, err := w.(Hijacker).Hijack() + if err != nil { + t.Error(err) + return + } defer conn.Close() if r.Method != "PRI" || r.RequestURI != "*" { t.Errorf("Got method/target %q %q; want PRI *", r.Method, r.RequestURI) @@ -5745,8 +5749,12 @@ func TestServerDuplicateBackgroundRead(t *testing.T) { setParallel(t) defer afterTest(t) - const goroutines = 5 - const requests = 2000 + goroutines := 5 + requests := 2000 + if testing.Short() { + goroutines = 3 + requests = 100 + } hts := httptest.NewServer(HandlerFunc(NotFound)) defer hts.Close() @@ -6021,6 +6029,143 @@ func TestStripPortFromHost(t *testing.T) { } } +func TestServerContexts(t *testing.T) { + setParallel(t) + defer afterTest(t) + type baseKey struct{} + type connKey struct{} + ch := make(chan context.Context, 1) + ts := httptest.NewUnstartedServer(HandlerFunc(func(rw ResponseWriter, r *Request) { + ch <- r.Context() + })) + ts.Config.BaseContext = func(ln net.Listener) context.Context { + if strings.Contains(reflect.TypeOf(ln).String(), "onceClose") { + t.Errorf("unexpected onceClose listener type %T", ln) + } + return context.WithValue(context.Background(), baseKey{}, "base") + } + ts.Config.ConnContext = func(ctx context.Context, c net.Conn) context.Context { + if got, want := ctx.Value(baseKey{}), "base"; got != want { + t.Errorf("in ConnContext, base context key = %#v; want %q", got, want) + } + return context.WithValue(ctx, connKey{}, "conn") + } + ts.Start() + defer ts.Close() + res, err := ts.Client().Get(ts.URL) + if err != nil { + t.Fatal(err) + } + res.Body.Close() + ctx := <-ch + if got, want := ctx.Value(baseKey{}), "base"; got != want { + t.Errorf("base context key = %#v; want %q", got, want) + } + if got, want := ctx.Value(connKey{}), "conn"; got != want { + t.Errorf("conn context key = %#v; want %q", got, want) + } +} + +func TestServerContextsHTTP2(t *testing.T) { + setParallel(t) + defer afterTest(t) + type baseKey struct{} + type connKey struct{} + ch := make(chan context.Context, 1) + ts := httptest.NewUnstartedServer(HandlerFunc(func(rw ResponseWriter, r *Request) { + if r.ProtoMajor != 2 { + t.Errorf("unexpected HTTP/1.x request") + } + ch <- r.Context() + })) + ts.Config.BaseContext = func(ln net.Listener) context.Context { + if strings.Contains(reflect.TypeOf(ln).String(), "onceClose") { + t.Errorf("unexpected onceClose listener type %T", ln) + } + return context.WithValue(context.Background(), baseKey{}, "base") + } + ts.Config.ConnContext = func(ctx context.Context, c net.Conn) context.Context { + if got, want := ctx.Value(baseKey{}), "base"; got != want { + t.Errorf("in ConnContext, base context key = %#v; want %q", got, want) + } + return context.WithValue(ctx, connKey{}, "conn") + } + ts.TLS = &tls.Config{ + NextProtos: []string{"h2", "http/1.1"}, + } + ts.StartTLS() + defer ts.Close() + ts.Client().Transport.(*Transport).ForceAttemptHTTP2 = true + res, err := ts.Client().Get(ts.URL) + if err != nil { + t.Fatal(err) + } + res.Body.Close() + ctx := <-ch + if got, want := ctx.Value(baseKey{}), "base"; got != want { + t.Errorf("base context key = %#v; want %q", got, want) + } + if got, want := ctx.Value(connKey{}), "conn"; got != want { + t.Errorf("conn context key = %#v; want %q", got, want) + } +} + +// Issue 30710: ensure that as per the spec, a server responds +// with 501 Not Implemented for unsupported transfer-encodings. +func TestUnsupportedTransferEncodingsReturn501(t *testing.T) { + cst := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + w.Write([]byte("Hello, World!")) + })) + defer cst.Close() + + serverURL, err := url.Parse(cst.URL) + if err != nil { + t.Fatalf("Failed to parse server URL: %v", err) + } + + unsupportedTEs := []string{ + "fugazi", + "foo-bar", + "unknown", + } + + for _, badTE := range unsupportedTEs { + http1ReqBody := fmt.Sprintf(""+ + "POST / HTTP/1.1\r\nConnection: close\r\n"+ + "Host: localhost\r\nTransfer-Encoding: %s\r\n\r\n", badTE) + + gotBody, err := fetchWireResponse(serverURL.Host, []byte(http1ReqBody)) + if err != nil { + t.Errorf("%q. unexpected error: %v", badTE, err) + continue + } + + wantBody := fmt.Sprintf("" + + "HTTP/1.1 501 Not Implemented\r\nContent-Type: text/plain; charset=utf-8\r\n" + + "Connection: close\r\n\r\nUnsupported transfer encoding") + + if string(gotBody) != wantBody { + t.Errorf("%q. body\ngot\n%q\nwant\n%q", badTE, gotBody, wantBody) + } + } +} + +// fetchWireResponse is a helper for dialing to host, +// sending http1ReqBody as the payload and retrieving +// the response as it was sent on the wire. +func fetchWireResponse(host string, http1ReqBody []byte) ([]byte, error) { + conn, err := net.Dial("tcp", host) + if err != nil { + return nil, err + } + defer conn.Close() + + if _, err := conn.Write(http1ReqBody); err != nil { + return nil, err + } + return ioutil.ReadAll(conn) +} + func BenchmarkResponseStatusLine(b *testing.B) { b.ReportAllocs() b.RunParallel(func(pb *testing.PB) { diff --git a/libgo/go/net/http/server.go b/libgo/go/net/http/server.go index cbf5673ee6b..74569bf7b07 100644 --- a/libgo/go/net/http/server.go +++ b/libgo/go/net/http/server.go @@ -29,7 +29,7 @@ import ( "sync/atomic" "time" - "internal/x/net/http/httpguts" + "golang.org/x/net/http/httpguts" ) // Errors used by the HTTP server. @@ -749,10 +749,8 @@ func (cr *connReader) handleReadError(_ error) { // may be called from multiple goroutines. func (cr *connReader) closeNotify() { res, _ := cr.conn.curReq.Load().(*response) - if res != nil { - if atomic.CompareAndSwapInt32(&res.didCloseNotify, 0, 1) { - res.closeNotifyCh <- true - } + if res != nil && atomic.CompareAndSwapInt32(&res.didCloseNotify, 0, 1) { + res.closeNotifyCh <- true } } @@ -1060,7 +1058,7 @@ func (w *response) Header() Header { // Accessing the header between logically writing it // and physically writing it means we need to allocate // a clone to snapshot the logically written state. - w.cw.header = w.handlerHeader.clone() + w.cw.header = w.handlerHeader.Clone() } w.calledHeader = true return w.handlerHeader @@ -1134,7 +1132,7 @@ func (w *response) WriteHeader(code int) { w.status = code if w.calledHeader && w.cw.header == nil { - w.cw.header = w.handlerHeader.clone() + w.cw.header = w.handlerHeader.Clone() } if cl := w.handlerHeader.get("Content-Length"); cl != "" { @@ -1803,7 +1801,7 @@ func (c *conn) serve(ctx context.Context) { *c.tlsState = tlsConn.ConnectionState() if proto := c.tlsState.NegotiatedProtocol; validNPN(proto) { if fn := c.server.TLSNextProto[proto]; fn != nil { - h := initNPNRequest{tlsConn, serverHandler{c.server}} + h := initNPNRequest{ctx, tlsConn, serverHandler{c.server}} fn(c.server, tlsConn, h) } return @@ -1829,7 +1827,8 @@ func (c *conn) serve(ctx context.Context) { if err != nil { const errorHeaders = "\r\nContent-Type: text/plain; charset=utf-8\r\nConnection: close\r\n\r\n" - if err == errTooLarge { + switch { + case err == errTooLarge: // Their HTTP client may or may not be // able to read this if we're // responding to them and hanging up @@ -1839,18 +1838,31 @@ func (c *conn) serve(ctx context.Context) { fmt.Fprintf(c.rwc, "HTTP/1.1 "+publicErr+errorHeaders+publicErr) c.closeWriteAndWait() return - } - if isCommonNetReadError(err) { + + case isUnsupportedTEError(err): + // Respond as per RFC 7230 Section 3.3.1 which says, + // A server that receives a request message with a + // transfer coding it does not understand SHOULD + // respond with 501 (Unimplemented). + code := StatusNotImplemented + + // We purposefully aren't echoing back the transfer-encoding's value, + // so as to mitigate the risk of cross side scripting by an attacker. + fmt.Fprintf(c.rwc, "HTTP/1.1 %d %s%sUnsupported transfer encoding", code, StatusText(code), errorHeaders) + return + + case isCommonNetReadError(err): return // don't reply - } - publicErr := "400 Bad Request" - if v, ok := err.(badRequestError); ok { - publicErr = publicErr + ": " + string(v) - } + default: + publicErr := "400 Bad Request" + if v, ok := err.(badRequestError); ok { + publicErr = publicErr + ": " + string(v) + } - fmt.Fprintf(c.rwc, "HTTP/1.1 "+publicErr+errorHeaders+publicErr) - return + fmt.Fprintf(c.rwc, "HTTP/1.1 "+publicErr+errorHeaders+publicErr) + return + } } // Expect 100 Continue support @@ -2505,7 +2517,9 @@ type Server struct { // ReadHeaderTimeout is the amount of time allowed to read // request headers. The connection's read deadline is reset // after reading the headers and the Handler can decide what - // is considered too slow for the body. + // is considered too slow for the body. If ReadHeaderTimeout + // is zero, the value of ReadTimeout is used. If both are + // zero, there is no timeout. ReadHeaderTimeout time.Duration // WriteTimeout is the maximum duration before timing out @@ -2517,7 +2531,7 @@ type Server struct { // IdleTimeout is the maximum amount of time to wait for the // next request when keep-alives are enabled. If IdleTimeout // is zero, the value of ReadTimeout is used. If both are - // zero, ReadHeaderTimeout is used. + // zero, there is no timeout. IdleTimeout time.Duration // MaxHeaderBytes controls the maximum number of bytes the @@ -2549,6 +2563,20 @@ type Server struct { // If nil, logging is done via the log package's standard logger. ErrorLog *log.Logger + // BaseContext optionally specifies a function that returns + // the base context for incoming requests on this server. + // The provided Listener is the specific Listener that's + // about to start accepting requests. + // If BaseContext is nil, the default is context.Background(). + // If non-nil, it must return a non-nil context. + BaseContext func(net.Listener) context.Context + + // ConnContext optionally specifies a function that modifies + // the context used for a new connection c. The provided ctx + // is derived from the base context and has a ServerContextKey + // value. + ConnContext func(ctx context.Context, c net.Conn) context.Context + disableKeepAlives int32 // accessed atomically. inShutdown int32 // accessed atomically (non-zero means we're in Shutdown) nextProtoOnce sync.Once // guards setupHTTP2_* init @@ -2799,7 +2827,7 @@ func (srv *Server) ListenAndServe() error { if err != nil { return err } - return srv.Serve(tcpKeepAliveListener{ln.(*net.TCPListener)}) + return srv.Serve(ln) } var testHookServerServe func(*Server, net.Listener) // used if non-nil @@ -2845,6 +2873,7 @@ func (srv *Server) Serve(l net.Listener) error { fn(srv, l) // call hook with unwrapped listener } + origListener := l l = &onceCloseListener{Listener: l} defer l.Close() @@ -2857,8 +2886,16 @@ func (srv *Server) Serve(l net.Listener) error { } defer srv.trackListener(&l, false) - var tempDelay time.Duration // how long to sleep on accept failure - baseCtx := context.Background() // base is always background, per Issue 16220 + var tempDelay time.Duration // how long to sleep on accept failure + + baseCtx := context.Background() + if srv.BaseContext != nil { + baseCtx = srv.BaseContext(origListener) + if baseCtx == nil { + panic("BaseContext returned a nil context") + } + } + ctx := context.WithValue(baseCtx, ServerContextKey, srv) for { rw, e := l.Accept() @@ -2883,6 +2920,12 @@ func (srv *Server) Serve(l net.Listener) error { } return e } + if cc := srv.ConnContext; cc != nil { + ctx = cc(ctx, rw) + if ctx == nil { + panic("ConnContext returned nil") + } + } tempDelay = 0 c := srv.newConn(rw) c.setState(c.rwc, StateNew) // before Serve can return @@ -3083,7 +3126,7 @@ func (srv *Server) ListenAndServeTLS(certFile, keyFile string) error { defer ln.Close() - return srv.ServeTLS(tcpKeepAliveListener{ln.(*net.TCPListener)}, certFile, keyFile) + return srv.ServeTLS(ln, certFile, keyFile) } // setupHTTP2_ServeTLS conditionally configures HTTP/2 on @@ -3228,6 +3271,25 @@ type timeoutWriter struct { code int } +var _ Pusher = (*timeoutWriter)(nil) +var _ Flusher = (*timeoutWriter)(nil) + +// Push implements the Pusher interface. +func (tw *timeoutWriter) Push(target string, opts *PushOptions) error { + if pusher, ok := tw.w.(Pusher); ok { + return pusher.Push(target, opts) + } + return ErrNotSupported +} + +// Flush implements the Flusher interface. +func (tw *timeoutWriter) Flush() { + f, ok := tw.w.(Flusher) + if ok { + f.Flush() + } +} + func (tw *timeoutWriter) Header() Header { return tw.h } func (tw *timeoutWriter) Write(p []byte) (int, error) { @@ -3257,24 +3319,6 @@ func (tw *timeoutWriter) writeHeader(code int) { tw.code = code } -// tcpKeepAliveListener sets TCP keep-alive timeouts on accepted -// connections. It's used by ListenAndServe and ListenAndServeTLS so -// dead TCP connections (e.g. closing laptop mid-download) eventually -// go away. -type tcpKeepAliveListener struct { - *net.TCPListener -} - -func (ln tcpKeepAliveListener) Accept() (net.Conn, error) { - tc, err := ln.AcceptTCP() - if err != nil { - return nil, err - } - tc.SetKeepAlive(true) - tc.SetKeepAlivePeriod(3 * time.Minute) - return tc, nil -} - // onceCloseListener wraps a net.Listener, protecting it from // multiple Close calls. type onceCloseListener struct { @@ -3310,10 +3354,17 @@ func (globalOptionsHandler) ServeHTTP(w ResponseWriter, r *Request) { // uninitialized fields in its *Request. Such partially-initialized // Requests come from NPN protocol handlers. type initNPNRequest struct { - c *tls.Conn - h serverHandler + ctx context.Context + c *tls.Conn + h serverHandler } +// BaseContext is an exported but unadvertised http.Handler method +// recognized by x/net/http2 to pass down a context; the TLSNextProto +// API predates context support so we shoehorn through the only +// interface we have available. +func (h initNPNRequest) BaseContext() context.Context { return h.ctx } + func (h initNPNRequest) ServeHTTP(rw ResponseWriter, req *Request) { if req.TLS == nil { req.TLS = &tls.ConnectionState{} diff --git a/libgo/go/net/http/sniff.go b/libgo/go/net/http/sniff.go index c1494abb4c8..67a7151b0cc 100644 --- a/libgo/go/net/http/sniff.go +++ b/libgo/go/net/http/sniff.go @@ -37,6 +37,8 @@ func DetectContentType(data []byte) string { return "application/octet-stream" // fallback } +// isWS reports whether the provided byte is a whitespace byte (0xWS) +// as defined in https://mimesniff.spec.whatwg.org/#terminology. func isWS(b byte) bool { switch b { case '\t', '\n', '\x0c', '\r', ' ': @@ -45,6 +47,16 @@ func isWS(b byte) bool { return false } +// isTT reports whether the provided byte is a tag-terminating byte (0xTT) +// as defined in https://mimesniff.spec.whatwg.org/#terminology. +func isTT(b byte) bool { + switch b { + case ' ', '>': + return true + } + return false +} + type sniffSig interface { // match returns the MIME type of the data, or "" if unknown. match(data []byte, firstNonWS int) string @@ -69,33 +81,57 @@ var sniffSignatures = []sniffSig{ htmlSig("<BR"), htmlSig("<P"), htmlSig("<!--"), - - &maskedSig{mask: []byte("\xFF\xFF\xFF\xFF\xFF"), pat: []byte("<?xml"), skipWS: true, ct: "text/xml; charset=utf-8"}, - + &maskedSig{ + mask: []byte("\xFF\xFF\xFF\xFF\xFF"), + pat: []byte("<?xml"), + skipWS: true, + ct: "text/xml; charset=utf-8"}, &exactSig{[]byte("%PDF-"), "application/pdf"}, &exactSig{[]byte("%!PS-Adobe-"), "application/postscript"}, // UTF BOMs. - &maskedSig{mask: []byte("\xFF\xFF\x00\x00"), pat: []byte("\xFE\xFF\x00\x00"), ct: "text/plain; charset=utf-16be"}, - &maskedSig{mask: []byte("\xFF\xFF\x00\x00"), pat: []byte("\xFF\xFE\x00\x00"), ct: "text/plain; charset=utf-16le"}, - &maskedSig{mask: []byte("\xFF\xFF\xFF\x00"), pat: []byte("\xEF\xBB\xBF\x00"), ct: "text/plain; charset=utf-8"}, + &maskedSig{ + mask: []byte("\xFF\xFF\x00\x00"), + pat: []byte("\xFE\xFF\x00\x00"), + ct: "text/plain; charset=utf-16be", + }, + &maskedSig{ + mask: []byte("\xFF\xFF\x00\x00"), + pat: []byte("\xFF\xFE\x00\x00"), + ct: "text/plain; charset=utf-16le", + }, + &maskedSig{ + mask: []byte("\xFF\xFF\xFF\x00"), + pat: []byte("\xEF\xBB\xBF\x00"), + ct: "text/plain; charset=utf-8", + }, + // Image types + // For posterity, we originally returned "image/vnd.microsoft.icon" from + // https://tools.ietf.org/html/draft-ietf-websec-mime-sniff-03#section-7 + // https://codereview.appspot.com/4746042 + // but that has since been replaced with "image/x-icon" in Section 6.2 + // of https://mimesniff.spec.whatwg.org/#matching-an-image-type-pattern + &exactSig{[]byte("\x00\x00\x01\x00"), "image/x-icon"}, + &exactSig{[]byte("\x00\x00\x02\x00"), "image/x-icon"}, + &exactSig{[]byte("BM"), "image/bmp"}, &exactSig{[]byte("GIF87a"), "image/gif"}, &exactSig{[]byte("GIF89a"), "image/gif"}, - &exactSig{[]byte("\x89\x50\x4E\x47\x0D\x0A\x1A\x0A"), "image/png"}, - &exactSig{[]byte("\xFF\xD8\xFF"), "image/jpeg"}, - &exactSig{[]byte("BM"), "image/bmp"}, &maskedSig{ mask: []byte("\xFF\xFF\xFF\xFF\x00\x00\x00\x00\xFF\xFF\xFF\xFF\xFF\xFF"), pat: []byte("RIFF\x00\x00\x00\x00WEBPVP"), ct: "image/webp", }, - &exactSig{[]byte("\x00\x00\x01\x00"), "image/vnd.microsoft.icon"}, + &exactSig{[]byte("\x89PNG\x0D\x0A\x1A\x0A"), "image/png"}, + &exactSig{[]byte("\xFF\xD8\xFF"), "image/jpeg"}, + // Audio and Video types + // Enforce the pattern match ordering as prescribed in + // https://mimesniff.spec.whatwg.org/#matching-an-audio-or-video-type-pattern &maskedSig{ - mask: []byte("\xFF\xFF\xFF\xFF\x00\x00\x00\x00\xFF\xFF\xFF\xFF"), - pat: []byte("RIFF\x00\x00\x00\x00WAVE"), - ct: "audio/wave", + mask: []byte("\xFF\xFF\xFF\xFF"), + pat: []byte(".snd"), + ct: "audio/basic", }, &maskedSig{ mask: []byte("\xFF\xFF\xFF\xFF\x00\x00\x00\x00\xFF\xFF\xFF\xFF"), @@ -103,9 +139,9 @@ var sniffSignatures = []sniffSig{ ct: "audio/aiff", }, &maskedSig{ - mask: []byte("\xFF\xFF\xFF\xFF"), - pat: []byte(".snd"), - ct: "audio/basic", + mask: []byte("\xFF\xFF\xFF"), + pat: []byte("ID3"), + ct: "audio/mpeg", }, &maskedSig{ mask: []byte("\xFF\xFF\xFF\xFF\xFF"), @@ -118,20 +154,24 @@ var sniffSignatures = []sniffSig{ ct: "audio/midi", }, &maskedSig{ - mask: []byte("\xFF\xFF\xFF"), - pat: []byte("ID3"), - ct: "audio/mpeg", - }, - &maskedSig{ mask: []byte("\xFF\xFF\xFF\xFF\x00\x00\x00\x00\xFF\xFF\xFF\xFF"), pat: []byte("RIFF\x00\x00\x00\x00AVI "), ct: "video/avi", }, + &maskedSig{ + mask: []byte("\xFF\xFF\xFF\xFF\x00\x00\x00\x00\xFF\xFF\xFF\xFF"), + pat: []byte("RIFF\x00\x00\x00\x00WAVE"), + ct: "audio/wave", + }, + // 6.2.0.2. video/mp4 + mp4Sig{}, + // 6.2.0.3. video/webm + &exactSig{[]byte("\x1A\x45\xDF\xA3"), "video/webm"}, - // Fonts + // Font types &maskedSig{ // 34 NULL bytes followed by the string "LP" - pat: []byte("\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x4C\x50"), + pat: []byte("\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00LP"), // 34 NULL bytes followed by \xF\xF mask: []byte("\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xFF\xFF"), ct: "application/vnd.ms-fontobject", @@ -142,15 +182,20 @@ var sniffSignatures = []sniffSig{ &exactSig{[]byte("wOFF"), "font/woff"}, &exactSig{[]byte("wOF2"), "font/woff2"}, - &exactSig{[]byte("\x1A\x45\xDF\xA3"), "video/webm"}, - &exactSig{[]byte("\x52\x61\x72\x20\x1A\x07\x00"), "application/x-rar-compressed"}, - &exactSig{[]byte("\x50\x4B\x03\x04"), "application/zip"}, + // Archive types &exactSig{[]byte("\x1F\x8B\x08"), "application/x-gzip"}, + &exactSig{[]byte("PK\x03\x04"), "application/zip"}, + // RAR's signatures are incorrectly defined by the MIME spec as per + // https://github.com/whatwg/mimesniff/issues/63 + // However, RAR Labs correctly defines it at: + // https://www.rarlab.com/technote.htm#rarsign + // so we use the definition from RAR Labs. + // TODO: do whatever the spec ends up doing. + &exactSig{[]byte("Rar!\x1A\x07\x00"), "application/x-rar-compressed"}, // RAR v1.5-v4.0 + &exactSig{[]byte("Rar!\x1A\x07\x01\x00"), "application/x-rar-compressed"}, // RAR v5+ &exactSig{[]byte("\x00\x61\x73\x6D"), "application/wasm"}, - mp4Sig{}, - textSig{}, // should be last } @@ -182,12 +227,12 @@ func (m *maskedSig) match(data []byte, firstNonWS int) string { if len(m.pat) != len(m.mask) { return "" } - if len(data) < len(m.mask) { + if len(data) < len(m.pat) { return "" } - for i, mask := range m.mask { - db := data[i] & mask - if db != m.pat[i] { + for i, pb := range m.pat { + maskedData := data[i] & m.mask[i] + if maskedData != pb { return "" } } @@ -210,8 +255,8 @@ func (h htmlSig) match(data []byte, firstNonWS int) string { return "" } } - // Next byte must be space or right angle bracket. - if db := data[len(h)]; db != ' ' && db != '>' { + // Next byte must be a tag-terminating byte(0xTT). + if !isTT(data[len(h)]) { return "" } return "text/html; charset=utf-8" @@ -229,7 +274,7 @@ func (mp4Sig) match(data []byte, firstNonWS int) string { return "" } boxSize := int(binary.BigEndian.Uint32(data[:4])) - if boxSize%4 != 0 || len(data) < boxSize { + if len(data) < boxSize || boxSize%4 != 0 { return "" } if !bytes.Equal(data[4:8], mp4ftype) { @@ -237,7 +282,7 @@ func (mp4Sig) match(data []byte, firstNonWS int) string { } for st := 8; st < boxSize; st += 4 { if st == 12 { - // minor version number + // Ignores the four bytes that correspond to the version number of the "major brand". continue } if bytes.Equal(data[st:st+3], mp4) { diff --git a/libgo/go/net/http/sniff_test.go b/libgo/go/net/http/sniff_test.go index b4d3c9f0cc8..a1157a0823e 100644 --- a/libgo/go/net/http/sniff_test.go +++ b/libgo/go/net/http/sniff_test.go @@ -36,8 +36,14 @@ var sniffTests = []struct { {"XML", []byte("\n<?xml!"), "text/xml; charset=utf-8"}, // Image types. + {"Windows icon", []byte("\x00\x00\x01\x00"), "image/x-icon"}, + {"Windows cursor", []byte("\x00\x00\x02\x00"), "image/x-icon"}, + {"BMP image", []byte("BM..."), "image/bmp"}, {"GIF 87a", []byte(`GIF87a`), "image/gif"}, {"GIF 89a", []byte(`GIF89a...`), "image/gif"}, + {"WEBP image", []byte("RIFF\x00\x00\x00\x00WEBPVP"), "image/webp"}, + {"PNG image", []byte("\x89PNG\x0D\x0A\x1A\x0A"), "image/png"}, + {"JPEG image", []byte("\xFF\xD8\xFF"), "image/jpeg"}, // Audio types. {"MIDI audio", []byte("MThd\x00\x00\x00\x06\x00\x01"), "audio/midi"}, @@ -66,6 +72,12 @@ var sniffTests = []struct { {"woff sample I", []byte("\x77\x4f\x46\x46\x00\x01\x00\x00\x00\x00\x30\x54\x00\x0d\x00\x00"), "font/woff"}, {"woff2 sample", []byte("\x77\x4f\x46\x32\x00\x01\x00\x00\x00"), "font/woff2"}, {"wasm sample", []byte("\x00\x61\x73\x6d\x01\x00"), "application/wasm"}, + + // Archive types + {"RAR v1.5-v4.0", []byte("Rar!\x1A\x07\x00"), "application/x-rar-compressed"}, + {"RAR v5+", []byte("Rar!\x1A\x07\x01\x00"), "application/x-rar-compressed"}, + {"Incorrect RAR v1.5-v4.0", []byte("Rar \x1A\x07\x00"), "application/octet-stream"}, + {"Incorrect RAR v5+", []byte("Rar \x1A\x07\x01\x00"), "application/octet-stream"}, } func TestDetectContentType(t *testing.T) { diff --git a/libgo/go/net/http/status.go b/libgo/go/net/http/status.go index 086f3d1a71e..286315f6395 100644 --- a/libgo/go/net/http/status.go +++ b/libgo/go/net/http/status.go @@ -10,6 +10,7 @@ const ( StatusContinue = 100 // RFC 7231, 6.2.1 StatusSwitchingProtocols = 101 // RFC 7231, 6.2.2 StatusProcessing = 102 // RFC 2518, 10.1 + StatusEarlyHints = 103 // RFC 8297 StatusOK = 200 // RFC 7231, 6.3.1 StatusCreated = 201 // RFC 7231, 6.3.2 @@ -79,6 +80,7 @@ var statusText = map[int]string{ StatusContinue: "Continue", StatusSwitchingProtocols: "Switching Protocols", StatusProcessing: "Processing", + StatusEarlyHints: "Early Hints", StatusOK: "OK", StatusCreated: "Created", diff --git a/libgo/go/net/http/transfer.go b/libgo/go/net/http/transfer.go index e8a93e9137e..2e01a07f84f 100644 --- a/libgo/go/net/http/transfer.go +++ b/libgo/go/net/http/transfer.go @@ -21,7 +21,7 @@ import ( "sync" "time" - "internal/x/net/http/httpguts" + "golang.org/x/net/http/httpguts" ) // ErrLineTooLong is returned when reading request or response bodies @@ -53,19 +53,6 @@ func (br *byteReader) Read(p []byte) (n int, err error) { return 1, io.EOF } -// transferBodyReader is an io.Reader that reads from tw.Body -// and records any non-EOF error in tw.bodyReadError. -// It is exactly 1 pointer wide to avoid allocations into interfaces. -type transferBodyReader struct{ tw *transferWriter } - -func (br transferBodyReader) Read(p []byte) (n int, err error) { - n, err = br.tw.Body.Read(p) - if err != nil && err != io.EOF { - br.tw.bodyReadError = err - } - return -} - // transferWriter inspects the fields of a user-supplied Request or Response, // sanitizes them without changing the user object and provides methods for // writing the respective header, body and trailer in wire format. @@ -347,15 +334,18 @@ func (t *transferWriter) writeBody(w io.Writer) error { var err error var ncopy int64 - // Write body + // Write body. We "unwrap" the body first if it was wrapped in a + // nopCloser. This is to ensure that we can take advantage of + // OS-level optimizations in the event that the body is an + // *os.File. if t.Body != nil { - var body = transferBodyReader{t} + var body = t.unwrapBody() if chunked(t.TransferEncoding) { if bw, ok := w.(*bufio.Writer); ok && !t.IsResponse { w = &internal.FlushAfterChunkWriter{Writer: bw} } cw := internal.NewChunkedWriter(w) - _, err = io.Copy(cw, body) + _, err = t.doBodyCopy(cw, body) if err == nil { err = cw.Close() } @@ -364,14 +354,14 @@ func (t *transferWriter) writeBody(w io.Writer) error { if t.Method == "CONNECT" { dst = bufioFlushWriter{dst} } - ncopy, err = io.Copy(dst, body) + ncopy, err = t.doBodyCopy(dst, body) } else { - ncopy, err = io.Copy(w, io.LimitReader(body, t.ContentLength)) + ncopy, err = t.doBodyCopy(w, io.LimitReader(body, t.ContentLength)) if err != nil { return err } var nextra int64 - nextra, err = io.Copy(ioutil.Discard, body) + nextra, err = t.doBodyCopy(ioutil.Discard, body) ncopy += nextra } if err != nil { @@ -402,6 +392,31 @@ func (t *transferWriter) writeBody(w io.Writer) error { return err } +// doBodyCopy wraps a copy operation, with any resulting error also +// being saved in bodyReadError. +// +// This function is only intended for use in writeBody. +func (t *transferWriter) doBodyCopy(dst io.Writer, src io.Reader) (n int64, err error) { + n, err = io.Copy(dst, src) + if err != nil && err != io.EOF { + t.bodyReadError = err + } + return +} + +// unwrapBodyReader unwraps the body's inner reader if it's a +// nopCloser. This is to ensure that body writes sourced from local +// files (*os.File types) are properly optimized. +// +// This function is only intended for use in writeBody. +func (t *transferWriter) unwrapBody() io.Reader { + if reflect.TypeOf(t.Body) == nopCloserType { + return reflect.ValueOf(t.Body).Field(0).Interface().(io.Reader) + } + + return t.Body +} + type transferReader struct { // Input Header Header @@ -574,6 +589,22 @@ func chunked(te []string) bool { return len(te) > 0 && te[0] == "chunked" } // Checks whether the encoding is explicitly "identity". func isIdentity(te []string) bool { return len(te) == 1 && te[0] == "identity" } +// unsupportedTEError reports unsupported transfer-encodings. +type unsupportedTEError struct { + err string +} + +func (uste *unsupportedTEError) Error() string { + return uste.err +} + +// isUnsupportedTEError checks if the error is of type +// unsupportedTEError. It is usually invoked with a non-nil err. +func isUnsupportedTEError(err error) bool { + _, ok := err.(*unsupportedTEError) + return ok +} + // fixTransferEncoding sanitizes t.TransferEncoding, if needed. func (t *transferReader) fixTransferEncoding() error { raw, present := t.Header["Transfer-Encoding"] @@ -600,7 +631,7 @@ func (t *transferReader) fixTransferEncoding() error { break } if encoding != "chunked" { - return &badStringError{"unsupported transfer encoding", encoding} + return &unsupportedTEError{fmt.Sprintf("unsupported transfer encoding: %q", encoding)} } te = te[0 : len(te)+1] te[len(te)-1] = encoding diff --git a/libgo/go/net/http/transfer_test.go b/libgo/go/net/http/transfer_test.go index 993ea4ef18c..65009ee8bf7 100644 --- a/libgo/go/net/http/transfer_test.go +++ b/libgo/go/net/http/transfer_test.go @@ -7,8 +7,12 @@ package http import ( "bufio" "bytes" + "crypto/rand" + "fmt" "io" "io/ioutil" + "os" + "reflect" "strings" "testing" ) @@ -90,3 +94,219 @@ func TestDetectInMemoryReaders(t *testing.T) { } } } + +type mockTransferWriter struct { + CalledReader io.Reader + WriteCalled bool +} + +var _ io.ReaderFrom = (*mockTransferWriter)(nil) + +func (w *mockTransferWriter) ReadFrom(r io.Reader) (int64, error) { + w.CalledReader = r + return io.Copy(ioutil.Discard, r) +} + +func (w *mockTransferWriter) Write(p []byte) (int, error) { + w.WriteCalled = true + return ioutil.Discard.Write(p) +} + +func TestTransferWriterWriteBodyReaderTypes(t *testing.T) { + fileType := reflect.TypeOf(&os.File{}) + bufferType := reflect.TypeOf(&bytes.Buffer{}) + + nBytes := int64(1 << 10) + newFileFunc := func() (r io.Reader, done func(), err error) { + f, err := ioutil.TempFile("", "net-http-newfilefunc") + if err != nil { + return nil, nil, err + } + + // Write some bytes to the file to enable reading. + if _, err := io.CopyN(f, rand.Reader, nBytes); err != nil { + return nil, nil, fmt.Errorf("failed to write data to file: %v", err) + } + if _, err := f.Seek(0, 0); err != nil { + return nil, nil, fmt.Errorf("failed to seek to front: %v", err) + } + + done = func() { + f.Close() + os.Remove(f.Name()) + } + + return f, done, nil + } + + newBufferFunc := func() (io.Reader, func(), error) { + return bytes.NewBuffer(make([]byte, nBytes)), func() {}, nil + } + + cases := []struct { + name string + bodyFunc func() (io.Reader, func(), error) + method string + contentLength int64 + transferEncoding []string + limitedReader bool + expectedReader reflect.Type + expectedWrite bool + }{ + { + name: "file, non-chunked, size set", + bodyFunc: newFileFunc, + method: "PUT", + contentLength: nBytes, + limitedReader: true, + expectedReader: fileType, + }, + { + name: "file, non-chunked, size set, nopCloser wrapped", + method: "PUT", + bodyFunc: func() (io.Reader, func(), error) { + r, cleanup, err := newFileFunc() + return ioutil.NopCloser(r), cleanup, err + }, + contentLength: nBytes, + limitedReader: true, + expectedReader: fileType, + }, + { + name: "file, non-chunked, negative size", + method: "PUT", + bodyFunc: newFileFunc, + contentLength: -1, + expectedReader: fileType, + }, + { + name: "file, non-chunked, CONNECT, negative size", + method: "CONNECT", + bodyFunc: newFileFunc, + contentLength: -1, + expectedReader: fileType, + }, + { + name: "file, chunked", + method: "PUT", + bodyFunc: newFileFunc, + transferEncoding: []string{"chunked"}, + expectedWrite: true, + }, + { + name: "buffer, non-chunked, size set", + bodyFunc: newBufferFunc, + method: "PUT", + contentLength: nBytes, + limitedReader: true, + expectedReader: bufferType, + }, + { + name: "buffer, non-chunked, size set, nopCloser wrapped", + method: "PUT", + bodyFunc: func() (io.Reader, func(), error) { + r, cleanup, err := newBufferFunc() + return ioutil.NopCloser(r), cleanup, err + }, + contentLength: nBytes, + limitedReader: true, + expectedReader: bufferType, + }, + { + name: "buffer, non-chunked, negative size", + method: "PUT", + bodyFunc: newBufferFunc, + contentLength: -1, + expectedWrite: true, + }, + { + name: "buffer, non-chunked, CONNECT, negative size", + method: "CONNECT", + bodyFunc: newBufferFunc, + contentLength: -1, + expectedWrite: true, + }, + { + name: "buffer, chunked", + method: "PUT", + bodyFunc: newBufferFunc, + transferEncoding: []string{"chunked"}, + expectedWrite: true, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + body, cleanup, err := tc.bodyFunc() + if err != nil { + t.Fatal(err) + } + defer cleanup() + + mw := &mockTransferWriter{} + tw := &transferWriter{ + Body: body, + ContentLength: tc.contentLength, + TransferEncoding: tc.transferEncoding, + } + + if err := tw.writeBody(mw); err != nil { + t.Fatal(err) + } + + if tc.expectedReader != nil { + if mw.CalledReader == nil { + t.Fatal("did not call ReadFrom") + } + + var actualReader reflect.Type + lr, ok := mw.CalledReader.(*io.LimitedReader) + if ok && tc.limitedReader { + actualReader = reflect.TypeOf(lr.R) + } else { + actualReader = reflect.TypeOf(mw.CalledReader) + } + + if tc.expectedReader != actualReader { + t.Fatalf("got reader %T want %T", actualReader, tc.expectedReader) + } + } + + if tc.expectedWrite && !mw.WriteCalled { + t.Fatal("did not invoke Write") + } + }) + } +} + +func TestFixTransferEncoding(t *testing.T) { + tests := []struct { + hdr Header + wantErr error + }{ + { + hdr: Header{"Transfer-Encoding": {"fugazi"}}, + wantErr: &unsupportedTEError{`unsupported transfer encoding: "fugazi"`}, + }, + { + hdr: Header{"Transfer-Encoding": {"chunked, chunked", "identity", "chunked"}}, + wantErr: &badStringError{"too many transfer encodings", "chunked,chunked"}, + }, + { + hdr: Header{"Transfer-Encoding": {"chunked"}}, + wantErr: nil, + }, + } + + for i, tt := range tests { + tr := &transferReader{ + Header: tt.hdr, + ProtoMajor: 1, + ProtoMinor: 1, + } + gotErr := tr.fixTransferEncoding() + if !reflect.DeepEqual(gotErr, tt.wantErr) { + t.Errorf("%d.\ngot error:\n%v\nwant error:\n%v\n\n", i, gotErr, tt.wantErr) + } + } +} diff --git a/libgo/go/net/http/transport.go b/libgo/go/net/http/transport.go index a8c5efe6aaf..26f642aa7a0 100644 --- a/libgo/go/net/http/transport.go +++ b/libgo/go/net/http/transport.go @@ -30,8 +30,8 @@ import ( "sync/atomic" "time" - "internal/x/net/http/httpguts" - "internal/x/net/http/httpproxy" + "golang.org/x/net/http/httpguts" + "golang.org/x/net/http/httpproxy" ) // DefaultTransport is the default implementation of Transport and is @@ -46,6 +46,7 @@ var DefaultTransport RoundTripper = &Transport{ KeepAlive: 30 * time.Second, DualStack: true, }).DialContext, + ForceAttemptHTTP2: true, MaxIdleConns: 100, IdleConnTimeout: 90 * time.Second, TLSHandshakeTimeout: 10 * time.Second, @@ -253,10 +254,76 @@ type Transport struct { // Zero means to use a default limit. MaxResponseHeaderBytes int64 + // WriteBufferSize specifies the size of the write buffer used + // when writing to the transport. + // If zero, a default (currently 4KB) is used. + WriteBufferSize int + + // ReadBufferSize specifies the size of the read buffer used + // when reading from the transport. + // If zero, a default (currently 4KB) is used. + ReadBufferSize int + // nextProtoOnce guards initialization of TLSNextProto and // h2transport (via onceSetNextProtoDefaults) - nextProtoOnce sync.Once - h2transport h2Transport // non-nil if http2 wired up + nextProtoOnce sync.Once + h2transport h2Transport // non-nil if http2 wired up + tlsNextProtoWasNil bool // whether TLSNextProto was nil when the Once fired + + // ForceAttemptHTTP2 controls whether HTTP/2 is enabled when a non-zero + // Dial, DialTLS, or DialContext func or TLSClientConfig is provided. + // By default, use of any those fields conservatively disables HTTP/2. + // To use a custom dialer or TLS config and still attempt HTTP/2 + // upgrades, set this to true. + ForceAttemptHTTP2 bool +} + +func (t *Transport) writeBufferSize() int { + if t.WriteBufferSize > 0 { + return t.WriteBufferSize + } + return 4 << 10 +} + +func (t *Transport) readBufferSize() int { + if t.ReadBufferSize > 0 { + return t.ReadBufferSize + } + return 4 << 10 +} + +// Clone returns a deep copy of t's exported fields. +func (t *Transport) Clone() *Transport { + t.nextProtoOnce.Do(t.onceSetNextProtoDefaults) + t2 := &Transport{ + Proxy: t.Proxy, + DialContext: t.DialContext, + Dial: t.Dial, + DialTLS: t.DialTLS, + TLSClientConfig: t.TLSClientConfig.Clone(), + TLSHandshakeTimeout: t.TLSHandshakeTimeout, + DisableKeepAlives: t.DisableKeepAlives, + DisableCompression: t.DisableCompression, + MaxIdleConns: t.MaxIdleConns, + MaxIdleConnsPerHost: t.MaxIdleConnsPerHost, + MaxConnsPerHost: t.MaxConnsPerHost, + IdleConnTimeout: t.IdleConnTimeout, + ResponseHeaderTimeout: t.ResponseHeaderTimeout, + ExpectContinueTimeout: t.ExpectContinueTimeout, + ProxyConnectHeader: t.ProxyConnectHeader.Clone(), + MaxResponseHeaderBytes: t.MaxResponseHeaderBytes, + ForceAttemptHTTP2: t.ForceAttemptHTTP2, + WriteBufferSize: t.WriteBufferSize, + ReadBufferSize: t.ReadBufferSize, + } + if !t.tlsNextProtoWasNil { + npm := map[string]func(authority string, c *tls.Conn) RoundTripper{} + for k, v := range t.TLSNextProto { + npm[k] = v + } + t2.TLSNextProto = npm + } + return t2 } // h2Transport is the interface we expect to be able to call from @@ -272,6 +339,7 @@ type h2Transport interface { // onceSetNextProtoDefaults initializes TLSNextProto. // It must be called via t.nextProtoOnce.Do. func (t *Transport) onceSetNextProtoDefaults() { + t.tlsNextProtoWasNil = (t.TLSNextProto == nil) if strings.Contains(os.Getenv("GODEBUG"), "http2client=0") { return } @@ -296,12 +364,13 @@ func (t *Transport) onceSetNextProtoDefaults() { // Transport. return } - if t.TLSClientConfig != nil || t.Dial != nil || t.DialTLS != nil { + if !t.ForceAttemptHTTP2 && (t.TLSClientConfig != nil || t.Dial != nil || t.DialTLS != nil || t.DialContext != nil) { // Be conservative and don't automatically enable // http2 if they've specified a custom TLS config or // custom dialers. Let them opt-in themselves via // http2.ConfigureTransport so we don't surprise them // by modifying their tls.Config. Issue 14275. + // However, if ForceAttemptHTTP2 is true, it overrides the above checks. return } t2, err := http2configureTransport(t) @@ -474,8 +543,8 @@ func (t *Transport) roundTrip(req *Request) (*Response, error) { var resp *Response if pconn.alt != nil { // HTTP/2 path. - t.decHostConnCount(cm.key()) // don't count cached http2 conns toward conns per host - t.setReqCanceler(req, nil) // not cancelable with CancelRequest + t.putOrCloseIdleConn(pconn) + t.setReqCanceler(req, nil) // not cancelable with CancelRequest resp, err = pconn.alt.RoundTrip(req) } else { resp, err = pconn.roundTrip(treq) @@ -483,7 +552,10 @@ func (t *Transport) roundTrip(req *Request) (*Response, error) { if err == nil { return resp, nil } - if !pconn.shouldRetryRequest(req, err) { + if http2isNoCachedConnError(err) { + t.removeIdleConn(pconn) + t.decHostConnCount(cm.key()) // clean up the persistent connection + } else if !pconn.shouldRetryRequest(req, err) { // Issue 16465: return underlying net.Conn.Read error from peek, // as we've historically done. if e, ok := err.(transportReadFromServerError); ok { @@ -717,6 +789,8 @@ type transportReadFromServerError struct { err error } +func (e transportReadFromServerError) Unwrap() error { return e.err } + func (e transportReadFromServerError) Error() string { return fmt.Sprintf("net/http: Transport failed to read from server: %v", e.err) } @@ -746,9 +820,6 @@ func (t *Transport) tryPutIdleConn(pconn *persistConn) error { if pconn.isBroken() { return errConnBroken } - if pconn.alt != nil { - return errNotCachingH2Conn - } pconn.markReused() key := pconn.cacheKey @@ -797,7 +868,10 @@ func (t *Transport) tryPutIdleConn(pconn *persistConn) error { if pconn.idleTimer != nil { pconn.idleTimer.Reset(t.IdleConnTimeout) } else { - pconn.idleTimer = time.AfterFunc(t.IdleConnTimeout, pconn.closeConnIfStillIdle) + // idleTimer does not apply to HTTP/2 + if pconn.alt == nil { + pconn.idleTimer = time.AfterFunc(t.IdleConnTimeout, pconn.closeConnIfStillIdle) + } } } pconn.idleAt = time.Now() @@ -1031,7 +1105,7 @@ func (t *Transport) getConn(treq *transportRequest, cm connectMethod) (*persistC t.decHostConnCount(cmKey) select { case <-req.Cancel: - // It was an error due to cancelation, so prioritize that + // It was an error due to cancellation, so prioritize that // error value. (Issue 16049) return nil, errRequestCanceledConn case <-req.Context().Done(): @@ -1042,7 +1116,7 @@ func (t *Transport) getConn(treq *transportRequest, cm connectMethod) (*persistC } return nil, err default: - // It wasn't an error due to cancelation, so + // It wasn't an error due to cancellation, so // return the original error message: return nil, v.err } @@ -1345,15 +1419,16 @@ func (t *Transport) dialConn(ctx context.Context, cm connectMethod) (*persistCon if s := pconn.tlsState; s != nil && s.NegotiatedProtocolIsMutual && s.NegotiatedProtocol != "" { if next, ok := t.TLSNextProto[s.NegotiatedProtocol]; ok { - return &persistConn{alt: next(cm.targetAddr, pconn.conn.(*tls.Conn))}, nil + return &persistConn{t: t, cacheKey: pconn.cacheKey, alt: next(cm.targetAddr, pconn.conn.(*tls.Conn))}, nil } } if t.MaxConnsPerHost > 0 { pconn.conn = &connCloseListener{Conn: pconn.conn, t: t, cmKey: pconn.cacheKey} } - pconn.br = bufio.NewReader(pconn) - pconn.bw = bufio.NewWriter(persistConnWriter{pconn}) + pconn.br = bufio.NewReaderSize(pconn, t.readBufferSize()) + pconn.bw = bufio.NewWriterSize(persistConnWriter{pconn}, t.writeBufferSize()) + go pconn.readLoop() go pconn.writeLoop() return pconn, nil @@ -1375,6 +1450,17 @@ func (w persistConnWriter) Write(p []byte) (n int, err error) { return } +// ReadFrom exposes persistConnWriter's underlying Conn to io.Copy and if +// the Conn implements io.ReaderFrom, it can take advantage of optimizations +// such as sendfile. +func (w persistConnWriter) ReadFrom(r io.Reader) (n int64, err error) { + n, err = io.Copy(w.pc.conn, r) + w.pc.nwrite += n + return +} + +var _ io.ReaderFrom = (*persistConnWriter)(nil) + // connectMethod is the map key (in its String form) for keeping persistent // TCP connections alive for subsequent HTTP requests. // @@ -1538,7 +1624,7 @@ func (pc *persistConn) isBroken() bool { } // canceled returns non-nil if the connection was closed due to -// CancelRequest or due to context cancelation. +// CancelRequest or due to context cancellation. func (pc *persistConn) canceled() error { pc.mu.Lock() defer pc.mu.Unlock() @@ -1794,7 +1880,7 @@ func (pc *persistConn) readLoop() { // Before looping back to the top of this function and peeking on // the bufio.Reader, wait for the caller goroutine to finish - // reading the response body. (or for cancelation or death) + // reading the response body. (or for cancellation or death) select { case bodyEOF := <-waitForBodyRead: pc.t.setReqCanceler(rc.req, nil) // before pc might return to idle pool @@ -1826,7 +1912,12 @@ func (pc *persistConn) readLoopPeekFailLocked(peekErr error) { } if n := pc.br.Buffered(); n > 0 { buf, _ := pc.br.Peek(n) - log.Printf("Unsolicited response received on idle HTTP channel starting with %q; err=%v", buf, peekErr) + if is408Message(buf) { + pc.closeLocked(errServerClosedIdle) + return + } else { + log.Printf("Unsolicited response received on idle HTTP channel starting with %q; err=%v", buf, peekErr) + } } if peekErr == io.EOF { // common case. @@ -1836,6 +1927,19 @@ func (pc *persistConn) readLoopPeekFailLocked(peekErr error) { } } +// is408Message reports whether buf has the prefix of an +// HTTP 408 Request Timeout response. +// See golang.org/issue/32310. +func is408Message(buf []byte) bool { + if len(buf) < len("HTTP/1.x 408") { + return false + } + if string(buf[:7]) != "HTTP/1." { + return false + } + return string(buf[8:12]) == " 408" +} + // readResponse reads an HTTP response (or two, in the case of "Expect: // 100-continue") from the server. It returns the final non-100 one. // trace is optional. @@ -2072,8 +2176,21 @@ func (e *httpError) Error() string { return e.err } func (e *httpError) Timeout() bool { return e.timeout } func (e *httpError) Temporary() bool { return true } +func (e *httpError) Is(target error) bool { + switch target { + case os.ErrTimeout: + return e.timeout + case os.ErrTemporary: + return true + } + return false +} + var errTimeout error = &httpError{err: "net/http: timeout awaiting response headers", timeout: true} -var errRequestCanceled = errors.New("net/http: request canceled") + +// errRequestCanceled is set to be identical to the one from h2 to facilitate +// testing. +var errRequestCanceled = http2errRequestCanceled var errRequestCanceledConn = errors.New("net/http: request canceled while waiting for connection") // TODO: unify? func nop() {} @@ -2258,13 +2375,8 @@ func (pc *persistConn) closeLocked(err error) { if pc.closed == nil { pc.closed = err if pc.alt != nil { - // Do nothing; can only get here via getConn's - // handlePendingDial's putOrCloseIdleConn when - // it turns out the abandoned connection in - // flight ended up negotiating an alternate - // protocol. We don't use the connection - // freelist for http2. That's done by the - // alternate protocol's RoundTripper. + // Clean up any host connection counting. + pc.t.decHostConnCount(pc.cacheKey) } else { if err != errCallerOwnsConn { pc.conn.Close() @@ -2408,6 +2520,10 @@ func (tlsHandshakeTimeoutError) Timeout() bool { return true } func (tlsHandshakeTimeoutError) Temporary() bool { return true } func (tlsHandshakeTimeoutError) Error() string { return "net/http: TLS handshake timeout" } +func (tlsHandshakeTimeoutError) Is(target error) bool { + return target == os.ErrTimeout || target == os.ErrTemporary +} + // fakeLocker is a sync.Locker which does nothing. It's used to guard // test-only fields when not under test, to avoid runtime atomic // overhead. diff --git a/libgo/go/net/http/transport_test.go b/libgo/go/net/http/transport_test.go index 6e075847dde..2b58e1daecb 100644 --- a/libgo/go/net/http/transport_test.go +++ b/libgo/go/net/http/transport_test.go @@ -20,6 +20,7 @@ import ( "encoding/binary" "errors" "fmt" + "go/token" "internal/nettrace" "io" "io/ioutil" @@ -42,7 +43,7 @@ import ( "testing" "time" - "internal/x/net/http/httpguts" + "golang.org/x/net/http/httpguts" ) // TODO: test 5 pipelined requests with responses: 1) OK, 2) OK, Connection: Close @@ -588,6 +589,106 @@ func TestTransportMaxConnsPerHostIncludeDialInProgress(t *testing.T) { <-reqComplete } +func TestTransportMaxConnsPerHost(t *testing.T) { + defer afterTest(t) + + h := HandlerFunc(func(w ResponseWriter, r *Request) { + _, err := w.Write([]byte("foo")) + if err != nil { + t.Fatalf("Write: %v", err) + } + }) + + testMaxConns := func(scheme string, ts *httptest.Server) { + defer ts.Close() + + c := ts.Client() + tr := c.Transport.(*Transport) + tr.MaxConnsPerHost = 1 + if err := ExportHttp2ConfigureTransport(tr); err != nil { + t.Fatalf("ExportHttp2ConfigureTransport: %v", err) + } + + connCh := make(chan net.Conn, 1) + var dialCnt, gotConnCnt, tlsHandshakeCnt int32 + tr.Dial = func(network, addr string) (net.Conn, error) { + atomic.AddInt32(&dialCnt, 1) + c, err := net.Dial(network, addr) + connCh <- c + return c, err + } + + doReq := func() { + trace := &httptrace.ClientTrace{ + GotConn: func(connInfo httptrace.GotConnInfo) { + if !connInfo.Reused { + atomic.AddInt32(&gotConnCnt, 1) + } + }, + TLSHandshakeStart: func() { + atomic.AddInt32(&tlsHandshakeCnt, 1) + }, + } + req, _ := NewRequest("GET", ts.URL, nil) + req = req.WithContext(httptrace.WithClientTrace(req.Context(), trace)) + + resp, err := c.Do(req) + if err != nil { + t.Fatalf("request failed: %v", err) + } + defer resp.Body.Close() + _, err = ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read body failed: %v", err) + } + } + + wg := sync.WaitGroup{} + for i := 0; i < 10; i++ { + wg.Add(1) + go func() { + defer wg.Done() + doReq() + }() + } + wg.Wait() + + expected := int32(tr.MaxConnsPerHost) + if dialCnt != expected { + t.Errorf("Too many dials (%s): %d", scheme, dialCnt) + } + if gotConnCnt != expected { + t.Errorf("Too many get connections (%s): %d", scheme, gotConnCnt) + } + if ts.TLS != nil && tlsHandshakeCnt != expected { + t.Errorf("Too many tls handshakes (%s): %d", scheme, tlsHandshakeCnt) + } + + (<-connCh).Close() + tr.CloseIdleConnections() + + doReq() + expected++ + if dialCnt != expected { + t.Errorf("Too many dials (%s): %d", scheme, dialCnt) + } + if gotConnCnt != expected { + t.Errorf("Too many get connections (%s): %d", scheme, gotConnCnt) + } + if ts.TLS != nil && tlsHandshakeCnt != expected { + t.Errorf("Too many tls handshakes (%s): %d", scheme, tlsHandshakeCnt) + } + } + + testMaxConns("http", httptest.NewServer(h)) + testMaxConns("https", httptest.NewTLSServer(h)) + + ts := httptest.NewUnstartedServer(h) + ts.TLS = &tls.Config{NextProtos: []string{"h2"}} + ts.StartTLS() + testMaxConns("http2", ts) +} + func TestTransportRemovesDeadIdleConnections(t *testing.T) { setParallel(t) defer afterTest(t) @@ -636,6 +737,8 @@ func TestTransportRemovesDeadIdleConnections(t *testing.T) { } } +// Test that the Transport notices when a server hangs up on its +// unexpectedly (a keep-alive connection is closed). func TestTransportServerClosingUnexpectedly(t *testing.T) { setParallel(t) defer afterTest(t) @@ -672,13 +775,14 @@ func TestTransportServerClosingUnexpectedly(t *testing.T) { body1 := fetch(1, 0) body2 := fetch(2, 0) - ts.CloseClientConnections() // surprise! - - // This test has an expected race. Sleeping for 25 ms prevents - // it on most fast machines, causing the next fetch() call to - // succeed quickly. But if we do get errors, fetch() will retry 5 - // times with some delays between. - time.Sleep(25 * time.Millisecond) + // Close all the idle connections in a way that's similar to + // the server hanging up on us. We don't use + // httptest.Server.CloseClientConnections because it's + // best-effort and stops blocking after 5 seconds. On a loaded + // machine running many tests concurrently it's possible for + // that method to be async and cause the body3 fetch below to + // run on an old connection. This function is synchronous. + ExportCloseTransportConnsAbruptly(c.Transport.(*Transport)) body3 := fetch(3, 5) @@ -865,6 +969,10 @@ func TestRoundTripGzip(t *testing.T) { req.Header.Set("Accept-Encoding", test.accept) } res, err := tr.RoundTrip(req) + if err != nil { + t.Errorf("%d. RoundTrip: %v", i, err) + continue + } var body []byte if test.compressed { var r *gzip.Reader @@ -2110,7 +2218,7 @@ func testCancelRequestWithChannelBeforeDo(t *testing.T, withCtx bool) { } } else { if err == nil || !strings.Contains(err.Error(), "canceled") { - t.Errorf("Do error = %v; want cancelation", err) + t.Errorf("Do error = %v; want cancellation", err) } } } @@ -3589,6 +3697,13 @@ func TestTransportAutomaticHTTP2(t *testing.T) { testTransportAutoHTTP(t, &Transport{}, true) } +func TestTransportAutomaticHTTP2_DialerAndTLSConfigSupportsHTTP2AndTLSConfig(t *testing.T) { + testTransportAutoHTTP(t, &Transport{ + ForceAttemptHTTP2: true, + TLSClientConfig: new(tls.Config), + }, true) +} + // golang.org/issue/14391: also check DefaultTransport func TestTransportAutomaticHTTP2_DefaultTransport(t *testing.T) { testTransportAutoHTTP(t, DefaultTransport.(*Transport), true) @@ -3619,6 +3734,13 @@ func TestTransportAutomaticHTTP2_Dial(t *testing.T) { }, false) } +func TestTransportAutomaticHTTP2_DialContext(t *testing.T) { + var d net.Dialer + testTransportAutoHTTP(t, &Transport{ + DialContext: d.DialContext, + }, false) +} + func TestTransportAutomaticHTTP2_DialTLS(t *testing.T) { testTransportAutoHTTP(t, &Transport{ DialTLS: func(network, addr string) (net.Conn, error) { @@ -5059,3 +5181,270 @@ func TestTransportRequestReplayable(t *testing.T) { }) } } + +// testMockTCPConn is a mock TCP connection used to test that +// ReadFrom is called when sending the request body. +type testMockTCPConn struct { + *net.TCPConn + + ReadFromCalled bool +} + +func (c *testMockTCPConn) ReadFrom(r io.Reader) (int64, error) { + c.ReadFromCalled = true + return c.TCPConn.ReadFrom(r) +} + +func TestTransportRequestWriteRoundTrip(t *testing.T) { + nBytes := int64(1 << 10) + newFileFunc := func() (r io.Reader, done func(), err error) { + f, err := ioutil.TempFile("", "net-http-newfilefunc") + if err != nil { + return nil, nil, err + } + + // Write some bytes to the file to enable reading. + if _, err := io.CopyN(f, rand.Reader, nBytes); err != nil { + return nil, nil, fmt.Errorf("failed to write data to file: %v", err) + } + if _, err := f.Seek(0, 0); err != nil { + return nil, nil, fmt.Errorf("failed to seek to front: %v", err) + } + + done = func() { + f.Close() + os.Remove(f.Name()) + } + + return f, done, nil + } + + newBufferFunc := func() (io.Reader, func(), error) { + return bytes.NewBuffer(make([]byte, nBytes)), func() {}, nil + } + + cases := []struct { + name string + readerFunc func() (io.Reader, func(), error) + contentLength int64 + expectedReadFrom bool + }{ + { + name: "file, length", + readerFunc: newFileFunc, + contentLength: nBytes, + expectedReadFrom: true, + }, + { + name: "file, no length", + readerFunc: newFileFunc, + }, + { + name: "file, negative length", + readerFunc: newFileFunc, + contentLength: -1, + }, + { + name: "buffer", + contentLength: nBytes, + readerFunc: newBufferFunc, + }, + { + name: "buffer, no length", + readerFunc: newBufferFunc, + }, + { + name: "buffer, length -1", + contentLength: -1, + readerFunc: newBufferFunc, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + r, cleanup, err := tc.readerFunc() + if err != nil { + t.Fatal(err) + } + defer cleanup() + + tConn := &testMockTCPConn{} + trFunc := func(tr *Transport) { + tr.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { + var d net.Dialer + conn, err := d.DialContext(ctx, network, addr) + if err != nil { + return nil, err + } + + tcpConn, ok := conn.(*net.TCPConn) + if !ok { + return nil, fmt.Errorf("%s/%s does not provide a *net.TCPConn", network, addr) + } + + tConn.TCPConn = tcpConn + return tConn, nil + } + } + + cst := newClientServerTest( + t, + h1Mode, + HandlerFunc(func(w ResponseWriter, r *Request) { + io.Copy(ioutil.Discard, r.Body) + r.Body.Close() + w.WriteHeader(200) + }), + trFunc, + ) + defer cst.close() + + req, err := NewRequest("PUT", cst.ts.URL, r) + if err != nil { + t.Fatal(err) + } + req.ContentLength = tc.contentLength + req.Header.Set("Content-Type", "application/octet-stream") + resp, err := cst.c.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + t.Fatalf("status code = %d; want 200", resp.StatusCode) + } + + if !tConn.ReadFromCalled && tc.expectedReadFrom { + t.Fatalf("did not call ReadFrom") + } + + if tConn.ReadFromCalled && !tc.expectedReadFrom { + t.Fatalf("ReadFrom was unexpectedly invoked") + } + }) + } +} + +func TestTransportClone(t *testing.T) { + tr := &Transport{ + Proxy: func(*Request) (*url.URL, error) { panic("") }, + DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { panic("") }, + Dial: func(network, addr string) (net.Conn, error) { panic("") }, + DialTLS: func(network, addr string) (net.Conn, error) { panic("") }, + TLSClientConfig: new(tls.Config), + TLSHandshakeTimeout: time.Second, + DisableKeepAlives: true, + DisableCompression: true, + MaxIdleConns: 1, + MaxIdleConnsPerHost: 1, + MaxConnsPerHost: 1, + IdleConnTimeout: time.Second, + ResponseHeaderTimeout: time.Second, + ExpectContinueTimeout: time.Second, + ProxyConnectHeader: Header{}, + MaxResponseHeaderBytes: 1, + ForceAttemptHTTP2: true, + TLSNextProto: map[string]func(authority string, c *tls.Conn) RoundTripper{ + "foo": func(authority string, c *tls.Conn) RoundTripper { panic("") }, + }, + ReadBufferSize: 1, + WriteBufferSize: 1, + } + tr2 := tr.Clone() + rv := reflect.ValueOf(tr2).Elem() + rt := rv.Type() + for i := 0; i < rt.NumField(); i++ { + sf := rt.Field(i) + if !token.IsExported(sf.Name) { + continue + } + if rv.Field(i).IsZero() { + t.Errorf("cloned field t2.%s is zero", sf.Name) + } + } + + if _, ok := tr2.TLSNextProto["foo"]; !ok { + t.Errorf("cloned Transport lacked TLSNextProto 'foo' key") + } + + // But test that a nil TLSNextProto is kept nil: + tr = new(Transport) + tr2 = tr.Clone() + if tr2.TLSNextProto != nil { + t.Errorf("Transport.TLSNextProto unexpected non-nil") + } +} + +func TestIs408(t *testing.T) { + tests := []struct { + in string + want bool + }{ + {"HTTP/1.0 408", true}, + {"HTTP/1.1 408", true}, + {"HTTP/1.8 408", true}, + {"HTTP/2.0 408", false}, // maybe h2c would do this? but false for now. + {"HTTP/1.1 408 ", true}, + {"HTTP/1.1 40", false}, + {"http/1.0 408", false}, + {"HTTP/1-1 408", false}, + } + for _, tt := range tests { + if got := Export_is408Message([]byte(tt.in)); got != tt.want { + t.Errorf("is408Message(%q) = %v; want %v", tt.in, got, tt.want) + } + } +} + +func TestTransportIgnores408(t *testing.T) { + // Not parallel. Relies on mutating the log package's global Output. + defer log.SetOutput(log.Writer()) + + var logout bytes.Buffer + log.SetOutput(&logout) + + defer afterTest(t) + const target = "backend:443" + + cst := newClientServerTest(t, h1Mode, HandlerFunc(func(w ResponseWriter, r *Request) { + nc, _, err := w.(Hijacker).Hijack() + if err != nil { + t.Error(err) + return + } + defer nc.Close() + nc.Write([]byte("HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")) + nc.Write([]byte("HTTP/1.1 408 bye\r\n")) // changing 408 to 409 makes test fail + })) + defer cst.close() + req, err := NewRequest("GET", cst.ts.URL, nil) + if err != nil { + t.Fatal(err) + } + res, err := cst.c.Do(req) + if err != nil { + t.Fatal(err) + } + slurp, err := ioutil.ReadAll(res.Body) + if err != nil { + t.Fatal(err) + } + if err != nil { + t.Fatal(err) + } + if string(slurp) != "ok" { + t.Fatalf("got %q; want ok", slurp) + } + + t0 := time.Now() + for i := 0; i < 50; i++ { + time.Sleep(time.Duration(i) * 5 * time.Millisecond) + if cst.tr.IdleConnKeyCountForTesting() == 0 { + if got := logout.String(); got != "" { + t.Fatalf("expected no log output; got: %s", got) + } + return + } + } + t.Fatalf("timeout after %v waiting for Transport connections to die off", time.Since(t0)) +} diff --git a/libgo/go/net/interface_aix.go b/libgo/go/net/interface_aix.go index 1fe9bbaf696..f57c5ff6622 100644 --- a/libgo/go/net/interface_aix.go +++ b/libgo/go/net/interface_aix.go @@ -5,6 +5,7 @@ package net import ( + "internal/poll" "internal/syscall/unix" "syscall" "unsafe" @@ -56,6 +57,12 @@ func interfaceTable(ifindex int) ([]Interface, error) { return nil, err } + sock, err := sysSocket(syscall.AF_INET, syscall.SOCK_DGRAM, 0) + if err != nil { + return nil, err + } + defer poll.CloseFunc(sock) + var ift []Interface for len(tab) > 0 { ifm := (*syscall.IfMsgHdr)(unsafe.Pointer(&tab[0])) @@ -73,11 +80,7 @@ func interfaceTable(ifindex int) ([]Interface, error) { // Retrieve MTU ifr := &ifreq{} copy(ifr.Name[:], ifi.Name) - sock, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, 0) - if err != nil { - return nil, err - } - err = unix.Ioctl(sock, _SIOCGIFMTU, uintptr(unsafe.Pointer(ifr))) + err = unix.Ioctl(sock, syscall.SIOCGIFMTU, uintptr(unsafe.Pointer(ifr))) if err != nil { return nil, err } diff --git a/libgo/go/net/interface_bsd.go b/libgo/go/net/interface_bsd.go index 77372964b10..d791cb30167 100644 --- a/libgo/go/net/interface_bsd.go +++ b/libgo/go/net/interface_bsd.go @@ -9,7 +9,7 @@ package net import ( "syscall" - "internal/x/net/route" + "golang.org/x/net/route" ) // If the ifindex is zero, interfaceTable returns mappings of all diff --git a/libgo/go/net/interface_bsdvar.go b/libgo/go/net/interface_bsdvar.go index 818fafe9708..a809b5f5ce4 100644 --- a/libgo/go/net/interface_bsdvar.go +++ b/libgo/go/net/interface_bsdvar.go @@ -9,7 +9,7 @@ package net import ( "syscall" - "internal/x/net/route" + "golang.org/x/net/route" ) func interfaceMessages(ifindex int) ([]route.Message, error) { diff --git a/libgo/go/net/interface_darwin.go b/libgo/go/net/interface_darwin.go index 6a6b3a58187..bb4fd73a987 100644 --- a/libgo/go/net/interface_darwin.go +++ b/libgo/go/net/interface_darwin.go @@ -7,7 +7,7 @@ package net import ( "syscall" - "internal/x/net/route" + "golang.org/x/net/route" ) func interfaceMessages(ifindex int) ([]route.Message, error) { diff --git a/libgo/go/net/interface_freebsd.go b/libgo/go/net/interface_freebsd.go index 8eee2aa0315..45badd64954 100644 --- a/libgo/go/net/interface_freebsd.go +++ b/libgo/go/net/interface_freebsd.go @@ -7,7 +7,7 @@ package net import ( "syscall" - "internal/x/net/route" + "golang.org/x/net/route" ) func interfaceMessages(ifindex int) ([]route.Message, error) { diff --git a/libgo/go/net/interface_plan9.go b/libgo/go/net/interface_plan9.go index e5d77390f86..8fe91384064 100644 --- a/libgo/go/net/interface_plan9.go +++ b/libgo/go/net/interface_plan9.go @@ -152,10 +152,14 @@ func interfaceAddrTable(ifi *Interface) ([]Addr, error) { } defer statusf.close() + // Read but ignore first line as it only contains the table header. + // See https://9p.io/magic/man2html/3/ip + if _, ok := statusf.readLine(); !ok { + return nil, errors.New("cannot read header line for interface: " + status) + } line, ok := statusf.readLine() - line, ok = statusf.readLine() if !ok { - return nil, errors.New("cannot parse IP address for interface: " + status) + return nil, errors.New("cannot read IP address for interface: " + status) } // This assumes only a single address for the interface. diff --git a/libgo/go/net/interface_solaris.go b/libgo/go/net/interface_solaris.go index 868d4174ed3..5f9367f996f 100644 --- a/libgo/go/net/interface_solaris.go +++ b/libgo/go/net/interface_solaris.go @@ -7,7 +7,7 @@ package net import ( "syscall" - "internal/x/net/lif" + "golang.org/x/net/lif" ) // If the ifindex is zero, interfaceTable returns mappings of all diff --git a/libgo/go/net/interface_test.go b/libgo/go/net/interface_test.go index c6b514abcec..fb6032fbc06 100644 --- a/libgo/go/net/interface_test.go +++ b/libgo/go/net/interface_test.go @@ -61,7 +61,7 @@ func TestInterfaces(t *testing.T) { t.Fatal(err) } switch runtime.GOOS { - case "solaris": + case "solaris", "illumos": if ifxi.Index != ifi.Index { t.Errorf("got %v; want %v", ifxi, ifi) } @@ -278,7 +278,7 @@ func checkUnicastStats(ifStats *ifStats, uniStats *routeStats) error { func checkMulticastStats(ifStats *ifStats, uniStats, multiStats *routeStats) error { switch runtime.GOOS { - case "aix", "dragonfly", "nacl", "netbsd", "openbsd", "plan9", "solaris": + case "aix", "dragonfly", "nacl", "netbsd", "openbsd", "plan9", "solaris", "illumos": default: // Test the existence of connected multicast route // clones for IPv4. Unlike IPv6, IPv4 multicast diff --git a/libgo/go/net/ip.go b/libgo/go/net/ip.go index 9a6fda00e84..cf90c0cd546 100644 --- a/libgo/go/net/ip.go +++ b/libgo/go/net/ip.go @@ -565,7 +565,7 @@ func parseIPv6Zone(s string) (IP, string) { return parseIPv6(s), zone } -// parseIPv6Zone parses s as a literal IPv6 address described in RFC 4291 +// parseIPv6 parses s as a literal IPv6 address described in RFC 4291 // and RFC 5952. func parseIPv6(s string) (ip IP) { ip = make(IP, IPv6len) diff --git a/libgo/go/net/listen_test.go b/libgo/go/net/listen_test.go index 6c9b92a9fc5..fef2b6405fa 100644 --- a/libgo/go/net/listen_test.go +++ b/libgo/go/net/listen_test.go @@ -534,8 +534,8 @@ func TestIPv4MulticastListener(t *testing.T) { switch runtime.GOOS { case "android", "nacl", "plan9": t.Skipf("not supported on %s", runtime.GOOS) - case "solaris": - t.Skipf("not supported on solaris, see golang.org/issue/7399") + case "solaris", "illumos": + t.Skipf("not supported on solaris or illumos, see golang.org/issue/7399") } if !supportsIPv4() { t.Skip("IPv4 is not supported") @@ -609,8 +609,8 @@ func TestIPv6MulticastListener(t *testing.T) { switch runtime.GOOS { case "plan9": t.Skipf("not supported on %s", runtime.GOOS) - case "solaris": - t.Skipf("not supported on solaris, see issue 7399") + case "solaris", "illumos": + t.Skipf("not supported on solaris or illumos, see issue 7399") } if !supportsIPv6() { t.Skip("IPv6 is not supported") @@ -674,7 +674,7 @@ func checkMulticastListener(c *UDPConn, ip IP) error { func multicastRIBContains(ip IP) (bool, error) { switch runtime.GOOS { - case "aix", "dragonfly", "netbsd", "openbsd", "plan9", "solaris", "windows": + case "aix", "dragonfly", "netbsd", "openbsd", "plan9", "solaris", "illumos", "windows": return true, nil // not implemented yet case "linux": if runtime.GOARCH == "arm" || runtime.GOARCH == "alpha" { diff --git a/libgo/go/net/lookup.go b/libgo/go/net/lookup.go index 08e8d013855..24d0d25c3a2 100644 --- a/libgo/go/net/lookup.go +++ b/libgo/go/net/lookup.go @@ -177,7 +177,7 @@ func (r *Resolver) LookupHost(ctx context.Context, host string) (addrs []string, // Make sure that no matter what we do later, host=="" is rejected. // parseIP, for example, does accept empty strings. if host == "" { - return nil, &DNSError{Err: errNoSuchHost.Error(), Name: host} + return nil, &DNSError{Err: errNoSuchHost.Error(), Name: host, IsNotFound: true} } if ip, _ := parseIPZone(host); ip != nil { return []string{host}, nil @@ -238,7 +238,7 @@ func (r *Resolver) lookupIPAddr(ctx context.Context, network, host string) ([]IP // Make sure that no matter what we do later, host=="" is rejected. // parseIP, for example, does accept empty strings. if host == "" { - return nil, &DNSError{Err: errNoSuchHost.Error(), Name: host} + return nil, &DNSError{Err: errNoSuchHost.Error(), Name: host, IsNotFound: true} } if ip, zone := parseIPZone(host); ip != nil { return []IPAddr{{IP: ip, Zone: zone}}, nil @@ -255,7 +255,7 @@ func (r *Resolver) lookupIPAddr(ctx context.Context, network, host string) ([]IP resolverFunc = alt } - // We don't want a cancelation of ctx to affect the + // We don't want a cancellation of ctx to affect the // lookupGroup operation. Otherwise if our context gets // canceled it might cause an error to be returned to a lookup // using a completely different context. However we need to preserve diff --git a/libgo/go/net/lookup_plan9.go b/libgo/go/net/lookup_plan9.go index 70805ddf4cd..6a2d48eedac 100644 --- a/libgo/go/net/lookup_plan9.go +++ b/libgo/go/net/lookup_plan9.go @@ -147,10 +147,12 @@ func (*Resolver) lookupHost(ctx context.Context, host string) (addrs []string, e // host names in local network (e.g. from /lib/ndb/local) lines, err := queryCS(ctx, "net", host, "1") if err != nil { + dnsError := &DNSError{Err: err.Error(), Name: host} if stringsHasSuffix(err.Error(), "dns failure") { - err = errNoSuchHost + dnsError.Err = errNoSuchHost.Error() + dnsError.IsNotFound = true } - return + return nil, dnsError } loop: for _, line := range lines { diff --git a/libgo/go/net/lookup_test.go b/libgo/go/net/lookup_test.go index 28a895e15d1..dd599c7c1c2 100644 --- a/libgo/go/net/lookup_test.go +++ b/libgo/go/net/lookup_test.go @@ -877,6 +877,9 @@ func TestLookupNonLDH(t *testing.T) { if !strings.HasSuffix(err.Error(), errNoSuchHost.Error()) { t.Fatalf("lookup error = %v, want %v", err, errNoSuchHost) } + if !err.(*DNSError).IsNotFound { + t.Fatalf("lookup error = %v, want true", err.(*DNSError).IsNotFound) + } } func TestLookupContextCancel(t *testing.T) { @@ -1181,3 +1184,13 @@ func TestWithUnexpiredValuesPreserved(t *testing.T) { t.Errorf("Lookup after expiry: Got %v want nil", g) } } + +// Issue 31586: don't crash on null byte in name +func TestLookupNullByte(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + testenv.SkipFlakyNet(t) + _, err := LookupHost("foo\x00bar") // used to crash on Windows + if err == nil { + t.Errorf("unexpected success") + } +} diff --git a/libgo/go/net/lookup_unix.go b/libgo/go/net/lookup_unix.go index 0199a87ee38..c27b1a0a195 100644 --- a/libgo/go/net/lookup_unix.go +++ b/libgo/go/net/lookup_unix.go @@ -12,7 +12,7 @@ import ( "sync" "syscall" - "internal/x/net/dns/dnsmessage" + "golang.org/x/net/dns/dnsmessage" ) var onceReadProtocols sync.Once diff --git a/libgo/go/net/lookup_windows.go b/libgo/go/net/lookup_windows.go index 8a68d18a674..d7b28f5e185 100644 --- a/libgo/go/net/lookup_windows.go +++ b/libgo/go/net/lookup_windows.go @@ -56,7 +56,12 @@ func lookupProtocol(ctx context.Context, name string) (int, error) { if proto, err := lookupProtocolMap(name); err == nil { return proto, nil } - r.err = &DNSError{Err: r.err.Error(), Name: name} + + dnsError := &DNSError{Err: r.err.Error(), Name: name} + if r.err == errNoSuchHost { + dnsError.IsNotFound = true + } + r.err = dnsError } return r.proto, r.err case <-ctx.Done(): @@ -96,9 +101,18 @@ func (r *Resolver) lookupIP(ctx context.Context, network, name string) ([]IPAddr Protocol: syscall.IPPROTO_IP, } var result *syscall.AddrinfoW - e := syscall.GetAddrInfoW(syscall.StringToUTF16Ptr(name), nil, &hints, &result) + name16p, err := syscall.UTF16PtrFromString(name) + if err != nil { + return nil, &DNSError{Name: name, Err: err.Error()} + } + e := syscall.GetAddrInfoW(name16p, nil, &hints, &result) if e != nil { - return nil, &DNSError{Err: winError("getaddrinfow", e).Error(), Name: name} + err := winError("getaddrinfow", e) + dnsError := &DNSError{Err: err.Error(), Name: name} + if err == errNoSuchHost { + dnsError.IsNotFound = true + } + return nil, dnsError } defer syscall.FreeAddrInfoW(result) addrs := make([]IPAddr, 0, 5) @@ -124,11 +138,14 @@ func (r *Resolver) lookupIP(ctx context.Context, network, name string) ([]IPAddr err error } - ch := make(chan ret, 1) - go func() { - addr, err := getaddr() - ch <- ret{addrs: addr, err: err} - }() + var ch chan ret + if ctx.Err() == nil { + ch = make(chan ret, 1) + go func() { + addr, err := getaddr() + ch <- ret{addrs: addr, err: err} + }() + } select { case r := <-ch: @@ -176,7 +193,12 @@ func (r *Resolver) lookupPort(ctx context.Context, network, service string) (int if port, err := lookupPortMap(network, service); err == nil { return port, nil } - return 0, &DNSError{Err: winError("getaddrinfow", e).Error(), Name: network + "/" + service} + err := winError("getaddrinfow", e) + dnsError := &DNSError{Err: err.Error(), Name: network + "/" + service} + if err == errNoSuchHost { + dnsError.IsNotFound = true + } + return 0, dnsError } defer syscall.FreeAddrInfoW(result) if result == nil { diff --git a/libgo/go/net/mac.go b/libgo/go/net/mac.go index f3b1694735c..373ac3d7e20 100644 --- a/libgo/go/net/mac.go +++ b/libgo/go/net/mac.go @@ -26,15 +26,15 @@ func (a HardwareAddr) String() string { // ParseMAC parses s as an IEEE 802 MAC-48, EUI-48, EUI-64, or a 20-octet // IP over InfiniBand link-layer address using one of the following formats: -// 01:23:45:67:89:ab -// 01:23:45:67:89:ab:cd:ef -// 01:23:45:67:89:ab:cd:ef:00:00:01:23:45:67:89:ab:cd:ef:00:00 -// 01-23-45-67-89-ab -// 01-23-45-67-89-ab-cd-ef -// 01-23-45-67-89-ab-cd-ef-00-00-01-23-45-67-89-ab-cd-ef-00-00 -// 0123.4567.89ab -// 0123.4567.89ab.cdef -// 0123.4567.89ab.cdef.0000.0123.4567.89ab.cdef.0000 +// 00:00:5e:00:53:01 +// 02:00:5e:10:00:00:00:01 +// 00:00:00:00:fe:80:00:00:00:00:00:00:02:00:5e:10:00:00:00:01 +// 00-00-5e-00-53-01 +// 02-00-5e-10-00-00-00-01 +// 00-00-00-00-fe-80-00-00-00-00-00-00-02-00-5e-10-00-00-00-01 +// 0000.5e00.5301 +// 0200.5e10.0000.0001 +// 0000.0000.fe80.0000.0000.0000.0200.5e10.0000.0001 func ParseMAC(s string) (hw HardwareAddr, err error) { if len(s) < 14 { goto error diff --git a/libgo/go/net/mac_test.go b/libgo/go/net/mac_test.go index 2630d190479..cad884fcf5d 100644 --- a/libgo/go/net/mac_test.go +++ b/libgo/go/net/mac_test.go @@ -15,49 +15,69 @@ var parseMACTests = []struct { out HardwareAddr err string }{ - {"01:23:45:67:89:AB", HardwareAddr{1, 0x23, 0x45, 0x67, 0x89, 0xab}, ""}, - {"01-23-45-67-89-AB", HardwareAddr{1, 0x23, 0x45, 0x67, 0x89, 0xab}, ""}, - {"0123.4567.89AB", HardwareAddr{1, 0x23, 0x45, 0x67, 0x89, 0xab}, ""}, - {"ab:cd:ef:AB:CD:EF", HardwareAddr{0xab, 0xcd, 0xef, 0xab, 0xcd, 0xef}, ""}, - {"01.02.03.04.05.06", nil, "invalid MAC address"}, - {"01:02:03:04:05:06:", nil, "invalid MAC address"}, - {"x1:02:03:04:05:06", nil, "invalid MAC address"}, - {"01002:03:04:05:06", nil, "invalid MAC address"}, - {"01:02003:04:05:06", nil, "invalid MAC address"}, - {"01:02:03004:05:06", nil, "invalid MAC address"}, - {"01:02:03:04005:06", nil, "invalid MAC address"}, - {"01:02:03:04:05006", nil, "invalid MAC address"}, - {"01-02:03:04:05:06", nil, "invalid MAC address"}, - {"01:02-03-04-05-06", nil, "invalid MAC address"}, - {"0123:4567:89AF", nil, "invalid MAC address"}, - {"0123-4567-89AF", nil, "invalid MAC address"}, - {"01:23:45:67:89:AB:CD:EF", HardwareAddr{1, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef}, ""}, - {"01-23-45-67-89-AB-CD-EF", HardwareAddr{1, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef}, ""}, - {"0123.4567.89AB.CDEF", HardwareAddr{1, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef}, ""}, + // See RFC 7042, Section 2.1.1. + {"00:00:5e:00:53:01", HardwareAddr{0x00, 0x00, 0x5e, 0x00, 0x53, 0x01}, ""}, + {"00-00-5e-00-53-01", HardwareAddr{0x00, 0x00, 0x5e, 0x00, 0x53, 0x01}, ""}, + {"0000.5e00.5301", HardwareAddr{0x00, 0x00, 0x5e, 0x00, 0x53, 0x01}, ""}, + + // See RFC 7042, Section 2.2.2. + {"02:00:5e:10:00:00:00:01", HardwareAddr{0x02, 0x00, 0x5e, 0x10, 0x00, 0x00, 0x00, 0x01}, ""}, + {"02-00-5e-10-00-00-00-01", HardwareAddr{0x02, 0x00, 0x5e, 0x10, 0x00, 0x00, 0x00, 0x01}, ""}, + {"0200.5e10.0000.0001", HardwareAddr{0x02, 0x00, 0x5e, 0x10, 0x00, 0x00, 0x00, 0x01}, ""}, + + // See RFC 4391, Section 9.1.1. { - "01:23:45:67:89:ab:cd:ef:00:00:01:23:45:67:89:ab:cd:ef:00:00", + "00:00:00:00:fe:80:00:00:00:00:00:00:02:00:5e:10:00:00:00:01", HardwareAddr{ - 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0x00, 0x00, - 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0xfe, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x5e, 0x10, 0x00, 0x00, 0x00, 0x01, }, "", }, { - "01-23-45-67-89-ab-cd-ef-00-00-01-23-45-67-89-ab-cd-ef-00-00", + "00-00-00-00-fe-80-00-00-00-00-00-00-02-00-5e-10-00-00-00-01", HardwareAddr{ - 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0x00, 0x00, - 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0xfe, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x5e, 0x10, 0x00, 0x00, 0x00, 0x01, }, "", }, { - "0123.4567.89ab.cdef.0000.0123.4567.89ab.cdef.0000", + "0000.0000.fe80.0000.0000.0000.0200.5e10.0000.0001", HardwareAddr{ - 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0x00, 0x00, - 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0xfe, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x5e, 0x10, 0x00, 0x00, 0x00, 0x01, }, "", }, + + {"ab:cd:ef:AB:CD:EF", HardwareAddr{0xab, 0xcd, 0xef, 0xab, 0xcd, 0xef}, ""}, + {"ab:cd:ef:AB:CD:EF:ab:cd", HardwareAddr{0xab, 0xcd, 0xef, 0xab, 0xcd, 0xef, 0xab, 0xcd}, ""}, + { + "ab:cd:ef:AB:CD:EF:ab:cd:ef:AB:CD:EF:ab:cd:ef:AB:CD:EF:ab:cd", + HardwareAddr{ + 0xab, 0xcd, 0xef, 0xab, + 0xcd, 0xef, 0xab, 0xcd, 0xef, 0xab, 0xcd, 0xef, + 0xab, 0xcd, 0xef, 0xab, 0xcd, 0xef, 0xab, 0xcd, + }, + "", + }, + + {"01.02.03.04.05.06", nil, "invalid MAC address"}, + {"01:02:03:04:05:06:", nil, "invalid MAC address"}, + {"x1:02:03:04:05:06", nil, "invalid MAC address"}, + {"01002:03:04:05:06", nil, "invalid MAC address"}, + {"01:02003:04:05:06", nil, "invalid MAC address"}, + {"01:02:03004:05:06", nil, "invalid MAC address"}, + {"01:02:03:04005:06", nil, "invalid MAC address"}, + {"01:02:03:04:05006", nil, "invalid MAC address"}, + {"01-02:03:04:05:06", nil, "invalid MAC address"}, + {"01:02-03-04-05-06", nil, "invalid MAC address"}, + {"0123:4567:89AF", nil, "invalid MAC address"}, + {"0123-4567-89AF", nil, "invalid MAC address"}, } func TestParseMAC(t *testing.T) { diff --git a/libgo/go/net/mail/message.go b/libgo/go/net/mail/message.go index 554377aa1da..e0907806ca6 100644 --- a/libgo/go/net/mail/message.go +++ b/libgo/go/net/mail/message.go @@ -342,6 +342,21 @@ func (p *addrParser) parseAddress(handleGroup bool) ([]*Address, error) { } // angle-addr = "<" addr-spec ">" if !p.consume('<') { + atext := true + for _, r := range displayName { + if !isAtext(r, true, false) { + atext = false + break + } + } + if atext { + // The input is like "foo.bar"; it's possible the input + // meant to be "foo.bar@domain", or "foo.bar <...>". + return nil, errors.New("mail: missing '@' or angle-addr") + } + // The input is like "Full Name", which couldn't possibly be a + // valid email address if followed by "@domain"; the input + // likely meant to be "Full Name <...>". return nil, errors.New("mail: no angle-addr") } spec, err = p.consumeAddrSpec() diff --git a/libgo/go/net/mail/message_test.go b/libgo/go/net/mail/message_test.go index 14ac9192a4a..2950bc4de92 100644 --- a/libgo/go/net/mail/message_test.go +++ b/libgo/go/net/mail/message_test.go @@ -144,6 +144,9 @@ func TestAddressParsingError(t *testing.T) { 12: {"root group: embed group: null@example.com;", "no angle-addr"}, 13: {"group not closed: null@example.com", "expected comma"}, 14: {"group: first@example.com, second@example.com;", "group with multiple addresses"}, + 15: {"john.doe", "missing '@' or angle-addr"}, + 16: {"john.doe@", "no angle-addr"}, + 17: {"John Doe@foo.bar", "no angle-addr"}, } for i, tc := range mustErrTestCases { diff --git a/libgo/go/net/net.go b/libgo/go/net/net.go index 77b8f69074e..b3f9b8ba07f 100644 --- a/libgo/go/net/net.go +++ b/libgo/go/net/net.go @@ -282,7 +282,7 @@ func (c *conn) SetWriteBuffer(bytes int) error { return nil } -// File returns a copy of the underlying os.File +// File returns a copy of the underlying os.File. // It is the caller's responsibility to close f when finished. // Closing c does not affect f, and closing f does not affect c. // @@ -448,6 +448,8 @@ type OpError struct { Err error } +func (e *OpError) Unwrap() error { return e.Err } + func (e *OpError) Error() string { if e == nil { return "<nil>" @@ -473,7 +475,7 @@ func (e *OpError) Error() string { var ( // aLongTimeAgo is a non-zero time, far in the past, used for - // immediate cancelation of dials. + // immediate cancellation of dials. aLongTimeAgo = time.Unix(1, 0) // nonDeadline and noCancel are just zero values for @@ -514,6 +516,16 @@ func (e *OpError) Temporary() bool { return ok && t.Temporary() } +func (e *OpError) Is(target error) bool { + switch target { + case os.ErrTemporary: + return e.Temporary() + case os.ErrTimeout: + return e.Timeout() + } + return false +} + // A ParseError is the error type of literal network address parsers. type ParseError struct { // Type is the type of string that was expected, such as @@ -563,6 +575,7 @@ type DNSConfigError struct { Err error } +func (e *DNSConfigError) Unwrap() error { return e.Err } func (e *DNSConfigError) Error() string { return "error reading DNS config: " + e.Err.Error() } func (e *DNSConfigError) Timeout() bool { return false } func (e *DNSConfigError) Temporary() bool { return false } @@ -579,6 +592,7 @@ type DNSError struct { Server string // server used IsTimeout bool // if true, timed out; not all timeouts set this IsTemporary bool // if true, error is temporary; not all errors set this + IsNotFound bool // if true, host could not be found } func (e *DNSError) Error() string { @@ -603,6 +617,16 @@ func (e *DNSError) Timeout() bool { return e.IsTimeout } // error and return a DNSError for which Temporary returns false. func (e *DNSError) Temporary() bool { return e.IsTimeout || e.IsTemporary } +func (e *DNSError) Is(target error) bool { + switch target { + case os.ErrTemporary: + return e.Temporary() + case os.ErrTimeout: + return e.Timeout() + } + return false +} + type writerOnly struct { io.Writer } diff --git a/libgo/go/net/pipe.go b/libgo/go/net/pipe.go index 9177fc40364..8cc127464b1 100644 --- a/libgo/go/net/pipe.go +++ b/libgo/go/net/pipe.go @@ -6,6 +6,7 @@ package net import ( "io" + "os" "sync" "time" ) @@ -84,6 +85,10 @@ func (timeoutError) Error() string { return "deadline exceeded" } func (timeoutError) Timeout() bool { return true } func (timeoutError) Temporary() bool { return true } +func (timeoutError) Is(target error) bool { + return target == os.ErrTemporary || target == os.ErrTimeout +} + type pipeAddr struct{} func (pipeAddr) Network() string { return "pipe" } diff --git a/libgo/go/net/pipe_test.go b/libgo/go/net/pipe_test.go index 53ddc16313d..9cc24148ca2 100644 --- a/libgo/go/net/pipe_test.go +++ b/libgo/go/net/pipe_test.go @@ -10,7 +10,7 @@ import ( "testing" "time" - "internal/x/net/nettest" + "golang.org/x/net/nettest" ) func TestPipe(t *testing.T) { diff --git a/libgo/go/net/platform_test.go b/libgo/go/net/platform_test.go index 7e9ad70d19b..10f55c971df 100644 --- a/libgo/go/net/platform_test.go +++ b/libgo/go/net/platform_test.go @@ -14,6 +14,23 @@ import ( "testing" ) +var unixEnabledOnAIX bool + +func init() { + if runtime.GOOS == "aix" { + // Unix network isn't properly working on AIX 7.2 with + // Technical Level < 2. + // The information is retrieved only once in this init() + // instead of everytime testableNetwork is called. + out, _ := exec.Command("oslevel", "-s").Output() + if len(out) >= len("7200-XX-ZZ-YYMM") { // AIX 7.2, Tech Level XX, Service Pack ZZ, date YYMM + aixVer := string(out[:4]) + tl, _ := strconv.Atoi(string(out[5:7])) + unixEnabledOnAIX = aixVer > "7200" || (aixVer == "7200" && tl >= 2) + } + } +} + // testableNetwork reports whether network is testable on the current // platform configuration. func testableNetwork(network string) bool { @@ -38,15 +55,7 @@ func testableNetwork(network string) bool { case "android", "nacl", "plan9", "windows": return false case "aix": - // Unix network isn't properly working on AIX 7.2 with Technical Level < 2 - out, err := exec.Command("oslevel", "-s").Output() - if err != nil { - return false - } - if tl, err := strconv.Atoi(string(out[5:7])); err != nil || tl < 2 { - return false - } - return true + return unixEnabledOnAIX } // iOS does not support unix, unixgram. if runtime.GOOS == "darwin" && (runtime.GOARCH == "arm" || runtime.GOARCH == "arm64") { diff --git a/libgo/go/net/rpc/client_test.go b/libgo/go/net/rpc/client_test.go index d116d2acc9a..03225e3d010 100644 --- a/libgo/go/net/rpc/client_test.go +++ b/libgo/go/net/rpc/client_test.go @@ -57,7 +57,7 @@ func TestGobError(t *testing.T) { if err == nil { t.Fatal("no error") } - if !strings.Contains("reading body EOF", err.(error).Error()) { + if !strings.Contains(err.(error).Error(), "reading body EOF") { t.Fatal("expected `reading body EOF', got", err) } }() diff --git a/libgo/go/net/rpc/server.go b/libgo/go/net/rpc/server.go index 7bb6476ffab..9cb928240f1 100644 --- a/libgo/go/net/rpc/server.go +++ b/libgo/go/net/rpc/server.go @@ -130,6 +130,7 @@ import ( "bufio" "encoding/gob" "errors" + "go/token" "io" "log" "net" @@ -137,8 +138,6 @@ import ( "reflect" "strings" "sync" - "unicode" - "unicode/utf8" ) const ( @@ -202,12 +201,6 @@ func NewServer() *Server { // DefaultServer is the default instance of *Server. var DefaultServer = NewServer() -// Is this an exported - upper case - name? -func isExported(name string) bool { - rune, _ := utf8.DecodeRuneInString(name) - return unicode.IsUpper(rune) -} - // Is this type exported or a builtin? func isExportedOrBuiltinType(t reflect.Type) bool { for t.Kind() == reflect.Ptr { @@ -215,7 +208,7 @@ func isExportedOrBuiltinType(t reflect.Type) bool { } // PkgPath will be non-empty even for an exported type, // so we need to check the type name as well. - return isExported(t.Name()) || t.PkgPath() == "" + return token.IsExported(t.Name()) || t.PkgPath() == "" } // Register publishes in the server the set of methods of the @@ -251,7 +244,7 @@ func (server *Server) register(rcvr interface{}, name string, useName bool) erro log.Print(s) return errors.New(s) } - if !isExported(sname) && !useName { + if !token.IsExported(sname) && !useName { s := "rpc.Register: type " + sname + " is not exported" log.Print(s) return errors.New(s) diff --git a/libgo/go/net/sendfile_unix_alt.go b/libgo/go/net/sendfile_unix_alt.go index 43df3bfd15e..8cededce58d 100644 --- a/libgo/go/net/sendfile_unix_alt.go +++ b/libgo/go/net/sendfile_unix_alt.go @@ -68,8 +68,8 @@ func sendFile(c *netFD, r io.Reader) (written int64, err error, handled bool) { written, werr = poll.SendFile(&c.pfd, int(fd), pos, remain) return true }) - if werr == nil { - werr = err + if err == nil { + err = werr } if lr != nil { diff --git a/libgo/go/net/smtp/smtp_test.go b/libgo/go/net/smtp/smtp_test.go index e366ef80158..8195f91419e 100644 --- a/libgo/go/net/smtp/smtp_test.go +++ b/libgo/go/net/smtp/smtp_test.go @@ -900,8 +900,8 @@ tN8URjVmyEo= -----END CERTIFICATE-----`) // localhostKey is the private key for localhostCert. -var localhostKey = []byte(` ------BEGIN RSA PRIVATE KEY----- +var localhostKey = []byte(testingKey(` +-----BEGIN RSA TESTING KEY----- MIICXgIBAAKBgQDScVtBC45ayNsa16NylbPXnc6XOJkzhtWMn2Niu43DmfZHLq5h AB9+Gpok4icKaZxV7ayImCWzIf1pGHq8qKhsFshRddRTUAF3np5sDOW3QuhsuXHu lkQzLfQuoiL0TrOYvdi90bOliWQVGdGurAS1ZhsYF/fOc7bnRLnoIJYfZQIDAQAB @@ -915,4 +915,6 @@ vNjJu3yvoEZeIeuzouX9TJE21/33FaeDdsXbRhQEj23cqR38qFHsF1qAYNMCQQDP QXLEiJoClkR2orAmqjPLVhR3t2oB3INcnEjLNSq8LHyQEfXyaFfu4U9l5+fRPL2i jiC0k/9L5dHUsF0XZothAkEA23ddgRs+Id/HxtojqqUT27B8MT/IGNrYsp4DvS/c qgkeluku4GjxRlDMBuXk94xOBEinUs+p/hwP1Alll80Tpg== ------END RSA PRIVATE KEY-----`) +-----END RSA TESTING KEY-----`)) + +func testingKey(s string) string { return strings.ReplaceAll(s, "TESTING KEY", "PRIVATE KEY") } diff --git a/libgo/go/net/splice_test.go b/libgo/go/net/splice_test.go index 324103edf8b..b14ab9f7edc 100644 --- a/libgo/go/net/splice_test.go +++ b/libgo/go/net/splice_test.go @@ -369,6 +369,7 @@ func startSpliceClient(conn Conn, op string, chunkSize, totalSize int) (func(), "GO_NET_TEST_SPLICE_OP=" + op, "GO_NET_TEST_SPLICE_CHUNK_SIZE=" + strconv.Itoa(chunkSize), "GO_NET_TEST_SPLICE_TOTAL_SIZE=" + strconv.Itoa(totalSize), + "TMPDIR=" + os.Getenv("TMPDIR"), }...) cmd.ExtraFiles = append(cmd.ExtraFiles, f) cmd.Stdout = os.Stdout diff --git a/libgo/go/net/tcpsock.go b/libgo/go/net/tcpsock.go index db5d1f8482e..0daa2f6487a 100644 --- a/libgo/go/net/tcpsock.go +++ b/libgo/go/net/tcpsock.go @@ -154,7 +154,7 @@ func (c *TCPConn) SetLinger(sec int) error { } // SetKeepAlive sets whether the operating system should send -// keepalive messages on the connection. +// keep-alive messages on the connection. func (c *TCPConn) SetKeepAlive(keepalive bool) error { if !c.ok() { return syscall.EINVAL @@ -165,7 +165,7 @@ func (c *TCPConn) SetKeepAlive(keepalive bool) error { return nil } -// SetKeepAlivePeriod sets period between keep alives. +// SetKeepAlivePeriod sets period between keep-alives. func (c *TCPConn) SetKeepAlivePeriod(d time.Duration) error { if !c.ok() { return syscall.EINVAL @@ -224,6 +224,7 @@ func DialTCP(network string, laddr, raddr *TCPAddr) (*TCPConn, error) { // use variables of type Listener instead of assuming TCP. type TCPListener struct { fd *netFD + lc ListenConfig } // SyscallConn returns a raw network connection. diff --git a/libgo/go/net/tcpsock_plan9.go b/libgo/go/net/tcpsock_plan9.go index f70ef6f43ab..e2e835957c0 100644 --- a/libgo/go/net/tcpsock_plan9.go +++ b/libgo/go/net/tcpsock_plan9.go @@ -44,7 +44,16 @@ func (ln *TCPListener) accept() (*TCPConn, error) { if err != nil { return nil, err } - return newTCPConn(fd), nil + tc := newTCPConn(fd) + if ln.lc.KeepAlive >= 0 { + setKeepAlive(fd, true) + ka := ln.lc.KeepAlive + if ln.lc.KeepAlive == 0 { + ka = defaultTCPKeepAlive + } + setKeepAlivePeriod(fd, ka) + } + return tc, nil } func (ln *TCPListener) close() error { @@ -74,5 +83,5 @@ func (sl *sysListener) listenTCP(ctx context.Context, laddr *TCPAddr) (*TCPListe if err != nil { return nil, err } - return &TCPListener{fd}, nil + return &TCPListener{fd: fd, lc: sl.ListenConfig}, nil } diff --git a/libgo/go/net/tcpsock_posix.go b/libgo/go/net/tcpsock_posix.go index 3d4579cf702..1ff0ec04616 100644 --- a/libgo/go/net/tcpsock_posix.go +++ b/libgo/go/net/tcpsock_posix.go @@ -140,7 +140,16 @@ func (ln *TCPListener) accept() (*TCPConn, error) { if err != nil { return nil, err } - return newTCPConn(fd), nil + tc := newTCPConn(fd) + if ln.lc.KeepAlive >= 0 { + setKeepAlive(fd, true) + ka := ln.lc.KeepAlive + if ln.lc.KeepAlive == 0 { + ka = defaultTCPKeepAlive + } + setKeepAlivePeriod(fd, ka) + } + return tc, nil } func (ln *TCPListener) close() error { @@ -160,5 +169,5 @@ func (sl *sysListener) listenTCP(ctx context.Context, laddr *TCPAddr) (*TCPListe if err != nil { return nil, err } - return &TCPListener{fd}, nil + return &TCPListener{fd: fd, lc: sl.ListenConfig}, nil } diff --git a/libgo/go/net/tcpsock_test.go b/libgo/go/net/tcpsock_test.go index 65a6ee579ea..a89f6217196 100644 --- a/libgo/go/net/tcpsock_test.go +++ b/libgo/go/net/tcpsock_test.go @@ -656,7 +656,7 @@ func TestTCPSelfConnect(t *testing.T) { n = 1000 } switch runtime.GOOS { - case "darwin", "dragonfly", "freebsd", "netbsd", "openbsd", "plan9", "solaris", "windows": + case "darwin", "dragonfly", "freebsd", "netbsd", "openbsd", "plan9", "illumos", "solaris", "windows": // Non-Linux systems take a long time to figure // out that there is nothing listening on localhost. n = 100 diff --git a/libgo/go/net/tcpsockopt_darwin.go b/libgo/go/net/tcpsockopt_darwin.go index 5b738d23e6d..da0d173453e 100644 --- a/libgo/go/net/tcpsockopt_darwin.go +++ b/libgo/go/net/tcpsockopt_darwin.go @@ -10,6 +10,7 @@ import ( "time" ) +// syscall.TCP_KEEPINTVL is missing on some darwin architectures. const sysTCP_KEEPINTVL = 0x101 func setKeepAlivePeriod(fd *netFD, d time.Duration) error { diff --git a/libgo/go/net/testdata/freebsd-usevc-resolv.conf b/libgo/go/net/testdata/freebsd-usevc-resolv.conf new file mode 100644 index 00000000000..4afb281c5bb --- /dev/null +++ b/libgo/go/net/testdata/freebsd-usevc-resolv.conf @@ -0,0 +1 @@ +options usevc
\ No newline at end of file diff --git a/libgo/go/net/testdata/linux-use-vc-resolv.conf b/libgo/go/net/testdata/linux-use-vc-resolv.conf new file mode 100644 index 00000000000..4e4a58b7a7e --- /dev/null +++ b/libgo/go/net/testdata/linux-use-vc-resolv.conf @@ -0,0 +1 @@ +options use-vc
\ No newline at end of file diff --git a/libgo/go/net/testdata/openbsd-tcp-resolv.conf b/libgo/go/net/testdata/openbsd-tcp-resolv.conf new file mode 100644 index 00000000000..7929e50e8df --- /dev/null +++ b/libgo/go/net/testdata/openbsd-tcp-resolv.conf @@ -0,0 +1 @@ +options tcp
\ No newline at end of file diff --git a/libgo/go/net/testdata/single-request-reopen-resolv.conf b/libgo/go/net/testdata/single-request-reopen-resolv.conf new file mode 100644 index 00000000000..9bddeb38444 --- /dev/null +++ b/libgo/go/net/testdata/single-request-reopen-resolv.conf @@ -0,0 +1 @@ +options single-request-reopen
\ No newline at end of file diff --git a/libgo/go/net/testdata/single-request-resolv.conf b/libgo/go/net/testdata/single-request-resolv.conf new file mode 100644 index 00000000000..5595d29a66a --- /dev/null +++ b/libgo/go/net/testdata/single-request-resolv.conf @@ -0,0 +1 @@ +options single-request
\ No newline at end of file diff --git a/libgo/go/net/textproto/reader.go b/libgo/go/net/textproto/reader.go index 2c4f25d5ae6..a5cab993b29 100644 --- a/libgo/go/net/textproto/reader.go +++ b/libgo/go/net/textproto/reader.go @@ -11,6 +11,7 @@ import ( "io/ioutil" "strconv" "strings" + "sync" ) // A Reader implements convenience methods for reading requests @@ -27,6 +28,7 @@ type Reader struct { // should be reading from an io.LimitReader or similar Reader to bound // the size of responses. func NewReader(r *bufio.Reader) *Reader { + commonHeaderOnce.Do(initCommonHeader) return &Reader{R: r} } @@ -571,6 +573,8 @@ func (r *Reader) upcomingHeaderNewlines() (n int) { // If s contains a space or invalid header field bytes, it is // returned without modifications. func CanonicalMIMEHeaderKey(s string) string { + commonHeaderOnce.Do(initCommonHeader) + // Quick check for canonical encoding. upper := true for i := 0; i < len(s); i++ { @@ -642,9 +646,12 @@ func canonicalMIMEHeaderKey(a []byte) string { } // commonHeader interns common header strings. -var commonHeader = make(map[string]string) +var commonHeader map[string]string + +var commonHeaderOnce sync.Once -func init() { +func initCommonHeader() { + commonHeader = make(map[string]string) for _, v := range []string{ "Accept", "Accept-Charset", diff --git a/libgo/go/net/textproto/reader_test.go b/libgo/go/net/textproto/reader_test.go index aec193a7cbd..e9ae04d37de 100644 --- a/libgo/go/net/textproto/reader_test.go +++ b/libgo/go/net/textproto/reader_test.go @@ -332,12 +332,13 @@ func TestReadMultiLineError(t *testing.T) { if msg != wantMsg { t.Errorf("ReadResponse: msg=%q, want %q", msg, wantMsg) } - if err.Error() != "550 "+wantMsg { + if err != nil && err.Error() != "550 "+wantMsg { t.Errorf("ReadResponse: error=%q, want %q", err.Error(), "550 "+wantMsg) } } func TestCommonHeaders(t *testing.T) { + commonHeaderOnce.Do(initCommonHeader) for h := range commonHeader { if h != CanonicalMIMEHeaderKey(h) { t.Errorf("Non-canonical header %q in commonHeader", h) diff --git a/libgo/go/net/textproto/writer.go b/libgo/go/net/textproto/writer.go index 1bc5974c6c2..33c146c0220 100644 --- a/libgo/go/net/textproto/writer.go +++ b/libgo/go/net/textproto/writer.go @@ -58,7 +58,8 @@ type dotWriter struct { } const ( - wstateBeginLine = iota // beginning of line; initial state; must be zero + wstateBegin = iota // initial state; must be zero + wstateBeginLine // beginning of line wstateCR // wrote \r (possibly at end of line) wstateData // writing data in middle of line ) @@ -68,7 +69,7 @@ func (d *dotWriter) Write(b []byte) (n int, err error) { for n < len(b) { c := b[n] switch d.state { - case wstateBeginLine: + case wstateBegin, wstateBeginLine: d.state = wstateData if c == '.' { // escape leading dot diff --git a/libgo/go/net/textproto/writer_test.go b/libgo/go/net/textproto/writer_test.go index ac03669fa2d..2afef11b5ee 100644 --- a/libgo/go/net/textproto/writer_test.go +++ b/libgo/go/net/textproto/writer_test.go @@ -33,3 +33,29 @@ func TestDotWriter(t *testing.T) { t.Fatalf("wrote %q", s) } } + +func TestDotWriterCloseEmptyWrite(t *testing.T) { + var buf bytes.Buffer + w := NewWriter(bufio.NewWriter(&buf)) + d := w.DotWriter() + n, err := d.Write([]byte{}) + if n != 0 || err != nil { + t.Fatalf("Write: %d, %s", n, err) + } + d.Close() + want := "\r\n.\r\n" + if s := buf.String(); s != want { + t.Fatalf("wrote %q; want %q", s, want) + } +} + +func TestDotWriterCloseNoWrite(t *testing.T) { + var buf bytes.Buffer + w := NewWriter(bufio.NewWriter(&buf)) + d := w.DotWriter() + d.Close() + want := "\r\n.\r\n" + if s := buf.String(); s != want { + t.Fatalf("wrote %q; want %q", s, want) + } +} diff --git a/libgo/go/net/timeout_test.go b/libgo/go/net/timeout_test.go index 9599fa1d3e8..4b9fe7eba97 100644 --- a/libgo/go/net/timeout_test.go +++ b/libgo/go/net/timeout_test.go @@ -7,7 +7,9 @@ package net import ( + "errors" "fmt" + "internal/oserror" "internal/poll" "internal/testenv" "io" @@ -88,6 +90,9 @@ func TestDialTimeout(t *testing.T) { if nerr, ok := err.(Error); !ok || !nerr.Timeout() { t.Fatalf("#%d: %v", i, err) } + if !errors.Is(err, oserror.ErrTimeout) { + t.Fatalf("#%d: Dial error is not os.ErrTimeout: %v", i, err) + } } } } diff --git a/libgo/go/net/url/url.go b/libgo/go/net/url/url.go index 64274a0a364..7f6ff93ce46 100644 --- a/libgo/go/net/url/url.go +++ b/libgo/go/net/url/url.go @@ -13,6 +13,7 @@ package url import ( "errors" "fmt" + "internal/oserror" "sort" "strconv" "strings" @@ -25,25 +26,10 @@ type Error struct { Err error } -func (e *Error) Error() string { return e.Op + " " + e.URL + ": " + e.Err.Error() } - -type timeout interface { - Timeout() bool -} - -func (e *Error) Timeout() bool { - t, ok := e.Err.(timeout) - return ok && t.Timeout() -} - -type temporary interface { - Temporary() bool -} - -func (e *Error) Temporary() bool { - t, ok := e.Err.(temporary) - return ok && t.Temporary() -} +func (e *Error) Unwrap() error { return e.Err } +func (e *Error) Error() string { return e.Op + " " + e.URL + ": " + e.Err.Error() } +func (e *Error) Timeout() bool { return oserror.IsTimeout(e.Err) } +func (e *Error) Temporary() bool { return oserror.IsTemporary(e.Err) } func ishex(c byte) bool { switch { @@ -100,7 +86,7 @@ func (e InvalidHostError) Error() string { // reserved characters correctly. See golang.org/issue/5684. func shouldEscape(c byte, mode encoding) bool { // ยง2.3 Unreserved characters (alphanum) - if 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' || '0' <= c && c <= '9' { + if 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || '0' <= c && c <= '9' { return false } @@ -250,29 +236,24 @@ func unescape(s string, mode encoding) (string, error) { return s, nil } - t := make([]byte, len(s)-2*n) - j := 0 - for i := 0; i < len(s); { + var t strings.Builder + t.Grow(len(s) - 2*n) + for i := 0; i < len(s); i++ { switch s[i] { case '%': - t[j] = unhex(s[i+1])<<4 | unhex(s[i+2]) - j++ - i += 3 + t.WriteByte(unhex(s[i+1])<<4 | unhex(s[i+2])) + i += 2 case '+': if mode == encodeQueryComponent { - t[j] = ' ' + t.WriteByte(' ') } else { - t[j] = '+' + t.WriteByte('+') } - j++ - i++ default: - t[j] = s[i] - j++ - i++ + t.WriteByte(s[i]) } } - return string(t), nil + return t.String(), nil } // QueryEscape escapes the string so it can be safely placed @@ -281,8 +262,8 @@ func QueryEscape(s string) string { return escape(s, encodeQueryComponent) } -// PathEscape escapes the string so it can be safely placed -// inside a URL path segment. +// PathEscape escapes the string so it can be safely placed inside a URL path segment, +// replacing special characters (including /) with %XX sequences as needed. func PathEscape(s string) string { return escape(s, encodePathSegment) } @@ -356,10 +337,11 @@ func escape(s string, mode encoding) string { // Note that the Path field is stored in decoded form: /%47%6f%2f becomes /Go/. // A consequence is that it is impossible to tell which slashes in the Path were // slashes in the raw URL and which were %2f. This distinction is rarely important, -// but when it is, code must not use Path directly. -// The Parse function sets both Path and RawPath in the URL it returns, -// and URL's String method uses RawPath if it is a valid encoding of Path, -// by calling the EscapedPath method. +// but when it is, the code should use RawPath, an optional field which only gets +// set if the default encoding is different from Path. +// +// URL's String method uses the EscapedPath method to obtain the path. See the +// EscapedPath method for more details. type URL struct { Scheme string Opaque string // encoded opaque data diff --git a/libgo/go/net/url/url_test.go b/libgo/go/net/url/url_test.go index c5fc90d5156..e6d6ef8a838 100644 --- a/libgo/go/net/url/url_test.go +++ b/libgo/go/net/url/url_test.go @@ -930,6 +930,11 @@ var pathEscapeTests = []EscapeTest{ nil, }, { + "a/b", + "a%2Fb", + nil, + }, + { "one two", "one%20two", nil, @@ -1629,6 +1634,12 @@ func TestURLHostname(t *testing.T) { {"[1:2:3:4]", "1:2:3:4"}, {"[1:2:3:4]:80", "1:2:3:4"}, {"[::1]:80", "::1"}, + {"[::1]", "::1"}, + {"localhost", "localhost"}, + {"localhost:443", "localhost"}, + {"some.super.long.domain.example.org:8080", "some.super.long.domain.example.org"}, + {"[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:17000", "2001:0db8:85a3:0000:0000:8a2e:0370:7334"}, + {"[2001:0db8:85a3:0000:0000:8a2e:0370:7334]", "2001:0db8:85a3:0000:0000:8a2e:0370:7334"}, } for _, tt := range tests { u := &URL{Host: tt.host} |