libgo: update to final 1.14.2 release

Reviewed-on: https://go-review.googlesource.com/c/gofrontend/+/227551

10 files changed, 163 insertions, 74 deletions

diff --git a/libgo/MERGE b/libgo/MERGE
index 4398cd7dc28..8cae45f6b8e 100644
--- a/libgo/MERGE
+++ b/libgo/MERGE
@@ -1,4 +1,4 @@
-edea4a79e8d7dea2456b688f492c8af33d381dc2
+96745b980cfde139e8611772e2bc0c59a8e6cdf7
 
 The first line of this file holds the git revision number of the
 last merge done from the master library sources.
diff --git a/libgo/go/cmd/go/alldocs.go b/libgo/go/cmd/go/alldocs.go
index 971a756b37d..c2678c3dd3c 100644
--- a/libgo/go/cmd/go/alldocs.go
+++ b/libgo/go/cmd/go/alldocs.go
@@ -2506,13 +2506,21 @@
 // The "go get" command remains permitted to update go.mod even with -mod=readonly,
 // and the "go mod" commands do not take the -mod flag (or any other build flags).
 //
-// If invoked with -mod=vendor, the go command assumes that the vendor
-// directory holds the correct copies of dependencies and ignores
-// the dependency descriptions in go.mod.
+// If invoked with -mod=vendor, the go command loads packages from the main
+// module's vendor directory instead of downloading modules to and loading packages
+// from the module cache. The go command assumes the vendor directory holds
+// correct copies of dependencies, and it does not compute the set of required
+// module versions from go.mod files. However, the go command does check that
+// vendor/modules.txt (generated by 'go mod vendor') contains metadata consistent
+// with go.mod.
 //
 // If invoked with -mod=mod, the go command loads modules from the module cache
 // even if there is a vendor directory present.
 //
+// If the go command is not invoked with a -mod flag and the vendor directory
+// is present and the "go" version in go.mod is 1.14 or higher, the go command
+// will act as if it were invoked with -mod=vendor.
+//
 // Pseudo-versions
 //
 // The go.mod file and the go command more generally use semantic versions as
@@ -2710,22 +2718,28 @@
 //
 // Modules and vendoring
 //
-// When using modules, the go command completely ignores vendor directories.
-//
-// By default, the go command satisfies dependencies by downloading modules
-// from their sources and using those downloaded copies (after verification,
-// as described in the previous section). To allow interoperation with older
-// versions of Go, or to ensure that all files used for a build are stored
-// together in a single file tree, 'go mod vendor' creates a directory named
-// vendor in the root directory of the main module and stores there all the
-// packages from dependency modules that are needed to support builds and
-// tests of packages in the main module.
-//
-// To build using the main module's top-level vendor directory to satisfy
-// dependencies (disabling use of the usual network sources and local
-// caches), use 'go build -mod=vendor'. Note that only the main module's
-// top-level vendor directory is used; vendor directories in other locations
-// are still ignored.
+// When using modules, the go command typically satisfies dependencies by
+// downloading modules from their sources and using those downloaded copies
+// (after verification, as described in the previous section). Vendoring may
+// be used to allow interoperation with older versions of Go, or to ensure
+// that all files used for a build are stored together in a single file tree.
+//
+// The command 'go mod vendor' constructs a directory named vendor in the main
+// module's root directory that contains copies of all packages needed to support
+// builds and tests of packages in the main module. 'go mod vendor' also
+// creates the file vendor/modules.txt that contains metadata about vendored
+// packages and module versions. This file should be kept consistent with go.mod:
+// when vendoring is used, 'go mod vendor' should be run after go.mod is updated.
+//
+// If the vendor directory is present in the main module's root directory, it will
+// be used automatically if the "go" version in the main module's go.mod file is
+// 1.14 or higher. Build commands like 'go build' and 'go test' will load packages
+// from the vendor directory instead of accessing the network or the local module
+// cache. To explicitly enable vendoring, invoke the go command with the flag
+// -mod=vendor. To disable vendoring, use the flag -mod=mod.
+//
+// Unlike vendoring in GOPATH, the go command ignores vendor directories in
+// locations other than the main module's root directory.
 //
 //
 // Module authentication using go.sum
diff --git a/libgo/go/cmd/go/internal/modcmd/verify.go b/libgo/go/cmd/go/internal/modcmd/verify.go
index 831e5cf85bb..ac3f1351c87 100644
--- a/libgo/go/cmd/go/internal/modcmd/verify.go
+++ b/libgo/go/cmd/go/internal/modcmd/verify.go
@@ -6,6 +6,7 @@ package modcmd
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -67,12 +68,10 @@ func verifyMod(mod module.Version) bool {
 		_, zipErr = os.Stat(zip)
 	}
 	dir, dirErr := modfetch.DownloadDir(mod)
-	if dirErr == nil {
-		_, dirErr = os.Stat(dir)
-	}
 	data, err := ioutil.ReadFile(zip + "hash")
 	if err != nil {
-		if zipErr != nil && os.IsNotExist(zipErr) && dirErr != nil && os.IsNotExist(dirErr) {
+		if zipErr != nil && errors.Is(zipErr, os.ErrNotExist) &&
+			dirErr != nil && errors.Is(dirErr, os.ErrNotExist) {
 			// Nothing downloaded yet. Nothing to verify.
 			return true
 		}
@@ -81,7 +80,7 @@ func verifyMod(mod module.Version) bool {
 	}
 	h := string(bytes.TrimSpace(data))
 
-	if zipErr != nil && os.IsNotExist(zipErr) {
+	if zipErr != nil && errors.Is(zipErr, os.ErrNotExist) {
 		// ok
 	} else {
 		hZ, err := dirhash.HashZip(zip, dirhash.DefaultHash)
@@ -93,7 +92,7 @@ func verifyMod(mod module.Version) bool {
 			ok = false
 		}
 	}
-	if dirErr != nil && os.IsNotExist(dirErr) {
+	if dirErr != nil && errors.Is(dirErr, os.ErrNotExist) {
 		// ok
 	} else {
 		hD, err := dirhash.HashDir(dir, mod.Path+"@"+mod.Version, dirhash.DefaultHash)
diff --git a/libgo/go/cmd/go/internal/modfetch/cache.go b/libgo/go/cmd/go/internal/modfetch/cache.go
index 947192bd83e..d6ff068e7bc 100644
--- a/libgo/go/cmd/go/internal/modfetch/cache.go
+++ b/libgo/go/cmd/go/internal/modfetch/cache.go
@@ -7,6 +7,7 @@ package modfetch
 import (
 	"bytes"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -56,8 +57,11 @@ func CachePath(m module.Version, suffix string) (string, error) {
 	return filepath.Join(dir, encVer+"."+suffix), nil
 }
 
-// DownloadDir returns the directory to which m should be downloaded.
-// Note that the directory may not yet exist.
+// DownloadDir returns the directory to which m should have been downloaded.
+// An error will be returned if the module path or version cannot be escaped.
+// An error satisfying errors.Is(err, os.ErrNotExist) will be returned
+// along with the directory if the directory does not exist or if the directory
+// is not completely populated.
 func DownloadDir(m module.Version) (string, error) {
 	if PkgMod == "" {
 		return "", fmt.Errorf("internal error: modfetch.PkgMod not set")
@@ -76,9 +80,39 @@ func DownloadDir(m module.Version) (string, error) {
 	if err != nil {
 		return "", err
 	}
-	return filepath.Join(PkgMod, enc+"@"+encVer), nil
+
+	dir := filepath.Join(PkgMod, enc+"@"+encVer)
+	if fi, err := os.Stat(dir); os.IsNotExist(err) {
+		return dir, err
+	} else if err != nil {
+		return dir, &DownloadDirPartialError{dir, err}
+	} else if !fi.IsDir() {
+		return dir, &DownloadDirPartialError{dir, errors.New("not a directory")}
+	}
+	partialPath, err := CachePath(m, "partial")
+	if err != nil {
+		return dir, err
+	}
+	if _, err := os.Stat(partialPath); err == nil {
+		return dir, &DownloadDirPartialError{dir, errors.New("not completely extracted")}
+	} else if !os.IsNotExist(err) {
+		return dir, err
+	}
+	return dir, nil
+}
+
+// DownloadDirPartialError is returned by DownloadDir if a module directory
+// exists but was not completely populated.
+//
+// DownloadDirPartialError is equivalent to os.ErrNotExist.
+type DownloadDirPartialError struct {
+	Dir string
+	Err error
 }
 
+func (e *DownloadDirPartialError) Error() string     { return fmt.Sprintf("%s: %v", e.Dir, e.Err) }
+func (e *DownloadDirPartialError) Is(err error) bool { return err == os.ErrNotExist }
+
 // lockVersion locks a file within the module cache that guards the downloading
 // and extraction of the zipfile for the given module version.
 func lockVersion(mod module.Version) (unlock func(), err error) {
diff --git a/libgo/go/cmd/go/internal/modfetch/fetch.go b/libgo/go/cmd/go/internal/modfetch/fetch.go
index 54fbd92b43e..aadf8834574 100644
--- a/libgo/go/cmd/go/internal/modfetch/fetch.go
+++ b/libgo/go/cmd/go/internal/modfetch/fetch.go
@@ -22,6 +22,7 @@ import (
 	"cmd/go/internal/lockedfile"
 	"cmd/go/internal/par"
 	"cmd/go/internal/renameio"
+	"cmd/go/internal/robustio"
 
 	"golang.org/x/mod/module"
 	"golang.org/x/mod/sumdb/dirhash"
@@ -45,24 +46,27 @@ func Download(mod module.Version) (dir string, err error) {
 		err error
 	}
 	c := downloadCache.Do(mod, func() interface{} {
-		dir, err := DownloadDir(mod)
+		dir, err := download(mod)
 		if err != nil {
 			return cached{"", err}
 		}
-		if err := download(mod, dir); err != nil {
-			return cached{"", err}
-		}
 		checkMod(mod)
 		return cached{dir, nil}
 	}).(cached)
 	return c.dir, c.err
 }
 
-func download(mod module.Version, dir string) (err error) {
-	// If the directory exists, the module has already been extracted.
-	fi, err := os.Stat(dir)
-	if err == nil && fi.IsDir() {
-		return nil
+func download(mod module.Version) (dir string, err error) {
+	// If the directory exists, and no .partial file exists,
+	// the module has already been completely extracted.
+	// .partial files may be created when future versions of cmd/go
+	// extract module zip directories in place instead of extracting
+	// to a random temporary directory and renaming.
+	dir, err = DownloadDir(mod)
+	if err == nil {
+		return dir, nil
+	} else if dir == "" || !errors.Is(err, os.ErrNotExist) {
+		return "", err
 	}
 
 	// To avoid cluttering the cache with extraneous files,
@@ -70,22 +74,24 @@ func download(mod module.Version, dir string) (err error) {
 	// Invoke DownloadZip before locking the file.
 	zipfile, err := DownloadZip(mod)
 	if err != nil {
-		return err
+		return "", err
 	}
 
 	unlock, err := lockVersion(mod)
 	if err != nil {
-		return err
+		return "", err
 	}
 	defer unlock()
 
 	// Check whether the directory was populated while we were waiting on the lock.
-	fi, err = os.Stat(dir)
-	if err == nil && fi.IsDir() {
-		return nil
+	_, dirErr := DownloadDir(mod)
+	if dirErr == nil {
+		return dir, nil
 	}
+	_, dirExists := dirErr.(*DownloadDirPartialError)
 
-	// Clean up any remaining temporary directories from previous runs.
+	// Clean up any remaining temporary directories from previous runs, as well
+	// as partially extracted diectories created by future versions of cmd/go.
 	// This is only safe to do because the lock file ensures that their writers
 	// are no longer active.
 	parentDir := filepath.Dir(dir)
@@ -95,6 +101,19 @@ func download(mod module.Version, dir string) (err error) {
 			RemoveAll(path) // best effort
 		}
 	}
+	if dirExists {
+		if err := RemoveAll(dir); err != nil {
+			return "", err
+		}
+	}
+
+	partialPath, err := CachePath(mod, "partial")
+	if err != nil {
+		return "", err
+	}
+	if err := os.Remove(partialPath); err != nil && !os.IsNotExist(err) {
+		return "", err
+	}
 
 	// Extract the zip file to a temporary directory, then rename it to the
 	// final path. That way, we can use the existence of the source directory to
@@ -102,11 +121,11 @@ func download(mod module.Version, dir string) (err error) {
 	// the entire directory (e.g. as an attempt to prune out file corruption)
 	// the module cache will still be left in a recoverable state.
 	if err := os.MkdirAll(parentDir, 0777); err != nil {
-		return err
+		return "", err
 	}
 	tmpDir, err := ioutil.TempDir(parentDir, tmpPrefix)
 	if err != nil {
-		return err
+		return "", err
 	}
 	defer func() {
 		if err != nil {
@@ -116,11 +135,11 @@ func download(mod module.Version, dir string) (err error) {
 
 	if err := modzip.Unzip(tmpDir, mod, zipfile); err != nil {
 		fmt.Fprintf(os.Stderr, "-> %s\n", err)
-		return err
+		return "", err
 	}
 
-	if err := os.Rename(tmpDir, dir); err != nil {
-		return err
+	if err := robustio.Rename(tmpDir, dir); err != nil {
+		return "", err
 	}
 
 	if !cfg.ModCacheRW {
@@ -128,7 +147,7 @@ func download(mod module.Version, dir string) (err error) {
 		// os.Rename was observed to fail for read-only directories on macOS.
 		makeDirsReadOnly(dir)
 	}
-	return nil
+	return dir, nil
 }
 
 var downloadZipCache par.Cache
diff --git a/libgo/go/cmd/go/internal/modload/build.go b/libgo/go/cmd/go/internal/modload/build.go
index d0642bccf84..2a69d5953d3 100644
--- a/libgo/go/cmd/go/internal/modload/build.go
+++ b/libgo/go/cmd/go/internal/modload/build.go
@@ -148,9 +148,7 @@ func moduleInfo(m module.Version, fromBuildList bool) *modinfo.ModulePublic {
 			}
 			dir, err := modfetch.DownloadDir(mod)
 			if err == nil {
-				if info, err := os.Stat(dir); err == nil && info.IsDir() {
-					m.Dir = dir
-				}
+				m.Dir = dir
 			}
 		}
 	}
diff --git a/libgo/go/cmd/go/internal/modload/help.go b/libgo/go/cmd/go/internal/modload/help.go
index 66c1f70db73..bd19bb43aa0 100644
--- a/libgo/go/cmd/go/internal/modload/help.go
+++ b/libgo/go/cmd/go/internal/modload/help.go
@@ -176,13 +176,21 @@ not need updates, such as in a continuous integration and testing system.
 The "go get" command remains permitted to update go.mod even with -mod=readonly,
 and the "go mod" commands do not take the -mod flag (or any other build flags).
 
-If invoked with -mod=vendor, the go command assumes that the vendor
-directory holds the correct copies of dependencies and ignores
-the dependency descriptions in go.mod.
+If invoked with -mod=vendor, the go command loads packages from the main
+module's vendor directory instead of downloading modules to and loading packages
+from the module cache. The go command assumes the vendor directory holds
+correct copies of dependencies, and it does not compute the set of required
+module versions from go.mod files. However, the go command does check that
+vendor/modules.txt (generated by 'go mod vendor') contains metadata consistent
+with go.mod.
 
 If invoked with -mod=mod, the go command loads modules from the module cache
 even if there is a vendor directory present.
 
+If the go command is not invoked with a -mod flag and the vendor directory
+is present and the "go" version in go.mod is 1.14 or higher, the go command
+will act as if it were invoked with -mod=vendor.
+
 Pseudo-versions
 
 The go.mod file and the go command more generally use semantic versions as
@@ -380,22 +388,28 @@ the format of the cached downloaded packages.
 
 Modules and vendoring
 
-When using modules, the go command completely ignores vendor directories.
-
-By default, the go command satisfies dependencies by downloading modules
-from their sources and using those downloaded copies (after verification,
-as described in the previous section). To allow interoperation with older
-versions of Go, or to ensure that all files used for a build are stored
-together in a single file tree, 'go mod vendor' creates a directory named
-vendor in the root directory of the main module and stores there all the
-packages from dependency modules that are needed to support builds and
-tests of packages in the main module.
-
-To build using the main module's top-level vendor directory to satisfy
-dependencies (disabling use of the usual network sources and local
-caches), use 'go build -mod=vendor'. Note that only the main module's
-top-level vendor directory is used; vendor directories in other locations
-are still ignored.
+When using modules, the go command typically satisfies dependencies by
+downloading modules from their sources and using those downloaded copies
+(after verification, as described in the previous section). Vendoring may
+be used to allow interoperation with older versions of Go, or to ensure
+that all files used for a build are stored together in a single file tree.
+
+The command 'go mod vendor' constructs a directory named vendor in the main
+module's root directory that contains copies of all packages needed to support
+builds and tests of packages in the main module. 'go mod vendor' also
+creates the file vendor/modules.txt that contains metadata about vendored
+packages and module versions. This file should be kept consistent with go.mod:
+when vendoring is used, 'go mod vendor' should be run after go.mod is updated.
+
+If the vendor directory is present in the main module's root directory, it will
+be used automatically if the "go" version in the main module's go.mod file is
+1.14 or higher. Build commands like 'go build' and 'go test' will load packages
+from the vendor directory instead of accessing the network or the local module
+cache. To explicitly enable vendoring, invoke the go command with the flag
+-mod=vendor. To disable vendoring, use the flag -mod=mod.
+
+Unlike vendoring in GOPATH, the go command ignores vendor directories in
+locations other than the main module's root directory.
 	`,
 }
 
diff --git a/libgo/go/cmd/go/internal/robustio/robustio_flaky.go b/libgo/go/cmd/go/internal/robustio/robustio_flaky.go
index e57c8c74c4c..d4cb7e6457f 100644
--- a/libgo/go/cmd/go/internal/robustio/robustio_flaky.go
+++ b/libgo/go/cmd/go/internal/robustio/robustio_flaky.go
@@ -15,7 +15,7 @@ import (
 	"time"
 )
 
-const arbitraryTimeout = 500 * time.Millisecond
+const arbitraryTimeout = 2000 * time.Millisecond
 
 // retry retries ephemeral errors from f up to an arbitrary timeout
 // to work around filesystem flakiness on Windows and Darwin.
diff --git a/libgo/go/runtime/mbitmap.go b/libgo/go/runtime/mbitmap.go
index 457da13687a..be8e00c2025 100644
--- a/libgo/go/runtime/mbitmap.go
+++ b/libgo/go/runtime/mbitmap.go
@@ -1941,7 +1941,11 @@ Run:
 // The bitmask starts at s.startAddr.
 // The result must be deallocated with dematerializeGCProg.
 func materializeGCProg(ptrdata uintptr, prog *byte) *mspan {
-	s := mheap_.allocManual((ptrdata/(8*sys.PtrSize)+pageSize-1)/pageSize, &memstats.gc_sys)
+	// Each word of ptrdata needs one bit in the bitmap.
+	bitmapBytes := divRoundUp(ptrdata, 8*sys.PtrSize)
+	// Compute the number of pages needed for bitmapBytes.
+	pages := divRoundUp(bitmapBytes, pageSize)
+	s := mheap_.allocManual(pages, &memstats.gc_sys)
 	runGCProg(addb(prog, 4), nil, (*byte)(unsafe.Pointer(s.startAddr)), 1)
 	return s
 }
diff --git a/libgo/go/runtime/stubs.go b/libgo/go/runtime/stubs.go
index ae6134dcb52..4a06da51fe5 100644
--- a/libgo/go/runtime/stubs.go
+++ b/libgo/go/runtime/stubs.go
@@ -250,6 +250,13 @@ func alignDown(n, a uintptr) uintptr {
 	return n &^ (a - 1)
 }
 
+// divRoundUp returns ceil(n / a).
+func divRoundUp(n, a uintptr) uintptr {
+	// a is generally a power of two. This will get inlined and
+	// the compiler will optimize the division.
+	return (n + a - 1) / a
+}
+
 // checkASM returns whether assembly runtime checks have passed.
 func checkASM() bool {
 	return true