Fix the intermittent TestGPGGit failures (#9360)

* Fix the intermittent TestGPGGit failures

Reattempt to open the listener if the port is busy with a delay up to a second
Switch from generating a private key each time, just use a known good key

go.mod

@@ -93,12 +93,12 @@ require (
 	github.com/unknwon/paginater v0.0.0-20151104151617-7748a72e0141
 	github.com/urfave/cli v1.20.0
 	github.com/yohcop/openid-go v0.0.0-20160914080427-2c050d2dae53
-	golang.org/x/crypto v0.0.0-20191117063200-497ca9f6d64f
+	golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413
 	golang.org/x/net v0.0.0-20191101175033-0deb6923b6d9
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
 	golang.org/x/sys v0.0.0-20191127021746-63cb32ae39b2
 	golang.org/x/text v0.3.2
-	golang.org/x/tools v0.0.0-20190910221609-7f5965fd7709 // indirect
+	golang.org/x/tools v0.0.0-20191213221258-04c2e8eff935 // indirect
 	gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
 	gopkg.in/asn1-ber.v1 v1.0.0-20150924051756-4e86f4367175 // indirect
 	gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df

go.sum

@@ -596,8 +596,9 @@ golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190927123631-a832865fa7ad h1:5E5raQxcv+6CZ11RrBYQe5WRbUIWpScjh0kvHZkZIrQ=
 golang.org/x/crypto v0.0.0-20190927123631-a832865fa7ad/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20191117063200-497ca9f6d64f h1:kz4KIr+xcPUsI3VMoqWfPMvtnJ6MGfiVwsWSVzphMO4=
-golang.org/x/crypto v0.0.0-20191117063200-497ca9f6d64f/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 h1:ULYEB3JvPRE/IfO+9uO7vKV/xzVTO7XPAwm8xbf4w2g=
+golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
 golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4=
@@ -612,6 +613,7 @@ golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU
 golang.org/x/mobile v0.0.0-20190814143026-e8b3e6111d02/go.mod h1:z5wpDCy2wbnXyFdvEuY3LhY9gBUL86/IOILm+Hsjx+E=
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
 golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -701,9 +703,10 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw
 golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
 golang.org/x/tools v0.0.0-20190808195139-e713427fea3f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190820033707-85edb9ef3283/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190910221609-7f5965fd7709 h1:2Ep+/X9v6ij0U1YP++QCLyZgWQHUwVJZkC6tSrH1Iuw=
-golang.org/x/tools v0.0.0-20190910221609-7f5965fd7709/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191213221258-04c2e8eff935 h1:kJQZhwFzSwJS2BxboKjdZzWczQOZx8VuH7Y8hhuGUtM=
+golang.org/x/tools v0.0.0-20191213221258-04c2e8eff935/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=

integrations/git_helper_for_declarative_test.go

@@ -87,6 +87,12 @@ func onGiteaRun(t *testing.T, callback func(*testing.T, *url.URL), prepare ...bo
 	u, err := url.Parse(setting.AppURL)
 	assert.NoError(t, err)
 	listener, err := net.Listen("tcp", u.Host)
+	i := 0
+	for err != nil && i <= 10 {
+		time.Sleep(100 * time.Millisecond)
+		listener, err = net.Listen("tcp", u.Host)
+		i++
+	}
 	assert.NoError(t, err)
 	u.Host = listener.Addr().String()
 

integrations/gpg_git_test.go

@@ -10,7 +10,6 @@ import (
 	"io/ioutil"
 	"net/url"
 	"os"
-	"path/filepath"
 	"testing"
 
 	"code.gitea.io/gitea/models"
@@ -40,8 +39,11 @@ func TestGPGGit(t *testing.T) {
 	defer os.Setenv("GNUPGHOME", oldGNUPGHome)
 
 	// Need to create a root key
-	rootKeyPair, err := createGPGKey(tmpDir, "gitea", "gitea@fake.local")
+	rootKeyPair, err := importTestingKey(tmpDir, "gitea", "gitea@fake.local")
 	assert.NoError(t, err)
+	if err != nil {
+		assert.FailNow(t, "Unable to import rootKeyPair")
+	}
 
 	rootKeyID := rootKeyPair.PrimaryKey.KeyIdShortString()
 
@@ -125,6 +127,11 @@ func TestGPGGit(t *testing.T) {
 			testCtx := NewAPITestContext(t, username, "initial-unsigned")
 			t.Run("CreateCRUDFile-Always", crudActionCreateFile(
 				t, testCtx, user, "master", "always", "signed-always.txt", func(t *testing.T, response api.FileResponse) {
+					assert.NotNil(t, response.Verification)
+					if response.Verification == nil {
+						assert.FailNow(t, "no verification provided with response! %v", response)
+						return
+					}
 					assert.True(t, response.Verification.Verified)
 					if !response.Verification.Verified {
 						t.FailNow()
@@ -134,6 +141,11 @@ func TestGPGGit(t *testing.T) {
 				}))
 			t.Run("CreateCRUDFile-ParentSigned-always", crudActionCreateFile(
 				t, testCtx, user, "parentsigned", "parentsigned-always", "signed-parent2.txt", func(t *testing.T, response api.FileResponse) {
+					assert.NotNil(t, response.Verification)
+					if response.Verification == nil {
+						assert.FailNow(t, "no verification provided with response! %v", response)
+						return
+					}
 					assert.True(t, response.Verification.Verified)
 					if !response.Verification.Verified {
 						t.FailNow()
@@ -152,6 +164,11 @@ func TestGPGGit(t *testing.T) {
 			testCtx := NewAPITestContext(t, username, "initial-unsigned")
 			t.Run("CreateCRUDFile-Always-ParentSigned", crudActionCreateFile(
 				t, testCtx, user, "always", "always-parentsigned", "signed-always-parentsigned.txt", func(t *testing.T, response api.FileResponse) {
+					assert.NotNil(t, response.Verification)
+					if response.Verification == nil {
+						assert.FailNow(t, "no verification provided with response! %v", response)
+						return
+					}
 					assert.True(t, response.Verification.Verified)
 					if !response.Verification.Verified {
 						t.FailNow()
@@ -171,7 +188,15 @@ func TestGPGGit(t *testing.T) {
 			t.Run("CreateRepository", doAPICreateRepository(testCtx, false))
 			t.Run("CheckMasterBranchSigned", doAPIGetBranch(testCtx, "master", func(t *testing.T, branch api.Branch) {
 				assert.NotNil(t, branch.Commit)
+				if branch.Commit == nil {
+					assert.FailNow(t, "no commit provided with branch! %v", branch)
+					return
+				}
 				assert.NotNil(t, branch.Commit.Verification)
+				if branch.Commit.Verification == nil {
+					assert.FailNow(t, "no verification provided with branch commit! %v", branch.Commit)
+					return
+				}
 				assert.True(t, branch.Commit.Verification.Verified)
 				if !branch.Commit.Verification.Verified {
 					t.FailNow()
@@ -318,43 +343,26 @@ func crudActionCreateFile(t *testing.T, ctx APITestContext, user *models.User, f
 	}, callback...)
 }
 
-func createGPGKey(tmpDir, name, email string) (*openpgp.Entity, error) {
-	keyPair, err := openpgp.NewEntity(name, "test", email, nil)
+func importTestingKey(tmpDir, name, email string) (*openpgp.Entity, error) {
+	if _, _, err := process.GetManager().Exec("gpg --import integrations/private-testing.key", "gpg", "--import", "integrations/private-testing.key"); err != nil {
+		return nil, err
+	}
+	keyringFile, err := os.Open("integrations/private-testing.key")
+	if err != nil {
+		return nil, err
+	}
+	defer keyringFile.Close()
+
+	block, err := armor.Decode(keyringFile)
 	if err != nil {
 		return nil, err
 	}
 
-	for _, id := range keyPair.Identities {
-		err := id.SelfSignature.SignUserId(id.UserId.Id, keyPair.PrimaryKey, keyPair.PrivateKey, nil)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	keyFile := filepath.Join(tmpDir, "temporary.key")
-	keyWriter, err := os.Create(keyFile)
+	keyring, err := openpgp.ReadKeyRing(block.Body)
 	if err != nil {
-		return nil, err
-	}
-	defer keyWriter.Close()
-	defer os.Remove(keyFile)
-
-	w, err := armor.Encode(keyWriter, openpgp.PrivateKeyType, nil)
-	if err != nil {
-		return nil, err
-	}
-	defer w.Close()
-
-	keyPair.SerializePrivate(w, nil)
-	if err := w.Close(); err != nil {
-		return nil, err
-	}
-	if err := keyWriter.Close(); err != nil {
-		return nil, err
+		return nil, fmt.Errorf("Keyring access failed: '%v'", err)
 	}
 
-	if _, _, err := process.GetManager().Exec("gpg --import temporary.key", "gpg", "--import", keyFile); err != nil {
-		return nil, err
-	}
-	return keyPair, nil
+	// There should only be one entity in this file.
+	return keyring[0], nil
 }

integrations/private-testing.key

@@ -0,0 +1,81 @@
+-----BEGIN PGP PRIVATE KEY BLOCK-----
+
+lQVYBF3190cBDADpfh1CvNtLl8N6lJQ03jymeE8h2ocvT61stAj31eefiOvZPDyx
+n0kRztBcrQ1OITBEslYqKiYHVfVptscIWf+5bARoHnLQIgpJEio00ggqO3AP8lIZ
+uiy9/ARDVgyPl3WgMza/J3Z7W0sBJTtN/6W35i+eNEY4Q0mScmNIVc75oo5ey/pL
+JSF0qumiy94o38dy6Vk2dPEf5LyxJWoA4dLvj49LL/QGPqnsAXAQXr1zULaRuwf2
+a8oKebr3m9wVkZcaH86duUmkU822k6OZSSxWCxZFFGVkC/VaA0uUjIL4a6SQqOOr
+PPOttmCLXJrtvpFdRwJ4PNwr9r2nPDDLwoajLJ8zo5jRumzwvkK9vALkY0BPuVoC
+qFdGY+2SGYKa4FTE7Mri/5j0NhWplhbcdGRe5doGDxJHbIN2UO21XjDlTBcscKep
+mWPVE2cdJrspYaZ9O0L6vhMVwGyyk+Qxmf6NbDw0q63AtqVe9qwbr2O3Irxseftw
+uWuSuLXzp+SMh/0AEQEAAQAL/2ExopuDwuNSHsh5bcIeGnAPV51Xentqtt2viaYk
+0AB8PfTVGsyzafa0OM7DKG0z6oRGGhD+L4tRMFGbiHlFAWqdeK4gsplJ+i8VlSUc
+otJ1oH262IsmEPban6mp+ZuSKCASAYGLu0m5JF0rMucSeli1RHAeAXbtJ4SDAin7
+sib/EDWMwjkikS0f8hZWt7kbAcqnMQA2qKKmlBdHZDtOxX/8KeFZ6kHpNtFrfcsK
+rOECIaVDDhr5HobCyl3E7tW5nrlrvSUkLVFl0IjcypqfzDlZp04PMdswhkdfBhu+
+0iY4K+d4uMPMzcpF1+mcn8C+7XK7jOqZysQa42bqgFHWEqljjJiUCuXfHbxnZWls
+0R2j9FLgTqtPQ33f3zMjhOyvdiy1DmfzU9MSu/I0VqCJnq6AwlW5rBQaKwAQuHMB
+UJ7bjMx/z41z41v0IFpxHnwSa+tkl49tV+y8zVtajfwXxJNy8j/ElX0ywfM5sDHa
+RAVwI7DSwMk5azp3F15DnA6XbwYA8O0b5AIeCo8edmIdKgY3vAi20j/lsTgsTUkY
+GTQ4BdMohr9gpZWHZZmQ1TeZokm4Auex7UgPblflufepkADassXixMmSNUsggGI+
+sR9qydNCw+qzgaJjchpwT5TdLJNHRbE+6VuGXJftcjdfXiKYZltEQBX8U4w7hui8
+D6dpzJK5mE1QebrFnJ7IKpAe+hWTc1+g9iHH3rInPMIzQW72WqSKndKIrRy1PZS5
+WM5MJzgWQaDzZSOQhrKA4yLIyzsrBgD4GfFLWh+sQ02sob5xmpDblfQUYVRy5TAx
+4WOLSflJqqyarrk7s1D7obqsSoAEdJk521dpE/0ciI5xT41fQKMXH1Qm9tu9uW5d
+1Y3oDxQXFJFa34gi5J9UbUBBIJRU0KyFcB1mGVF+fKbAKGPFR2lMCmkeqAYjVohM
+PG+tluArQrQYCwkZroR460TqvSadmPUekEjYsIzwlaOkJhGf7r40G5Djgyb2/LoC
+JY28zH7P9MXxIc7WAWuMJniUOqvslXcGAOkfZ1KVI61AIAvkEoRUpKwNSofs2PDQ
+1K5Q9DN0NK5UNAAr+Wn91mw/MBXqxdheq9wjmcsvx8OAhvw7O89QMuTviCTUQjSl
+Wzel6gpoZhpOgVb2RTxV7yVrp2fgYKkeUr7hiGhSxw78guF2jLgfBgb1ef+XKIMk
+5anUqKcsHHiouBQbcUCDyKBcVeIUKjAuh9ADpqn1v1oVshugnjpx32Oq1AW6Mn9e
+SmxBoR7YIvsy79P2IonjixEAjSp1chkGpNQTtBhnaXRlYSA8Z2l0ZWFAZmFrZS5s
+b2NhbD6JAdQEEwEKAD4WIQQ4G/p4KVUOUEVu5g5R68KXFICqDwUCXfX3RwIbAwUJ
+A8JnAAULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRBR68KXFICqD/8/C/4wYdr1
+Q6fnXiAdBZPQDOHUjCSjH1d6tbVYaCz0ku8rK1uU0oTToKRSTXH9K59erHl//MEX
+Rte6loI22y3GFA8wAWzFKGwb6wDUr0SkH21espsp+plKUI/gHV8JyfWs0pLmy/C0
+tOr9XhdnQepFjpDAquWszSO0B70G32Tl82Tzbrho6+ePvU+2X0fYj1F8q/2bmegB
+lL1CcdVuivBqYglj6tzlurPXFq1QenJdssZNn0fizGiGfTY/7kgrvKHc4KN03i9d
+PUrPMQw7J59KSFNdkE3KYdedmEeWBVmrbfC8QBEO1zcTJN9wwV8fVv4qOhKN8yIO
+QLuhBZTeChtP3i2FCPHQqbeD2f0SG+yBbWu/OyfSC2YHcGyjbNV8D9upNg5eIJ34
+Sm5i0tGUYEdq9QQROacXn2/MhyJuJYbFrTcsHLsSiwygUXvHOqi0G0gEjWl67yMd
+9YIq1sZlNs3WY7ASrV+QcB7B9WKJAyh5YWz/G4MlThU91YUfltAb3QmNFeadBVgE
+XfX3RwEMALH7pae22J/BmhFFBCjMld166hTknXjOR459csv5ppky91Yl82VFFg+J
+G6hbkGzvCQ5XDJu8xgWeXowe2sXkyDwhTRaB6MEnA4KW5PUazL7KlDGsR5oPvBlE
+dSQDGzTV/RPcszSNdcN9MRNbfAf0ZFV6D9R3CIlNZAm6HwML7lZ0JmCiLORz3TbF
+4kg1KDZIQAhY7Y7AuMdoXfnpUqFLba2ZxZBvdcrMcuYz8GkmFsYdi1/JuXEK3//B
+Mo7Pg78zsq7UolUcT2p3qKb7hB3CEtwa3xffwzgAcFSKYrCE/5/IYjHhS97uKWor
+8dh59wUCuPCmAiuIz3aD84rxZIHgBGPy03TEWCrCBCVxAdH/2Ezpn3DpuZyCuanJ
+0WGSzrPBw+twA8bk9BATvFVQ/7Bs9deSsMAOI1uj9lTy1R9LU/KHEr8BEz61+Bgk
++m4ev6OQAVDY6QpRtf+zfB2xO95Wu4l1pIFuz7OJaZLCm2ApeAKsCCUDdTSJn5e7
+0i1E4SIVgwARAQABAAv/WHaZqbiqBw21RCwnmxfEzWbQfj37PxZYXqxfqJ6XfcHl
+Sb5nMcia5HHje1S3fk15FNWTgLzdN+G1YLPdTUsfczOiGzPKumZnyjqx5lnBtnr+
+GYpltF9pwK1UA+g/V42c0oh50f8Vr2rEP7jS9ykzzYBz6ciYR5ZdyK/nxh3iArqM
+cK9q3MnyA81rYTR6njBfE0cQHEoSDZsESrj7xwu0ofqyRc4AoCHqYh0iu0ChRSte
+IOgk8djT6Uzfkjf2ZcyNiD2/iFzAXaI8CpoiiRJDn/qIhtSFqjb284wwbmTUE+Nw
+LjeMbpKQiWqnsw2GKhlXVvTLjrCb8TIKjbLtFH2HlEaIjL332GcVqkVy2TMtjZRi
+lhy/uSY2kzkBkoGJXp5isJFk3ZcOHHsG4VQ+08vq++GoqQE8U1t8zMfAbBFoFlpP
+nkRjZs0MwY9u6C0IiXRDrYrMIW12LjsRBiebGHUhzv10/4T0XZ8FKTewxcszcuMf
+lpbIotF1ItGqCXqgufnhBgDBhi8BErxO59ksWLAHdozDyiXwQ5ua6WesGrIRvw23
+Z8a5wxTByXmd1fMgJ509hpXbxUC94TGObJoUo23YE3TpqlTLs5NqeFzzU+OjWSb0
+Wo1hpFlzyatuynpz2aXbKbjw5dgyeIxj9t+NhGo2SW6v+RHtYAWJRFBFOPVOLwTy
+an733pA3MSUT0oEh+aggDkXEJLBum0P3Onnma7wR7Xj2Nk1SGLCMgVmKbtGlvyrj
+yc5FZhzuvOfeuLSYoa0KE4sGAOtxED2jDkV2HS67bxrUzMLvAAhOnRmunA7Qk4F7
+B2uMYa03O7vnUJAINpmZVu/ubWz1/JRV6M3/1lQTH+2B9kZ5v6kHrczCsoSP2dXD
+7CQnxSm6zngdgwkoo+9pgFztGUZM071SjRW+r1IwE/XBZNwFya5PM02/Akb0ejuB
+6K2ClnIFf7gflndUZ0mhZn48I88b6mzEG4X4uUZG+4vW8EZEInl+nMA9f3S6YT0U
+ZG4JC8JMKsmoYLye/BuedHxk6QX6AnMFBjK7cnfBnViJkmXhDLxmcCjwjUUBraRI
+QbyzHzY2Jq1VyhTJ1HZxE+vj26MzFFzjpe84r1Ggrcowx53RHstBBYBA5OjRy+cN
+vDzqqWz4cDKU/XlwJhRnG+PcY3c47obpvjjagcwG7xU4df15fDetKajnIloA5r22
+hbmVmTAqljyWLnvSNYrvf5QDqqg6tBuHITUiZhYgECpIoeEj9hU8MZSvQOscK0kx
+Vn8SqUjxDcNazQM8NoxNB10wfJw63hCJAbwEGAEKACYWIQQ4G/p4KVUOUEVu5g5R
+68KXFICqDwUCXfX3RwIbDAUJA8JnAAAKCRBR68KXFICqD54+DAC4VZpKrU6Oo04z
+/gJeC+3fNon6W9Pdxx7KimDOttkpCiss8JydO6aSB97xrWdvMBTui333qGo2exE/
+XFA4RF7K4lAKUWbwaR1brLQfGVYOltmMb986/LeE3OsmMt4vbxUnGvHVX+QXDWAr
+p6q4DZvMgQQhbWp+rMjXtRr10iQnSlM5CYhyawdiiahFqgoo8395l/2JA2YGhUgU
+nARUPZ9SqaUmRm+KGsSyoYnvN9apiDk5KVQoyfrmweNN7DCIIcoh/B9Ax8nmouKz
+yBB2fjCM/bJNtN/AsgYbZIScuYK/xqTkwNtbe5WdCyD/QJOHTsPJzx59hgSVo6gf
+Fe8VBnxHtrY8gPSUU3gkhYLvLzyVX+YLNzRcffobd8gJbfumwFJUkz91oGvYz7xg
+XN2qmsgBNCbTIzWZMpRDMAbY+n2QFImGf+EJZlMdj6gOrIYq8N4+nMW1FwJivsOb
+muqySyjZnD2AYjEA6OYPXfCVhaB5fTfhQXbIrZbgsEh4ob/eIdM=
+=oSDR
+-----END PGP PRIVATE KEY BLOCK-----

vendor/golang.org/x/crypto/acme/jws.go

@@ -11,6 +11,7 @@ import (
 	"crypto/rsa"
 	"crypto/sha256"
 	_ "crypto/sha512" // need for EC keys
+	"encoding/asn1"
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
@@ -126,21 +127,23 @@ func jwkEncode(pub crypto.PublicKey) (string, error) {
 
 // jwsSign signs the digest using the given key.
 // The hash is unused for ECDSA keys.
-//
-// Note: non-stdlib crypto.Signer implementations are expected to return
-// the signature in the format as specified in RFC7518.
-// See https://tools.ietf.org/html/rfc7518 for more details.
 func jwsSign(key crypto.Signer, hash crypto.Hash, digest []byte) ([]byte, error) {
-	if key, ok := key.(*ecdsa.PrivateKey); ok {
-		// The key.Sign method of ecdsa returns ASN1-encoded signature.
-		// So, we use the package Sign function instead
-		// to get R and S values directly and format the result accordingly.
-		r, s, err := ecdsa.Sign(rand.Reader, key, digest)
+	switch pub := key.Public().(type) {
+	case *rsa.PublicKey:
+		return key.Sign(rand.Reader, digest, hash)
+	case *ecdsa.PublicKey:
+		sigASN1, err := key.Sign(rand.Reader, digest, hash)
 		if err != nil {
 			return nil, err
 		}
-		rb, sb := r.Bytes(), s.Bytes()
-		size := key.Params().BitSize / 8
+
+		var rs struct{ R, S *big.Int }
+		if _, err := asn1.Unmarshal(sigASN1, &rs); err != nil {
+			return nil, err
+		}
+
+		rb, sb := rs.R.Bytes(), rs.S.Bytes()
+		size := pub.Params().BitSize / 8
 		if size%8 > 0 {
 			size++
 		}
@@ -149,7 +152,7 @@ func jwsSign(key crypto.Signer, hash crypto.Hash, digest []byte) ([]byte, error)
 		copy(sig[size*2-len(sb):], sb)
 		return sig, nil
 	}
-	return key.Sign(rand.Reader, digest, hash)
+	return nil, ErrUnsupportedKey
 }
 
 // jwsHasher indicates suitable JWS algorithm name and a hash function

vendor/golang.org/x/crypto/ssh/certs.go

@@ -17,12 +17,14 @@ import (
 // These constants from [PROTOCOL.certkeys] represent the algorithm names
 // for certificate types supported by this package.
 const (
-	CertAlgoRSAv01      = "ssh-rsa-cert-v01@openssh.com"
-	CertAlgoDSAv01      = "ssh-dss-cert-v01@openssh.com"
-	CertAlgoECDSA256v01 = "ecdsa-sha2-nistp256-cert-v01@openssh.com"
-	CertAlgoECDSA384v01 = "ecdsa-sha2-nistp384-cert-v01@openssh.com"
-	CertAlgoECDSA521v01 = "ecdsa-sha2-nistp521-cert-v01@openssh.com"
-	CertAlgoED25519v01  = "ssh-ed25519-cert-v01@openssh.com"
+	CertAlgoRSAv01        = "ssh-rsa-cert-v01@openssh.com"
+	CertAlgoDSAv01        = "ssh-dss-cert-v01@openssh.com"
+	CertAlgoECDSA256v01   = "ecdsa-sha2-nistp256-cert-v01@openssh.com"
+	CertAlgoECDSA384v01   = "ecdsa-sha2-nistp384-cert-v01@openssh.com"
+	CertAlgoECDSA521v01   = "ecdsa-sha2-nistp521-cert-v01@openssh.com"
+	CertAlgoSKECDSA256v01 = "sk-ecdsa-sha2-nistp256-cert-v01@openssh.com"
+	CertAlgoED25519v01    = "ssh-ed25519-cert-v01@openssh.com"
+	CertAlgoSKED25519v01  = "sk-ssh-ed25519-cert-v01@openssh.com"
 )
 
 // Certificate types distinguish between host and user
@@ -37,6 +39,7 @@ const (
 type Signature struct {
 	Format string
 	Blob   []byte
+	Rest   []byte `ssh:"rest"`
 }
 
 // CertTimeInfinity can be used for OpenSSHCertV01.ValidBefore to indicate that
@@ -429,12 +432,14 @@ func (c *Certificate) SignCert(rand io.Reader, authority Signer) error {
 }
 
 var certAlgoNames = map[string]string{
-	KeyAlgoRSA:      CertAlgoRSAv01,
-	KeyAlgoDSA:      CertAlgoDSAv01,
-	KeyAlgoECDSA256: CertAlgoECDSA256v01,
-	KeyAlgoECDSA384: CertAlgoECDSA384v01,
-	KeyAlgoECDSA521: CertAlgoECDSA521v01,
-	KeyAlgoED25519:  CertAlgoED25519v01,
+	KeyAlgoRSA:        CertAlgoRSAv01,
+	KeyAlgoDSA:        CertAlgoDSAv01,
+	KeyAlgoECDSA256:   CertAlgoECDSA256v01,
+	KeyAlgoECDSA384:   CertAlgoECDSA384v01,
+	KeyAlgoECDSA521:   CertAlgoECDSA521v01,
+	KeyAlgoSKECDSA256: CertAlgoSKECDSA256v01,
+	KeyAlgoED25519:    CertAlgoED25519v01,
+	KeyAlgoSKED25519:  CertAlgoSKED25519v01,
 }
 
 // certToPrivAlgo returns the underlying algorithm for a certificate algorithm.
@@ -518,6 +523,12 @@ func parseSignatureBody(in []byte) (out *Signature, rest []byte, ok bool) {
 		return
 	}
 
+	switch out.Format {
+	case KeyAlgoSKECDSA256, CertAlgoSKECDSA256v01, KeyAlgoSKED25519, CertAlgoSKED25519v01:
+		out.Rest = in
+		return out, nil, ok
+	}
+
 	return out, in, ok
 }
 

vendor/golang.org/x/crypto/ssh/keys.go

@@ -30,12 +30,14 @@ import (
 // These constants represent the algorithm names for key types supported by this
 // package.
 const (
-	KeyAlgoRSA      = "ssh-rsa"
-	KeyAlgoDSA      = "ssh-dss"
-	KeyAlgoECDSA256 = "ecdsa-sha2-nistp256"
-	KeyAlgoECDSA384 = "ecdsa-sha2-nistp384"
-	KeyAlgoECDSA521 = "ecdsa-sha2-nistp521"
-	KeyAlgoED25519  = "ssh-ed25519"
+	KeyAlgoRSA        = "ssh-rsa"
+	KeyAlgoDSA        = "ssh-dss"
+	KeyAlgoECDSA256   = "ecdsa-sha2-nistp256"
+	KeyAlgoSKECDSA256 = "sk-ecdsa-sha2-nistp256@openssh.com"
+	KeyAlgoECDSA384   = "ecdsa-sha2-nistp384"
+	KeyAlgoECDSA521   = "ecdsa-sha2-nistp521"
+	KeyAlgoED25519    = "ssh-ed25519"
+	KeyAlgoSKED25519  = "sk-ssh-ed25519@openssh.com"
 )
 
 // These constants represent non-default signature algorithms that are supported
@@ -58,9 +60,13 @@ func parsePubKey(in []byte, algo string) (pubKey PublicKey, rest []byte, err err
 		return parseDSA(in)
 	case KeyAlgoECDSA256, KeyAlgoECDSA384, KeyAlgoECDSA521:
 		return parseECDSA(in)
+	case KeyAlgoSKECDSA256:
+		return parseSKECDSA(in)
 	case KeyAlgoED25519:
 		return parseED25519(in)
-	case CertAlgoRSAv01, CertAlgoDSAv01, CertAlgoECDSA256v01, CertAlgoECDSA384v01, CertAlgoECDSA521v01, CertAlgoED25519v01:
+	case KeyAlgoSKED25519:
+		return parseSKEd25519(in)
+	case CertAlgoRSAv01, CertAlgoDSAv01, CertAlgoECDSA256v01, CertAlgoECDSA384v01, CertAlgoECDSA521v01, CertAlgoSKECDSA256v01, CertAlgoED25519v01, CertAlgoSKED25519v01:
 		cert, err := parseCert(in, certToPrivAlgo(algo))
 		if err != nil {
 			return nil, nil, err
@@ -685,6 +691,218 @@ func (k *ecdsaPublicKey) CryptoPublicKey() crypto.PublicKey {
 	return (*ecdsa.PublicKey)(k)
 }
 
+// skFields holds the additional fields present in U2F/FIDO2 signatures.
+// See openssh/PROTOCOL.u2f 'SSH U2F Signatures' for details.
+type skFields struct {
+	// Flags contains U2F/FIDO2 flags such as 'user present'
+	Flags byte
+	// Counter is a monotonic signature counter which can be
+	// used to detect concurrent use of a private key, should
+	// it be extracted from hardware.
+	Counter uint32
+}
+
+type skECDSAPublicKey struct {
+	// application is a URL-like string, typically "ssh:" for SSH.
+	// see openssh/PROTOCOL.u2f for details.
+	application string
+	ecdsa.PublicKey
+}
+
+func (k *skECDSAPublicKey) Type() string {
+	return KeyAlgoSKECDSA256
+}
+
+func (k *skECDSAPublicKey) nistID() string {
+	return "nistp256"
+}
+
+func parseSKECDSA(in []byte) (out PublicKey, rest []byte, err error) {
+	var w struct {
+		Curve       string
+		KeyBytes    []byte
+		Application string
+		Rest        []byte `ssh:"rest"`
+	}
+
+	if err := Unmarshal(in, &w); err != nil {
+		return nil, nil, err
+	}
+
+	key := new(skECDSAPublicKey)
+	key.application = w.Application
+
+	if w.Curve != "nistp256" {
+		return nil, nil, errors.New("ssh: unsupported curve")
+	}
+	key.Curve = elliptic.P256()
+
+	key.X, key.Y = elliptic.Unmarshal(key.Curve, w.KeyBytes)
+	if key.X == nil || key.Y == nil {
+		return nil, nil, errors.New("ssh: invalid curve point")
+	}
+
+	return key, w.Rest, nil
+}
+
+func (k *skECDSAPublicKey) Marshal() []byte {
+	// See RFC 5656, section 3.1.
+	keyBytes := elliptic.Marshal(k.Curve, k.X, k.Y)
+	w := struct {
+		Name        string
+		ID          string
+		Key         []byte
+		Application string
+	}{
+		k.Type(),
+		k.nistID(),
+		keyBytes,
+		k.application,
+	}
+
+	return Marshal(&w)
+}
+
+func (k *skECDSAPublicKey) Verify(data []byte, sig *Signature) error {
+	if sig.Format != k.Type() {
+		return fmt.Errorf("ssh: signature type %s for key type %s", sig.Format, k.Type())
+	}
+
+	h := ecHash(k.Curve).New()
+	h.Write([]byte(k.application))
+	appDigest := h.Sum(nil)
+
+	h.Reset()
+	h.Write(data)
+	dataDigest := h.Sum(nil)
+
+	var ecSig struct {
+		R *big.Int
+		S *big.Int
+	}
+	if err := Unmarshal(sig.Blob, &ecSig); err != nil {
+		return err
+	}
+
+	var skf skFields
+	if err := Unmarshal(sig.Rest, &skf); err != nil {
+		return err
+	}
+
+	blob := struct {
+		ApplicationDigest []byte `ssh:"rest"`
+		Flags             byte
+		Counter           uint32
+		MessageDigest     []byte `ssh:"rest"`
+	}{
+		appDigest,
+		skf.Flags,
+		skf.Counter,
+		dataDigest,
+	}
+
+	original := Marshal(blob)
+
+	h.Reset()
+	h.Write(original)
+	digest := h.Sum(nil)
+
+	if ecdsa.Verify((*ecdsa.PublicKey)(&k.PublicKey), digest, ecSig.R, ecSig.S) {
+		return nil
+	}
+	return errors.New("ssh: signature did not verify")
+}
+
+type skEd25519PublicKey struct {
+	// application is a URL-like string, typically "ssh:" for SSH.
+	// see openssh/PROTOCOL.u2f for details.
+	application string
+	ed25519.PublicKey
+}
+
+func (k *skEd25519PublicKey) Type() string {
+	return KeyAlgoSKED25519
+}
+
+func parseSKEd25519(in []byte) (out PublicKey, rest []byte, err error) {
+	var w struct {
+		KeyBytes    []byte
+		Application string
+		Rest        []byte `ssh:"rest"`
+	}
+
+	if err := Unmarshal(in, &w); err != nil {
+		return nil, nil, err
+	}
+
+	key := new(skEd25519PublicKey)
+	key.application = w.Application
+	key.PublicKey = ed25519.PublicKey(w.KeyBytes)
+
+	return key, w.Rest, nil
+}
+
+func (k *skEd25519PublicKey) Marshal() []byte {
+	w := struct {
+		Name        string
+		KeyBytes    []byte
+		Application string
+	}{
+		KeyAlgoSKED25519,
+		[]byte(k.PublicKey),
+		k.application,
+	}
+	return Marshal(&w)
+}
+
+func (k *skEd25519PublicKey) Verify(data []byte, sig *Signature) error {
+	if sig.Format != k.Type() {
+		return fmt.Errorf("ssh: signature type %s for key type %s", sig.Format, k.Type())
+	}
+
+	h := sha256.New()
+	h.Write([]byte(k.application))
+	appDigest := h.Sum(nil)
+
+	h.Reset()
+	h.Write(data)
+	dataDigest := h.Sum(nil)
+
+	var edSig struct {
+		Signature []byte `ssh:"rest"`
+	}
+
+	if err := Unmarshal(sig.Blob, &edSig); err != nil {
+		return err
+	}
+
+	var skf skFields
+	if err := Unmarshal(sig.Rest, &skf); err != nil {
+		return err
+	}
+
+	blob := struct {
+		ApplicationDigest []byte `ssh:"rest"`
+		Flags             byte
+		Counter           uint32
+		MessageDigest     []byte `ssh:"rest"`
+	}{
+		appDigest,
+		skf.Flags,
+		skf.Counter,
+		dataDigest,
+	}
+
+	original := Marshal(blob)
+
+	edKey := (ed25519.PublicKey)(k.PublicKey)
+	if ok := ed25519.Verify(edKey, original, edSig.Signature); !ok {
+		return errors.New("ssh: signature did not verify")
+	}
+
+	return nil
+}
+
 // NewSignerFromKey takes an *rsa.PrivateKey, *dsa.PrivateKey,
 // *ecdsa.PrivateKey or any other crypto.Signer and returns a
 // corresponding Signer instance. ECDSA keys must use P-256, P-384 or
@@ -837,7 +1055,8 @@ func NewPublicKey(key interface{}) (PublicKey, error) {
 }
 
 // ParsePrivateKey returns a Signer from a PEM encoded private key. It supports
-// the same keys as ParseRawPrivateKey.
+// the same keys as ParseRawPrivateKey. If the private key is encrypted, it
+// will return a PassphraseMissingError.
 func ParsePrivateKey(pemBytes []byte) (Signer, error) {
 	key, err := ParseRawPrivateKey(pemBytes)
 	if err != nil {
@@ -850,8 +1069,8 @@ func ParsePrivateKey(pemBytes []byte) (Signer, error) {
 // ParsePrivateKeyWithPassphrase returns a Signer from a PEM encoded private
 // key and passphrase. It supports the same keys as
 // ParseRawPrivateKeyWithPassphrase.
-func ParsePrivateKeyWithPassphrase(pemBytes, passPhrase []byte) (Signer, error) {
-	key, err := ParseRawPrivateKeyWithPassphrase(pemBytes, passPhrase)
+func ParsePrivateKeyWithPassphrase(pemBytes, passphrase []byte) (Signer, error) {
+	key, err := ParseRawPrivateKeyWithPassphrase(pemBytes, passphrase)
 	if err != nil {
 		return nil, err
 	}
@@ -867,8 +1086,21 @@ func encryptedBlock(block *pem.Block) bool {
 	return strings.Contains(block.Headers["Proc-Type"], "ENCRYPTED")
 }
 
+// A PassphraseMissingError indicates that parsing this private key requires a
+// passphrase. Use ParsePrivateKeyWithPassphrase.
+type PassphraseMissingError struct {
+	// PublicKey will be set if the private key format includes an unencrypted
+	// public key along with the encrypted private key.
+	PublicKey PublicKey
+}
+
+func (*PassphraseMissingError) Error() string {
+	return "ssh: this private key is passphrase protected"
+}
+
 // ParseRawPrivateKey returns a private key from a PEM encoded private key. It
-// supports RSA (PKCS#1), PKCS#8, DSA (OpenSSL), and ECDSA private keys.
+// supports RSA (PKCS#1), PKCS#8, DSA (OpenSSL), and ECDSA private keys. If the
+// private key is encrypted, it will return a PassphraseMissingError.
 func ParseRawPrivateKey(pemBytes []byte) (interface{}, error) {
 	block, _ := pem.Decode(pemBytes)
 	if block == nil {
@@ -876,7 +1108,7 @@ func ParseRawPrivateKey(pemBytes []byte) (interface{}, error) {
 	}
 
 	if encryptedBlock(block) {
-		return nil, errors.New("ssh: cannot decode encrypted private keys")
+		return nil, &PassphraseMissingError{}
 	}
 
 	switch block.Type {
@@ -899,24 +1131,22 @@ func ParseRawPrivateKey(pemBytes []byte) (interface{}, error) {
 // ParseRawPrivateKeyWithPassphrase returns a private key decrypted with
 // passphrase from a PEM encoded private key. If wrong passphrase, return
 // x509.IncorrectPasswordError.
-func ParseRawPrivateKeyWithPassphrase(pemBytes, passPhrase []byte) (interface{}, error) {
+func ParseRawPrivateKeyWithPassphrase(pemBytes, passphrase []byte) (interface{}, error) {
 	block, _ := pem.Decode(pemBytes)
 	if block == nil {
 		return nil, errors.New("ssh: no key found")
 	}
-	buf := block.Bytes
 
-	if encryptedBlock(block) {
-		if x509.IsEncryptedPEMBlock(block) {
-			var err error
-			buf, err = x509.DecryptPEMBlock(block, passPhrase)
-			if err != nil {
-				if err == x509.IncorrectPasswordError {
-					return nil, err
-				}
-				return nil, fmt.Errorf("ssh: cannot decode encrypted private keys: %v", err)
-			}
+	if !encryptedBlock(block) || !x509.IsEncryptedPEMBlock(block) {
+		return nil, errors.New("ssh: not an encrypted key")
+	}
+
+	buf, err := x509.DecryptPEMBlock(block, passphrase)
+	if err != nil {
+		if err == x509.IncorrectPasswordError {
+			return nil, err
 		}
+		return nil, fmt.Errorf("ssh: cannot decode encrypted private keys: %v", err)
 	}
 
 	switch block.Type {
@@ -926,8 +1156,6 @@ func ParseRawPrivateKeyWithPassphrase(pemBytes, passPhrase []byte) (interface{},
 		return x509.ParseECPrivateKey(buf)
 	case "DSA PRIVATE KEY":
 		return ParseDSAPrivateKey(buf)
-	case "OPENSSH PRIVATE KEY":
-		return parseOpenSSHPrivateKey(buf)
 	default:
 		return nil, fmt.Errorf("ssh: unsupported key type %q", block.Type)
 	}

vendor/golang.org/x/crypto/ssh/server.go

@@ -284,8 +284,8 @@ func (s *connection) serverHandshake(config *ServerConfig) (*Permissions, error)
 
 func isAcceptableAlgo(algo string) bool {
 	switch algo {
-	case KeyAlgoRSA, KeyAlgoDSA, KeyAlgoECDSA256, KeyAlgoECDSA384, KeyAlgoECDSA521, KeyAlgoED25519,
-		CertAlgoRSAv01, CertAlgoDSAv01, CertAlgoECDSA256v01, CertAlgoECDSA384v01, CertAlgoECDSA521v01, CertAlgoED25519v01:
+	case KeyAlgoRSA, KeyAlgoDSA, KeyAlgoECDSA256, KeyAlgoECDSA384, KeyAlgoECDSA521, KeyAlgoSKECDSA256, KeyAlgoED25519, KeyAlgoSKED25519,
+		CertAlgoRSAv01, CertAlgoDSAv01, CertAlgoECDSA256v01, CertAlgoECDSA384v01, CertAlgoECDSA521v01, CertAlgoSKECDSA256v01, CertAlgoED25519v01, CertAlgoSKED25519v01:
 		return true
 	}
 	return false

vendor/golang.org/x/tools/go/buildutil/overlay.go

@@ -65,7 +65,7 @@ func OverlayContext(orig *build.Context, overlay map[string][]byte) *build.Conte
 //
 // The archive consists of a series of files. Each file consists of a
 // name, a decimal file size and the file contents, separated by
-// newlinews. No newline follows after the file contents.
+// newlines. No newline follows after the file contents.
 func ParseOverlayArchive(archive io.Reader) (map[string][]byte, error) {
 	overlay := make(map[string][]byte)
 	r := bufio.NewReader(archive)

vendor/golang.org/x/tools/go/gcexportdata/gcexportdata.go

@@ -100,7 +100,7 @@ func Read(in io.Reader, fset *token.FileSet, imports map[string]*types.Package,
 // Write writes encoded type information for the specified package to out.
 // The FileSet provides file position information for named objects.
 func Write(out io.Writer, fset *token.FileSet, pkg *types.Package) error {
-	b, err := gcimporter.BExportData(fset, pkg)
+	b, err := gcimporter.IExportData(fset, pkg)
 	if err != nil {
 		return err
 	}

vendor/golang.org/x/tools/go/internal/gcimporter/bimport.go

@@ -332,7 +332,7 @@ func (p *importer) pos() token.Pos {
 	p.prevFile = file
 	p.prevLine = line
 
-	return p.fake.pos(file, line)
+	return p.fake.pos(file, line, 0)
 }
 
 // Synthesize a token.Pos
@@ -341,7 +341,9 @@ type fakeFileSet struct {
 	files map[string]*token.File
 }
 
-func (s *fakeFileSet) pos(file string, line int) token.Pos {
+func (s *fakeFileSet) pos(file string, line, column int) token.Pos {
+	// TODO(mdempsky): Make use of column.
+
 	// Since we don't know the set of needed file positions, we
 	// reserve maxlines positions per file.
 	const maxlines = 64 * 1024

vendor/golang.org/x/tools/go/internal/gcimporter/iexport.go

@@ -6,8 +6,6 @@
 // This file was derived from $GOROOT/src/cmd/compile/internal/gc/iexport.go;
 // see that file for specification of the format.
 
-// +build go1.11
-
 package gcimporter
 
 import (
@@ -28,7 +26,10 @@ import (
 const iexportVersion = 0
 
 // IExportData returns the binary export data for pkg.
+//
 // If no file set is provided, position info will be missing.
+// The package path of the top-level package will not be recorded,
+// so that calls to IImportData can override with a provided package path.
 func IExportData(fset *token.FileSet, pkg *types.Package) (b []byte, err error) {
 	defer func() {
 		if e := recover(); e != nil {
@@ -48,6 +49,7 @@ func IExportData(fset *token.FileSet, pkg *types.Package) (b []byte, err error)
 		stringIndex: map[string]uint64{},
 		declIndex:   map[types.Object]uint64{},
 		typIndex:    map[types.Type]uint64{},
+		localpkg:    pkg,
 	}
 
 	for i, pt := range predeclared() {
@@ -73,7 +75,7 @@ func IExportData(fset *token.FileSet, pkg *types.Package) (b []byte, err error)
 	// Append indices to data0 section.
 	dataLen := uint64(p.data0.Len())
 	w := p.newWriter()
-	w.writeIndex(p.declIndex, pkg)
+	w.writeIndex(p.declIndex)
 	w.flush()
 
 	// Assemble header.
@@ -95,14 +97,14 @@ func IExportData(fset *token.FileSet, pkg *types.Package) (b []byte, err error)
 // we're writing out the main index, which is also read by
 // non-compiler tools and includes a complete package description
 // (i.e., name and height).
-func (w *exportWriter) writeIndex(index map[types.Object]uint64, localpkg *types.Package) {
+func (w *exportWriter) writeIndex(index map[types.Object]uint64) {
 	// Build a map from packages to objects from that package.
 	pkgObjs := map[*types.Package][]types.Object{}
 
 	// For the main index, make sure to include every package that
 	// we reference, even if we're not exporting (or reexporting)
 	// any symbols from it.
-	pkgObjs[localpkg] = nil
+	pkgObjs[w.p.localpkg] = nil
 	for pkg := range w.p.allPkgs {
 		pkgObjs[pkg] = nil
 	}
@@ -121,12 +123,12 @@ func (w *exportWriter) writeIndex(index map[types.Object]uint64, localpkg *types
 	}
 
 	sort.Slice(pkgs, func(i, j int) bool {
-		return pkgs[i].Path() < pkgs[j].Path()
+		return w.exportPath(pkgs[i]) < w.exportPath(pkgs[j])
 	})
 
 	w.uint64(uint64(len(pkgs)))
 	for _, pkg := range pkgs {
-		w.string(pkg.Path())
+		w.string(w.exportPath(pkg))
 		w.string(pkg.Name())
 		w.uint64(uint64(0)) // package height is not needed for go/types
 
@@ -143,6 +145,8 @@ type iexporter struct {
 	fset *token.FileSet
 	out  *bytes.Buffer
 
+	localpkg *types.Package
+
 	// allPkgs tracks all packages that have been referenced by
 	// the export data, so we can ensure to include them in the
 	// main index.
@@ -195,6 +199,13 @@ type exportWriter struct {
 	prevLine int64
 }
 
+func (w *exportWriter) exportPath(pkg *types.Package) string {
+	if pkg == w.p.localpkg {
+		return ""
+	}
+	return pkg.Path()
+}
+
 func (p *iexporter) doDecl(obj types.Object) {
 	w := p.newWriter()
 	w.setPkg(obj.Pkg(), false)
@@ -267,6 +278,11 @@ func (w *exportWriter) tag(tag byte) {
 }
 
 func (w *exportWriter) pos(pos token.Pos) {
+	if w.p.fset == nil {
+		w.int64(0)
+		return
+	}
+
 	p := w.p.fset.Position(pos)
 	file := p.Filename
 	line := int64(p.Line)
@@ -299,7 +315,7 @@ func (w *exportWriter) pkg(pkg *types.Package) {
 	// Ensure any referenced packages are declared in the main index.
 	w.p.allPkgs[pkg] = true
 
-	w.string(pkg.Path())
+	w.string(w.exportPath(pkg))
 }
 
 func (w *exportWriter) qualifiedIdent(obj types.Object) {
@@ -394,7 +410,7 @@ func (w *exportWriter) doTyp(t types.Type, pkg *types.Package) {
 			w.pos(f.Pos())
 			w.string(f.Name())
 			w.typ(f.Type(), pkg)
-			w.bool(f.Embedded())
+			w.bool(f.Anonymous())
 			w.string(t.Tag(i)) // note (or tag)
 		}
 

vendor/golang.org/x/tools/go/internal/gcimporter/iimport.go

@@ -63,8 +63,8 @@ const (
 // If the export data version is not recognized or the format is otherwise
 // compromised, an error is returned.
 func IImportData(fset *token.FileSet, imports map[string]*types.Package, data []byte, path string) (_ int, pkg *types.Package, err error) {
-	const currentVersion = 0
-	version := -1
+	const currentVersion = 1
+	version := int64(-1)
 	defer func() {
 		if e := recover(); e != nil {
 			if version > currentVersion {
@@ -77,9 +77,9 @@ func IImportData(fset *token.FileSet, imports map[string]*types.Package, data []
 
 	r := &intReader{bytes.NewReader(data), path}
 
-	version = int(r.uint64())
+	version = int64(r.uint64())
 	switch version {
-	case currentVersion:
+	case currentVersion, 0:
 	default:
 		errorf("unknown iexport format version %d", version)
 	}
@@ -93,7 +93,8 @@ func IImportData(fset *token.FileSet, imports map[string]*types.Package, data []
 	r.Seek(sLen+dLen, io.SeekCurrent)
 
 	p := iimporter{
-		ipath: path,
+		ipath:   path,
+		version: int(version),
 
 		stringData:  stringData,
 		stringCache: make(map[uint64]string),
@@ -142,20 +143,18 @@ func IImportData(fset *token.FileSet, imports map[string]*types.Package, data []
 		p.pkgIndex[pkg] = nameIndex
 		pkgList[i] = pkg
 	}
-	var localpkg *types.Package
-	for _, pkg := range pkgList {
-		if pkg.Path() == path {
-			localpkg = pkg
-		}
+	if len(pkgList) == 0 {
+		errorf("no packages found for %s", path)
+		panic("unreachable")
 	}
-
-	names := make([]string, 0, len(p.pkgIndex[localpkg]))
-	for name := range p.pkgIndex[localpkg] {
+	p.ipkg = pkgList[0]
+	names := make([]string, 0, len(p.pkgIndex[p.ipkg]))
+	for name := range p.pkgIndex[p.ipkg] {
 		names = append(names, name)
 	}
 	sort.Strings(names)
 	for _, name := range names {
-		p.doDecl(localpkg, name)
+		p.doDecl(p.ipkg, name)
 	}
 
 	for _, typ := range p.interfaceList {
@@ -165,17 +164,19 @@ func IImportData(fset *token.FileSet, imports map[string]*types.Package, data []
 	// record all referenced packages as imports
 	list := append(([]*types.Package)(nil), pkgList[1:]...)
 	sort.Sort(byPath(list))
-	localpkg.SetImports(list)
+	p.ipkg.SetImports(list)
 
 	// package was imported completely and without errors
-	localpkg.MarkComplete()
+	p.ipkg.MarkComplete()
 
 	consumed, _ := r.Seek(0, io.SeekCurrent)
-	return int(consumed), localpkg, nil
+	return int(consumed), p.ipkg, nil
 }
 
 type iimporter struct {
-	ipath string
+	ipath   string
+	ipkg    *types.Package
+	version int
 
 	stringData  []byte
 	stringCache map[uint64]string
@@ -226,6 +227,9 @@ func (p *iimporter) pkgAt(off uint64) *types.Package {
 		return pkg
 	}
 	path := p.stringAt(off)
+	if path == p.ipath {
+		return p.ipkg
+	}
 	errorf("missing package %q in %q", path, p.ipath)
 	return nil
 }
@@ -255,6 +259,7 @@ type importReader struct {
 	currPkg    *types.Package
 	prevFile   string
 	prevLine   int64
+	prevColumn int64
 }
 
 func (r *importReader) obj(name string) {
@@ -448,6 +453,19 @@ func (r *importReader) qualifiedIdent() (*types.Package, string) {
 }
 
 func (r *importReader) pos() token.Pos {
+	if r.p.version >= 1 {
+		r.posv1()
+	} else {
+		r.posv0()
+	}
+
+	if r.prevFile == "" && r.prevLine == 0 && r.prevColumn == 0 {
+		return token.NoPos
+	}
+	return r.p.fake.pos(r.prevFile, int(r.prevLine), int(r.prevColumn))
+}
+
+func (r *importReader) posv0() {
 	delta := r.int64()
 	if delta != deltaNewFile {
 		r.prevLine += delta
@@ -457,12 +475,18 @@ func (r *importReader) pos() token.Pos {
 		r.prevFile = r.string()
 		r.prevLine = l
 	}
+}
 
-	if r.prevFile == "" && r.prevLine == 0 {
-		return token.NoPos
+func (r *importReader) posv1() {
+	delta := r.int64()
+	r.prevColumn += delta >> 1
+	if delta&1 != 0 {
+		delta = r.int64()
+		r.prevLine += delta >> 1
+		if delta&1 != 0 {
+			r.prevFile = r.string()
+		}
 	}
-
-	return r.p.fake.pos(r.prevFile, int(r.prevLine))
 }
 
 func (r *importReader) typ() types.Type {

vendor/golang.org/x/tools/go/internal/packagesdriver/sizes.go

@@ -81,13 +81,13 @@ func GetSizesGolist(ctx context.Context, buildFlags, env []string, dir string, u
 	args := []string{"list", "-f", "{{context.GOARCH}} {{context.Compiler}}"}
 	args = append(args, buildFlags...)
 	args = append(args, "--", "unsafe")
-	stdout, err := InvokeGo(ctx, env, dir, usesExportData, args...)
+	stdout, stderr, err := invokeGo(ctx, env, dir, usesExportData, args...)
 	var goarch, compiler string
 	if err != nil {
 		if strings.Contains(err.Error(), "cannot find main module") {
 			// User's running outside of a module. All bets are off. Get GOARCH and guess compiler is gc.
 			// TODO(matloob): Is this a problem in practice?
-			envout, enverr := InvokeGo(ctx, env, dir, usesExportData, "env", "GOARCH")
+			envout, _, enverr := invokeGo(ctx, env, dir, usesExportData, "env", "GOARCH")
 			if enverr != nil {
 				return nil, err
 			}
@@ -99,7 +99,8 @@ func GetSizesGolist(ctx context.Context, buildFlags, env []string, dir string, u
 	} else {
 		fields := strings.Fields(stdout.String())
 		if len(fields) < 2 {
-			return nil, fmt.Errorf("could not determine GOARCH and Go compiler")
+			return nil, fmt.Errorf("could not parse GOARCH and Go compiler in format \"<GOARCH> <compiler>\" from stdout of go command:\n%s\ndir: %s\nstdout: <<%s>>\nstderr: <<%s>>",
+				cmdDebugStr(env, args...), dir, stdout.String(), stderr.String())
 		}
 		goarch = fields[0]
 		compiler = fields[1]
@@ -107,8 +108,8 @@ func GetSizesGolist(ctx context.Context, buildFlags, env []string, dir string, u
 	return types.SizesFor(compiler, goarch), nil
 }
 
-// InvokeGo returns the stdout of a go command invocation.
-func InvokeGo(ctx context.Context, env []string, dir string, usesExportData bool, args ...string) (*bytes.Buffer, error) {
+// invokeGo returns the stdout and stderr of a go command invocation.
+func invokeGo(ctx context.Context, env []string, dir string, usesExportData bool, args ...string) (*bytes.Buffer, *bytes.Buffer, error) {
 	if debug {
 		defer func(start time.Time) { log.Printf("%s for %v", time.Since(start), cmdDebugStr(env, args...)) }(time.Now())
 	}
@@ -131,7 +132,7 @@ func InvokeGo(ctx context.Context, env []string, dir string, usesExportData bool
 			// Catastrophic error:
 			// - executable not found
 			// - context cancellation
-			return nil, fmt.Errorf("couldn't exec 'go %v': %s %T", args, err, err)
+			return nil, nil, fmt.Errorf("couldn't exec 'go %v': %s %T", args, err, err)
 		}
 
 		// Export mode entails a build.
@@ -139,7 +140,7 @@ func InvokeGo(ctx context.Context, env []string, dir string, usesExportData bool
 		// (despite the -e flag) and the Export field is blank.
 		// Do not fail in that case.
 		if !usesExportData {
-			return nil, fmt.Errorf("go %v: %s: %s", args, exitErr, stderr)
+			return nil, nil, fmt.Errorf("go %v: %s: %s", args, exitErr, stderr)
 		}
 	}
 
@@ -158,7 +159,7 @@ func InvokeGo(ctx context.Context, env []string, dir string, usesExportData bool
 		fmt.Fprintf(os.Stderr, "%s stdout: <<%s>>\n", cmdDebugStr(env, args...), stdout)
 	}
 
-	return stdout, nil
+	return stdout, stderr, nil
 }
 
 func cmdDebugStr(envlist []string, args ...string) string {

vendor/golang.org/x/tools/go/packages/doc.go

@@ -60,8 +60,7 @@ causes Load to run in LoadFiles mode, collecting minimal information.
 See the documentation for type Config for details.
 
 As noted earlier, the Config.Mode controls the amount of detail
-reported about the loaded packages, with each mode returning all the data of the
-previous mode with some extra added. See the documentation for type LoadMode
+reported about the loaded packages. See the documentation for type LoadMode
 for details.
 
 Most tools should pass their command-line arguments (after any flags)

vendor/golang.org/x/tools/go/packages/external.go

@@ -12,6 +12,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"os"
 	"os/exec"
 	"strings"
 )
@@ -76,15 +77,21 @@ func findExternalDriver(cfg *Config) driver {
 		}
 
 		buf := new(bytes.Buffer)
+		stderr := new(bytes.Buffer)
 		cmd := exec.CommandContext(cfg.Context, tool, words...)
 		cmd.Dir = cfg.Dir
 		cmd.Env = cfg.Env
 		cmd.Stdin = bytes.NewReader(req)
 		cmd.Stdout = buf
-		cmd.Stderr = new(bytes.Buffer)
+		cmd.Stderr = stderr
+
 		if err := cmd.Run(); err != nil {
 			return nil, fmt.Errorf("%v: %v: %s", tool, err, cmd.Stderr)
 		}
+		if len(stderr.Bytes()) != 0 && os.Getenv("GOPACKAGESPRINTDRIVERERRORS") != "" {
+			fmt.Fprintf(os.Stderr, "%s stderr: <<%s>>\n", cmdDebugStr(cmd, words...), stderr)
+		}
+
 		var response driverResponse
 		if err := json.Unmarshal(buf.Bytes(), &response); err != nil {
 			return nil, err

vendor/golang.org/x/tools/go/packages/golist.go

@@ -109,6 +109,7 @@ func goListDriver(cfg *Config, patterns ...string) (*driverResponse, error) {
 			sizeswg.Done()
 		}()
 	}
+	defer sizeswg.Wait()
 
 	// start fetching rootDirs
 	var info goInfo
@@ -127,6 +128,10 @@ func goListDriver(cfg *Config, patterns ...string) (*driverResponse, error) {
 		return &info
 	}
 
+	// Ensure that we don't leak goroutines: Load is synchronous, so callers will
+	// not expect it to access the fields of cfg after the call returns.
+	defer getGoInfo()
+
 	// always pass getGoInfo to golistDriver
 	golistDriver := func(cfg *Config, patterns ...string) (*driverResponse, error) {
 		return golistDriver(cfg, getGoInfo, patterns...)
@@ -248,12 +253,7 @@ func addNeededOverlayPackages(cfg *Config, driver driver, response *responseDedu
 	if len(pkgs) == 0 {
 		return nil
 	}
-	drivercfg := *cfg
-	if getGoInfo().env.modulesOn {
-		drivercfg.BuildFlags = append(drivercfg.BuildFlags, "-mod=readonly")
-	}
-	dr, err := driver(&drivercfg, pkgs...)
-
+	dr, err := driver(cfg, pkgs...)
 	if err != nil {
 		return err
 	}
@@ -264,10 +264,7 @@ func addNeededOverlayPackages(cfg *Config, driver driver, response *responseDedu
 	if err != nil {
 		return err
 	}
-	if err := addNeededOverlayPackages(cfg, driver, response, needPkgs, getGoInfo); err != nil {
-		return err
-	}
-	return nil
+	return addNeededOverlayPackages(cfg, driver, response, needPkgs, getGoInfo)
 }
 
 func runContainsQueries(cfg *Config, driver driver, response *responseDeduper, queries []string, goInfo func() *goInfo) error {
@@ -291,18 +288,31 @@ func runContainsQueries(cfg *Config, driver driver, response *responseDeduper, q
 				// Return the original error if the attempt to fall back failed.
 				return err
 			}
+			// If we get nothing back from `go list`, try to make this file into its own ad-hoc package.
+			if len(dirResponse.Packages) == 0 && queryErr == nil {
+				dirResponse.Packages = append(dirResponse.Packages, &Package{
+					ID:              "command-line-arguments",
+					PkgPath:         query,
+					GoFiles:         []string{query},
+					CompiledGoFiles: []string{query},
+					Imports:         make(map[string]*Package),
+				})
+				dirResponse.Roots = append(dirResponse.Roots, "command-line-arguments")
+			}
 			// Special case to handle issue #33482:
 			// If this is a file= query for ad-hoc packages where the file only exists on an overlay,
 			// and exists outside of a module, add the file in for the package.
-			if len(dirResponse.Packages) == 1 && len(dirResponse.Packages) == 1 &&
-				dirResponse.Packages[0].ID == "command-line-arguments" && len(dirResponse.Packages[0].GoFiles) == 0 {
-				filename := filepath.Join(pattern, filepath.Base(query)) // avoid recomputing abspath
-				// TODO(matloob): check if the file is outside of a root dir?
-				for path := range cfg.Overlay {
-					if path == filename {
-						dirResponse.Packages[0].Errors = nil
-						dirResponse.Packages[0].GoFiles = []string{path}
-						dirResponse.Packages[0].CompiledGoFiles = []string{path}
+			if len(dirResponse.Packages) == 1 && (dirResponse.Packages[0].ID == "command-line-arguments" ||
+				filepath.ToSlash(dirResponse.Packages[0].PkgPath) == filepath.ToSlash(query)) {
+				if len(dirResponse.Packages[0].GoFiles) == 0 {
+					filename := filepath.Join(pattern, filepath.Base(query)) // avoid recomputing abspath
+					// TODO(matloob): check if the file is outside of a root dir?
+					for path := range cfg.Overlay {
+						if path == filename {
+							dirResponse.Packages[0].Errors = nil
+							dirResponse.Packages[0].GoFiles = []string{path}
+							dirResponse.Packages[0].CompiledGoFiles = []string{path}
+						}
 					}
 				}
 			}
@@ -395,6 +405,10 @@ func runNamedQueries(cfg *Config, driver driver, response *responseDeduper, quer
 		}
 
 		files, err := ioutil.ReadDir(modRoot)
+		if err != nil {
+			panic(err) // See above.
+		}
+
 		for _, f := range files {
 			if strings.HasSuffix(f.Name(), ".go") {
 				simpleMatches = append(simpleMatches, rel)
@@ -462,7 +476,7 @@ func runNamedQueries(cfg *Config, driver driver, response *responseDeduper, quer
 		// We're only trying to look at stuff in the module cache, so
 		// disable the network. This should speed things up, and has
 		// prevented errors in at least one case, #28518.
-		tmpCfg.Env = append(append([]string{"GOPROXY=off"}, cfg.Env...))
+		tmpCfg.Env = append([]string{"GOPROXY=off"}, cfg.Env...)
 
 		var err error
 		tmpCfg.Dir, err = ioutil.TempDir("", "gopackages-modquery")
@@ -510,17 +524,29 @@ func roots(cfg *Config) ([]gopathwalk.Root, string, error) {
 
 	var roots []gopathwalk.Root
 	// Always add GOROOT.
-	roots = append(roots, gopathwalk.Root{filepath.Join(goroot, "/src"), gopathwalk.RootGOROOT})
+	roots = append(roots, gopathwalk.Root{
+		Path: filepath.Join(goroot, "/src"),
+		Type: gopathwalk.RootGOROOT,
+	})
 	// If modules are enabled, scan the module dir.
 	if modDir != "" {
-		roots = append(roots, gopathwalk.Root{modDir, gopathwalk.RootCurrentModule})
+		roots = append(roots, gopathwalk.Root{
+			Path: modDir,
+			Type: gopathwalk.RootCurrentModule,
+		})
 	}
 	// Add either GOPATH/src or GOPATH/pkg/mod, depending on module mode.
 	for _, p := range gopath {
 		if modDir != "" {
-			roots = append(roots, gopathwalk.Root{filepath.Join(p, "/pkg/mod"), gopathwalk.RootModuleCache})
+			roots = append(roots, gopathwalk.Root{
+				Path: filepath.Join(p, "/pkg/mod"),
+				Type: gopathwalk.RootModuleCache,
+			})
 		} else {
-			roots = append(roots, gopathwalk.Root{filepath.Join(p, "/src"), gopathwalk.RootGOPATH})
+			roots = append(roots, gopathwalk.Root{
+				Path: filepath.Join(p, "/src"),
+				Type: gopathwalk.RootGOPATH,
+			})
 		}
 	}
 
@@ -637,9 +663,9 @@ func golistDriver(cfg *Config, rootsDirs func() *goInfo, words ...string) (*driv
 	// go list uses the following identifiers in ImportPath and Imports:
 	//
 	// 	"p"			-- importable package or main (command)
-	//      "q.test"		-- q's test executable
+	// 	"q.test"		-- q's test executable
 	// 	"p [q.test]"		-- variant of p as built for q's test executable
-	//	"q_test [q.test]"	-- q's external test package
+	// 	"q_test [q.test]"	-- q's external test package
 	//
 	// The packages p that are built differently for a test q.test
 	// are q itself, plus any helpers used by the external test q_test,
@@ -678,11 +704,11 @@ func golistDriver(cfg *Config, rootsDirs func() *goInfo, words ...string) (*driv
 		// go list -e, when given an absolute path, will find the package contained at
 		// that directory. But when no package exists there, it will return a fake package
 		// with an error and the ImportPath set to the absolute path provided to go list.
-		// Try toto convert that absolute path to what its package path would be if it's
+		// Try to convert that absolute path to what its package path would be if it's
 		// contained in a known module or GOPATH entry. This will allow the package to be
 		// properly "reclaimed" when overlays are processed.
 		if filepath.IsAbs(p.ImportPath) && p.Error != nil {
-			pkgPath, ok := getPkgPath(p.ImportPath, rootsDirs)
+			pkgPath, ok := getPkgPath(cfg, p.ImportPath, rootsDirs)
 			if ok {
 				p.ImportPath = pkgPath
 			}
@@ -779,9 +805,14 @@ func golistDriver(cfg *Config, rootsDirs func() *goInfo, words ...string) (*driv
 		}
 
 		if p.Error != nil {
+			msg := strings.TrimSpace(p.Error.Err) // Trim to work around golang.org/issue/32363.
+			// Address golang.org/issue/35964 by appending import stack to error message.
+			if msg == "import cycle not allowed" && len(p.Error.ImportStack) != 0 {
+				msg += fmt.Sprintf(": import stack: %v", p.Error.ImportStack)
+			}
 			pkg.Errors = append(pkg.Errors, Error{
 				Pos: p.Error.Pos,
-				Msg: strings.TrimSpace(p.Error.Err), // Trim to work around golang.org/issue/32363.
+				Msg: msg,
 			})
 		}
 
@@ -792,15 +823,31 @@ func golistDriver(cfg *Config, rootsDirs func() *goInfo, words ...string) (*driv
 }
 
 // getPkgPath finds the package path of a directory if it's relative to a root directory.
-func getPkgPath(dir string, goInfo func() *goInfo) (string, bool) {
+func getPkgPath(cfg *Config, dir string, goInfo func() *goInfo) (string, bool) {
+	absDir, err := filepath.Abs(dir)
+	if err != nil {
+		cfg.Logf("error getting absolute path of %s: %v", dir, err)
+		return "", false
+	}
 	for rdir, rpath := range goInfo().rootDirs {
+		absRdir, err := filepath.Abs(rdir)
+		if err != nil {
+			cfg.Logf("error getting absolute path of %s: %v", rdir, err)
+			continue
+		}
+		// Make sure that the directory is in the module,
+		// to avoid creating a path relative to another module.
+		if !strings.HasPrefix(absDir, absRdir) {
+			cfg.Logf("%s does not have prefix %s", absDir, absRdir)
+			continue
+		}
 		// TODO(matloob): This doesn't properly handle symlinks.
 		r, err := filepath.Rel(rdir, dir)
 		if err != nil {
 			continue
 		}
 		if rpath != "" {
-			// We choose only ore root even though the directory even it can belong in multiple modules
+			// We choose only one root even though the directory even it can belong in multiple modules
 			// or GOPATH entries. This is okay because we only need to work with absolute dirs when a
 			// file is missing from disk, for instance when gopls calls go/packages in an overlay.
 			// Once the file is saved, gopls, or the next invocation of the tool will get the correct
@@ -808,6 +855,7 @@ func getPkgPath(dir string, goInfo func() *goInfo) (string, bool) {
 			// TODO(matloob): Implement module tiebreaking?
 			return path.Join(rpath, filepath.ToSlash(r)), true
 		}
+		return filepath.ToSlash(r), true
 	}
 	return "", false
 }
@@ -832,8 +880,7 @@ func golistargs(cfg *Config, words []string) []string {
 		fmt.Sprintf("-compiled=%t", cfg.Mode&(NeedCompiledGoFiles|NeedSyntax|NeedTypesInfo|NeedTypesSizes) != 0),
 		fmt.Sprintf("-test=%t", cfg.Tests),
 		fmt.Sprintf("-export=%t", usesExportData(cfg)),
-		fmt.Sprintf("-deps=%t", cfg.Mode&NeedDeps != 0 ||
-			cfg.Mode&NeedTypesInfo != 0), // Dependencies are required to do typechecking on sources, which is required for the TypesInfo.
+		fmt.Sprintf("-deps=%t", cfg.Mode&NeedImports != 0),
 		// go list doesn't let you pass -test and -find together,
 		// probably because you'd just get the TestMain.
 		fmt.Sprintf("-find=%t", !cfg.Tests && cfg.Mode&findFlags == 0),
@@ -860,7 +907,7 @@ func invokeGo(cfg *Config, args ...string) (*bytes.Buffer, error) {
 	cmd.Stdout = stdout
 	cmd.Stderr = stderr
 	defer func(start time.Time) {
-		cfg.Logf("%s for %v, stderr: <<%s>>\n", time.Since(start), cmdDebugStr(cmd, args...), stderr)
+		cfg.Logf("%s for %v, stderr: <<%s>> stdout: <<%s>>\n", time.Since(start), cmdDebugStr(cmd, args...), stderr, stdout)
 	}(time.Now())
 
 	if err := cmd.Run(); err != nil {
@@ -897,7 +944,7 @@ func invokeGo(cfg *Config, args ...string) (*bytes.Buffer, error) {
 			//    (the Graphic characters without spaces) and may also exclude the
 			//    characters !"#$%&'()*,:;<=>?[\]^`{|} and the Unicode replacement character U+FFFD.
 			return unicode.IsOneOf([]*unicode.RangeTable{unicode.L, unicode.M, unicode.N, unicode.P, unicode.S}, r) &&
-				strings.IndexRune("!\"#$%&'()*,:;<=>?[\\]^`{|}\uFFFD", r) == -1
+				!strings.ContainsRune("!\"#$%&'()*,:;<=>?[\\]^`{|}\uFFFD", r)
 		}
 		if len(stderr.String()) > 0 && strings.HasPrefix(stderr.String(), "# ") {
 			if strings.HasPrefix(strings.TrimLeftFunc(stderr.String()[len("# "):], isPkgPathRune), "\n") {
@@ -941,10 +988,31 @@ func invokeGo(cfg *Config, args ...string) (*bytes.Buffer, error) {
 			return bytes.NewBufferString(output), nil
 		}
 
+		// Workaround for #34273. go list -e with GO111MODULE=on has incorrect behavior when listing a
+		// directory outside any module.
+		if len(stderr.String()) > 0 && strings.Contains(stderr.String(), "outside available modules") {
+			output := fmt.Sprintf(`{"ImportPath": %q,"Incomplete": true,"Error": {"Pos": "","Err": %q}}`,
+				// TODO(matloob): command-line-arguments isn't correct here.
+				"command-line-arguments", strings.Trim(stderr.String(), "\n"))
+			return bytes.NewBufferString(output), nil
+		}
+
+		// Another variation of the previous error
+		if len(stderr.String()) > 0 && strings.Contains(stderr.String(), "outside module root") {
+			output := fmt.Sprintf(`{"ImportPath": %q,"Incomplete": true,"Error": {"Pos": "","Err": %q}}`,
+				// TODO(matloob): command-line-arguments isn't correct here.
+				"command-line-arguments", strings.Trim(stderr.String(), "\n"))
+			return bytes.NewBufferString(output), nil
+		}
+
 		// Workaround for an instance of golang.org/issue/26755: go list -e  will return a non-zero exit
 		// status if there's a dependency on a package that doesn't exist. But it should return
 		// a zero exit status and set an error on that package.
 		if len(stderr.String()) > 0 && strings.Contains(stderr.String(), "no Go files in") {
+			// Don't clobber stdout if `go list` actually returned something.
+			if len(stdout.String()) > 0 {
+				return stdout, nil
+			}
 			// try to extract package name from string
 			stderrStr := stderr.String()
 			var importPath string
@@ -978,12 +1046,6 @@ func invokeGo(cfg *Config, args ...string) (*bytes.Buffer, error) {
 	if len(stderr.Bytes()) != 0 && os.Getenv("GOPACKAGESPRINTGOLISTERRORS") != "" {
 		fmt.Fprintf(os.Stderr, "%s stderr: <<%s>>\n", cmdDebugStr(cmd, args...), stderr)
 	}
-
-	// debugging
-	if false {
-		fmt.Fprintf(os.Stderr, "%s stdout: <<%s>>\n", cmdDebugStr(cmd, args...), stdout)
-	}
-
 	return stdout, nil
 }
 

vendor/golang.org/x/tools/go/packages/golist_overlay.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"go/parser"
 	"go/token"
-	"path"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -38,10 +37,10 @@ func processGolistOverlay(cfg *Config, response *responseDeduper, rootDirs func(
 	for opath, contents := range cfg.Overlay {
 		base := filepath.Base(opath)
 		dir := filepath.Dir(opath)
-		var pkg *Package
+		var pkg *Package           // if opath belongs to both a package and its test variant, this will be the test variant
 		var testVariantOf *Package // if opath is a test file, this is the package it is testing
 		var fileExists bool
-		isTest := strings.HasSuffix(opath, "_test.go")
+		isTestFile := strings.HasSuffix(opath, "_test.go")
 		pkgName, ok := extractPackageName(opath, contents)
 		if !ok {
 			// Don't bother adding a file that doesn't even have a parsable package statement
@@ -57,13 +56,26 @@ func processGolistOverlay(cfg *Config, response *responseDeduper, rootDirs func(
 				if !sameFile(filepath.Dir(f), dir) {
 					continue
 				}
-				if isTest && !hasTestFiles(p) {
+				// Make sure to capture information on the package's test variant, if needed.
+				if isTestFile && !hasTestFiles(p) {
 					// TODO(matloob): Are there packages other than the 'production' variant
 					// of a package that this can match? This shouldn't match the test main package
 					// because the file is generated in another directory.
 					testVariantOf = p
 					continue nextPackage
 				}
+				if pkg != nil && p != pkg && pkg.PkgPath == p.PkgPath {
+					// If we've already seen the test variant,
+					// make sure to label which package it is a test variant of.
+					if hasTestFiles(pkg) {
+						testVariantOf = p
+						continue nextPackage
+					}
+					// If we have already seen the package of which this is a test variant.
+					if hasTestFiles(p) {
+						testVariantOf = pkg
+					}
+				}
 				pkg = p
 				if filepath.Base(f) == base {
 					fileExists = true
@@ -74,32 +86,16 @@ func processGolistOverlay(cfg *Config, response *responseDeduper, rootDirs func(
 		if pkg == nil {
 			// Try to find the module or gopath dir the file is contained in.
 			// Then for modules, add the module opath to the beginning.
-			var pkgPath string
-			for rdir, rpath := range rootDirs().rootDirs {
-				// TODO(matloob): This doesn't properly handle symlinks.
-				r, err := filepath.Rel(rdir, dir)
-				if err != nil {
-					continue
-				}
-				pkgPath = filepath.ToSlash(r)
-				if rpath != "" {
-					pkgPath = path.Join(rpath, pkgPath)
-				}
-				// We only create one new package even it can belong in multiple modules or GOPATH entries.
-				// This is okay because tools (such as the LSP) that use overlays will recompute the overlay
-				// once the file is saved, and golist will do the right thing.
-				// TODO(matloob): Implement module tiebreaking?
+			pkgPath, ok := getPkgPath(cfg, dir, rootDirs)
+			if !ok {
 				break
 			}
-			if pkgPath == "" {
-				continue
-			}
 			isXTest := strings.HasSuffix(pkgName, "_test")
 			if isXTest {
 				pkgPath += "_test"
 			}
 			id := pkgPath
-			if isTest && !isXTest {
+			if isTestFile && !isXTest {
 				id = fmt.Sprintf("%s [%s.test]", pkgPath, pkgPath)
 			}
 			// Try to reclaim a package with the same id if it exists in the response.
@@ -115,7 +111,7 @@ func processGolistOverlay(cfg *Config, response *responseDeduper, rootDirs func(
 				response.addPackage(pkg)
 				havePkgs[pkg.PkgPath] = id
 				// Add the production package's sources for a test variant.
-				if isTest && !isXTest && testVariantOf != nil {
+				if isTestFile && !isXTest && testVariantOf != nil {
 					pkg.GoFiles = append(pkg.GoFiles, testVariantOf.GoFiles...)
 					pkg.CompiledGoFiles = append(pkg.CompiledGoFiles, testVariantOf.CompiledGoFiles...)
 				}
@@ -138,12 +134,16 @@ func processGolistOverlay(cfg *Config, response *responseDeduper, rootDirs func(
 			if !found {
 				overlayAddsImports = true
 				// TODO(matloob): Handle cases when the following block isn't correct.
-				// These include imports of test variants, imports of vendored packages, etc.
+				// These include imports of vendored packages, etc.
 				id, ok := havePkgs[imp]
 				if !ok {
 					id = imp
 				}
 				pkg.Imports[imp] = &Package{ID: id}
+				// Add dependencies to the non-test variant version of this package as wel.
+				if testVariantOf != nil {
+					testVariantOf.Imports[imp] = &Package{ID: id}
+				}
 			}
 		}
 		continue

vendor/golang.org/x/tools/go/packages/loadmode_string.go

@@ -0,0 +1,57 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package packages
+
+import (
+	"fmt"
+	"strings"
+)
+
+var allModes = []LoadMode{
+	NeedName,
+	NeedFiles,
+	NeedCompiledGoFiles,
+	NeedImports,
+	NeedDeps,
+	NeedExportsFile,
+	NeedTypes,
+	NeedSyntax,
+	NeedTypesInfo,
+	NeedTypesSizes,
+}
+
+var modeStrings = []string{
+	"NeedName",
+	"NeedFiles",
+	"NeedCompiledGoFiles",
+	"NeedImports",
+	"NeedDeps",
+	"NeedExportsFile",
+	"NeedTypes",
+	"NeedSyntax",
+	"NeedTypesInfo",
+	"NeedTypesSizes",
+}
+
+func (mod LoadMode) String() string {
+	m := mod
+	if m == 0 {
+		return fmt.Sprintf("LoadMode(0)")
+	}
+	var out []string
+	for i, x := range allModes {
+		if x > m {
+			break
+		}
+		if (m & x) != 0 {
+			out = append(out, modeStrings[i])
+			m = m ^ x
+		}
+	}
+	if m != 0 {
+		out = append(out, "Unknown")
+	}
+	return fmt.Sprintf("LoadMode(%s)", strings.Join(out, "|"))
+}

vendor/golang.org/x/tools/go/packages/packages.go

@@ -48,8 +48,7 @@ const (
 	// "placeholder" Packages with only the ID set.
 	NeedImports
 
-	// NeedDeps adds the fields requested by the LoadMode in the packages in Imports. If NeedImports
-	// is not set NeedDeps has no effect.
+	// NeedDeps adds the fields requested by the LoadMode in the packages in Imports.
 	NeedDeps
 
 	// NeedExportsFile adds ExportsFile.
@@ -75,7 +74,7 @@ const (
 
 	// Deprecated: LoadImports exists for historical compatibility
 	// and should not be used. Please directly specify the needed fields using the Need values.
-	LoadImports = LoadFiles | NeedImports | NeedDeps
+	LoadImports = LoadFiles | NeedImports
 
 	// Deprecated: LoadTypes exists for historical compatibility
 	// and should not be used. Please directly specify the needed fields using the Need values.
@@ -87,7 +86,7 @@ const (
 
 	// Deprecated: LoadAllSyntax exists for historical compatibility
 	// and should not be used. Please directly specify the needed fields using the Need values.
-	LoadAllSyntax = LoadSyntax
+	LoadAllSyntax = LoadSyntax | NeedDeps
 )
 
 // A Config specifies details about how packages should be loaded.
@@ -416,11 +415,13 @@ type loader struct {
 	parseCacheMu sync.Mutex
 	exportMu     sync.Mutex // enforces mutual exclusion of exportdata operations
 
-	// TODO(matloob): Add an implied mode here and use that instead of mode.
-	// Implied mode would contain all the fields we need the data for so we can
-	// get the actually requested fields. We'll zero them out before returning
-	// packages to the user. This will make it easier for us to get the conditions
-	// where we need certain modes right.
+	// Config.Mode contains the implied mode (see impliedLoadMode).
+	// Implied mode contains all the fields we need the data for.
+	// In requestedMode there are the actually requested fields.
+	// We'll zero them out before returning packages to the user.
+	// This makes it easier for us to get the conditions where
+	// we need certain modes right.
+	requestedMode LoadMode
 }
 
 type parseValue struct {
@@ -462,7 +463,11 @@ func newLoader(cfg *Config) *loader {
 		}
 	}
 
-	if ld.Mode&NeedTypes != 0 {
+	// Save the actually requested fields. We'll zero them out before returning packages to the user.
+	ld.requestedMode = ld.Mode
+	ld.Mode = impliedLoadMode(ld.Mode)
+
+	if ld.Mode&NeedTypes != 0 || ld.Mode&NeedSyntax != 0 {
 		if ld.Fset == nil {
 			ld.Fset = token.NewFileSet()
 		}
@@ -476,6 +481,7 @@ func newLoader(cfg *Config) *loader {
 			}
 		}
 	}
+
 	return ld
 }
 
@@ -496,7 +502,7 @@ func (ld *loader) refine(roots []string, list ...*Package) ([]*Package, error) {
 		}
 		lpkg := &loaderPackage{
 			Package:   pkg,
-			needtypes: (ld.Mode&(NeedTypes|NeedTypesInfo) != 0 && rootIndex < 0) || rootIndex >= 0,
+			needtypes: (ld.Mode&(NeedTypes|NeedTypesInfo) != 0 && ld.Mode&NeedDeps != 0 && rootIndex < 0) || rootIndex >= 0,
 			needsrc: (ld.Mode&(NeedSyntax|NeedTypesInfo) != 0 && ld.Mode&NeedDeps != 0 && rootIndex < 0) || rootIndex >= 0 ||
 				len(ld.Overlay) > 0 || // Overlays can invalidate export data. TODO(matloob): make this check fine-grained based on dependencies on overlaid files
 				pkg.ExportFile == "" && pkg.PkgPath != "unsafe",
@@ -544,10 +550,8 @@ func (ld *loader) refine(roots []string, list ...*Package) ([]*Package, error) {
 		lpkg.color = grey
 		stack = append(stack, lpkg) // push
 		stubs := lpkg.Imports       // the structure form has only stubs with the ID in the Imports
-		// If NeedTypesInfo we need dependencies (at least for the roots) to typecheck the package.
 		// If NeedImports isn't set, the imports fields will all be zeroed out.
-		// If NeedDeps isn't also set we want to keep the stubs.
-		if ld.Mode&NeedTypesInfo != 0 || (ld.Mode&NeedImports != 0 && ld.Mode&NeedDeps != 0) {
+		if ld.Mode&NeedImports != 0 {
 			lpkg.Imports = make(map[string]*Package, len(stubs))
 			for importPath, ipkg := range stubs {
 				var importErr error
@@ -566,11 +570,8 @@ func (ld *loader) refine(roots []string, list ...*Package) ([]*Package, error) {
 					continue
 				}
 
-				// If !NeedDeps, just fill Imports for the root. No need to recurse further.
-				if ld.Mode&NeedDeps != 0 {
-					if visit(imp) {
-						lpkg.needsrc = true
-					}
+				if visit(imp) {
+					lpkg.needsrc = true
 				}
 				lpkg.Imports[importPath] = imp.Package
 			}
@@ -587,7 +588,7 @@ func (ld *loader) refine(roots []string, list ...*Package) ([]*Package, error) {
 		return lpkg.needsrc
 	}
 
-	if ld.Mode&(NeedImports|NeedDeps|NeedTypesInfo) == 0 {
+	if ld.Mode&NeedImports == 0 {
 		// We do this to drop the stub import packages that we are not even going to try to resolve.
 		for _, lpkg := range initial {
 			lpkg.Imports = nil
@@ -598,7 +599,7 @@ func (ld *loader) refine(roots []string, list ...*Package) ([]*Package, error) {
 			visit(lpkg)
 		}
 	}
-	if ld.Mode&NeedDeps != 0 { // TODO(matloob): This is only the case if NeedTypes is also set, right?
+	if ld.Mode&NeedImports != 0 && ld.Mode&NeedTypes != 0 {
 		for _, lpkg := range srcPkgs {
 			// Complete type information is required for the
 			// immediate dependencies of each source package.
@@ -608,9 +609,9 @@ func (ld *loader) refine(roots []string, list ...*Package) ([]*Package, error) {
 			}
 		}
 	}
-	// Load type data if needed, starting at
+	// Load type data and syntax if needed, starting at
 	// the initial packages (roots of the import DAG).
-	if ld.Mode&NeedTypes != 0 {
+	if ld.Mode&NeedTypes != 0 || ld.Mode&NeedSyntax != 0 {
 		var wg sync.WaitGroup
 		for _, lpkg := range initial {
 			wg.Add(1)
@@ -623,54 +624,44 @@ func (ld *loader) refine(roots []string, list ...*Package) ([]*Package, error) {
 	}
 
 	result := make([]*Package, len(initial))
-	importPlaceholders := make(map[string]*Package)
 	for i, lpkg := range initial {
 		result[i] = lpkg.Package
 	}
 	for i := range ld.pkgs {
 		// Clear all unrequested fields, for extra de-Hyrum-ization.
-		if ld.Mode&NeedName == 0 {
+		if ld.requestedMode&NeedName == 0 {
 			ld.pkgs[i].Name = ""
 			ld.pkgs[i].PkgPath = ""
 		}
-		if ld.Mode&NeedFiles == 0 {
+		if ld.requestedMode&NeedFiles == 0 {
 			ld.pkgs[i].GoFiles = nil
 			ld.pkgs[i].OtherFiles = nil
 		}
-		if ld.Mode&NeedCompiledGoFiles == 0 {
+		if ld.requestedMode&NeedCompiledGoFiles == 0 {
 			ld.pkgs[i].CompiledGoFiles = nil
 		}
-		if ld.Mode&NeedImports == 0 {
+		if ld.requestedMode&NeedImports == 0 {
 			ld.pkgs[i].Imports = nil
 		}
-		if ld.Mode&NeedExportsFile == 0 {
+		if ld.requestedMode&NeedExportsFile == 0 {
 			ld.pkgs[i].ExportFile = ""
 		}
-		if ld.Mode&NeedTypes == 0 {
+		if ld.requestedMode&NeedTypes == 0 {
 			ld.pkgs[i].Types = nil
 			ld.pkgs[i].Fset = nil
 			ld.pkgs[i].IllTyped = false
 		}
-		if ld.Mode&NeedSyntax == 0 {
+		if ld.requestedMode&NeedSyntax == 0 {
 			ld.pkgs[i].Syntax = nil
 		}
-		if ld.Mode&NeedTypesInfo == 0 {
+		if ld.requestedMode&NeedTypesInfo == 0 {
 			ld.pkgs[i].TypesInfo = nil
 		}
-		if ld.Mode&NeedTypesSizes == 0 {
+		if ld.requestedMode&NeedTypesSizes == 0 {
 			ld.pkgs[i].TypesSizes = nil
 		}
-		if ld.Mode&NeedDeps == 0 {
-			for j, pkg := range ld.pkgs[i].Imports {
-				ph, ok := importPlaceholders[pkg.ID]
-				if !ok {
-					ph = &Package{ID: pkg.ID}
-					importPlaceholders[pkg.ID] = ph
-				}
-				ld.pkgs[i].Imports[j] = ph
-			}
-		}
 	}
+
 	return result, nil
 }
 
@@ -691,7 +682,6 @@ func (ld *loader) loadRecursive(lpkg *loaderPackage) {
 			}(imp)
 		}
 		wg.Wait()
-
 		ld.loadPackage(lpkg)
 	})
 }
@@ -780,12 +770,23 @@ func (ld *loader) loadPackage(lpkg *loaderPackage) {
 		lpkg.Errors = append(lpkg.Errors, errs...)
 	}
 
+	if ld.Config.Mode&NeedTypes != 0 && len(lpkg.CompiledGoFiles) == 0 && lpkg.ExportFile != "" {
+		// The config requested loading sources and types, but sources are missing.
+		// Add an error to the package and fall back to loading from export data.
+		appendError(Error{"-", fmt.Sprintf("sources missing for package %s", lpkg.ID), ParseError})
+		ld.loadFromExportData(lpkg)
+		return // can't get syntax trees for this package
+	}
+
 	files, errs := ld.parseFiles(lpkg.CompiledGoFiles)
 	for _, err := range errs {
 		appendError(err)
 	}
 
 	lpkg.Syntax = files
+	if ld.Config.Mode&NeedTypes == 0 {
+		return
+	}
 
 	lpkg.TypesInfo = &types.Info{
 		Types:      make(map[ast.Expr]types.TypeAndValue),
@@ -818,7 +819,7 @@ func (ld *loader) loadPackage(lpkg *loaderPackage) {
 		if ipkg.Types != nil && ipkg.Types.Complete() {
 			return ipkg.Types, nil
 		}
-		log.Fatalf("internal error: nil Pkg importing %q from %q", path, lpkg)
+		log.Fatalf("internal error: package %q without types was imported from %q", path, lpkg)
 		panic("unreachable")
 	})
 
@@ -829,7 +830,7 @@ func (ld *loader) loadPackage(lpkg *loaderPackage) {
 		// Type-check bodies of functions only in non-initial packages.
 		// Example: for import graph A->B->C and initial packages {A,C},
 		// we can ignore function bodies in B.
-		IgnoreFuncBodies: (ld.Mode&(NeedDeps|NeedTypesInfo) == 0) && !lpkg.initial,
+		IgnoreFuncBodies: ld.Mode&NeedDeps == 0 && !lpkg.initial,
 
 		Error: appendError,
 		Sizes: ld.sizes,
@@ -1091,10 +1092,25 @@ func (ld *loader) loadFromExportData(lpkg *loaderPackage) (*types.Package, error
 	return tpkg, nil
 }
 
-func usesExportData(cfg *Config) bool {
-	return cfg.Mode&NeedExportsFile != 0 ||
-		// If NeedTypes but not NeedTypesInfo we won't typecheck using sources, so we need export data.
-		(cfg.Mode&NeedTypes != 0 && cfg.Mode&NeedTypesInfo == 0) ||
-		// If NeedTypesInfo but not NeedDeps, we're typechecking a package using its sources plus its dependencies' export data
-		(cfg.Mode&NeedTypesInfo != 0 && cfg.Mode&NeedDeps == 0)
+// impliedLoadMode returns loadMode with its dependencies.
+func impliedLoadMode(loadMode LoadMode) LoadMode {
+	if loadMode&NeedTypesInfo != 0 && loadMode&NeedImports == 0 {
+		// If NeedTypesInfo, go/packages needs to do typechecking itself so it can
+		// associate type info with the AST. To do so, we need the export data
+		// for dependencies, which means we need to ask for the direct dependencies.
+		// NeedImports is used to ask for the direct dependencies.
+		loadMode |= NeedImports
+	}
+
+	if loadMode&NeedDeps != 0 && loadMode&NeedImports == 0 {
+		// With NeedDeps we need to load at least direct dependencies.
+		// NeedImports is used to ask for the direct dependencies.
+		loadMode |= NeedImports
+	}
+
+	return loadMode
+}
+
+func usesExportData(cfg *Config) bool {
+	return cfg.Mode&NeedExportsFile != 0 || cfg.Mode&NeedTypes != 0 && cfg.Mode&NeedDeps == 0
 }

vendor/golang.org/x/tools/internal/gopathwalk/walk.go

@@ -16,6 +16,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"golang.org/x/tools/internal/fastwalk"
 )
@@ -83,8 +84,9 @@ func walkDir(root Root, add func(Root, string), skip func(root Root, dir string)
 		}
 		return
 	}
+	start := time.Now()
 	if opts.Debug {
-		log.Printf("scanning %s", root.Path)
+		log.Printf("gopathwalk: scanning %s", root.Path)
 	}
 	w := &walker{
 		root: root,
@@ -98,7 +100,7 @@ func walkDir(root Root, add func(Root, string), skip func(root Root, dir string)
 	}
 
 	if opts.Debug {
-		log.Printf("scanned %s", root.Path)
+		log.Printf("gopathwalk: scanned %s in %v", root.Path, time.Since(start))
 	}
 }
 

vendor/golang.org/x/tools/internal/imports/fix.go

@@ -17,6 +17,7 @@ import (
 	"os/exec"
 	"path"
 	"path/filepath"
+	"reflect"
 	"sort"
 	"strconv"
 	"strings"
@@ -301,7 +302,7 @@ func (p *pass) importIdentifier(imp *ImportInfo) string {
 	if known != nil && known.name != "" {
 		return known.name
 	}
-	return importPathToAssumedName(imp.ImportPath)
+	return ImportPathToAssumedName(imp.ImportPath)
 }
 
 // load reads in everything necessary to run a pass, and reports whether the
@@ -434,7 +435,7 @@ func (p *pass) importSpecName(imp *ImportInfo) string {
 	}
 
 	ident := p.importIdentifier(imp)
-	if ident == importPathToAssumedName(imp.ImportPath) {
+	if ident == ImportPathToAssumedName(imp.ImportPath) {
 		return "" // ident not needed since the assumed and real names are the same.
 	}
 	return ident
@@ -475,9 +476,9 @@ func (p *pass) assumeSiblingImportsValid() {
 		}
 		for left, rights := range refs {
 			if imp, ok := importsByName[left]; ok {
-				if _, ok := stdlib[imp.ImportPath]; ok {
+				if m, ok := stdlib[imp.ImportPath]; ok {
 					// We have the stdlib in memory; no need to guess.
-					rights = stdlib[imp.ImportPath]
+					rights = copyExports(m)
 				}
 				p.addCandidate(imp, &packageInfo{
 					// no name; we already know it.
@@ -584,29 +585,133 @@ func getFixes(fset *token.FileSet, f *ast.File, filename string, env *ProcessEnv
 	return fixes, nil
 }
 
+// getCandidatePkgs returns the list of pkgs that are accessible from filename,
+// optionall filtered to only packages named pkgName.
+func getCandidatePkgs(pkgName, filename string, env *ProcessEnv) ([]*pkg, error) {
+	// TODO(heschi): filter out current package. (Don't forget x_test can import x.)
+
+	var result []*pkg
+	// Start off with the standard library.
+	for importPath := range stdlib {
+		if pkgName != "" && path.Base(importPath) != pkgName {
+			continue
+		}
+		result = append(result, &pkg{
+			dir:             filepath.Join(env.GOROOT, "src", importPath),
+			importPathShort: importPath,
+			packageName:     path.Base(importPath),
+			relevance:       0,
+		})
+	}
+
+	// Exclude goroot results -- getting them is relatively expensive, not cached,
+	// and generally redundant with the in-memory version.
+	exclude := []gopathwalk.RootType{gopathwalk.RootGOROOT}
+	// Only the go/packages resolver uses the first argument, and nobody uses that resolver.
+	scannedPkgs, err := env.GetResolver().scan(nil, true, exclude)
+	if err != nil {
+		return nil, err
+	}
+
+	dupCheck := map[string]struct{}{}
+	for _, pkg := range scannedPkgs {
+		if pkgName != "" && pkg.packageName != pkgName {
+			continue
+		}
+		if !canUse(filename, pkg.dir) {
+			continue
+		}
+		if _, ok := dupCheck[pkg.importPathShort]; ok {
+			continue
+		}
+		dupCheck[pkg.importPathShort] = struct{}{}
+		result = append(result, pkg)
+	}
+
+	// Sort first by relevance, then by package name, with import path as a tiebreaker.
+	sort.Slice(result, func(i, j int) bool {
+		pi, pj := result[i], result[j]
+		if pi.relevance != pj.relevance {
+			return pi.relevance < pj.relevance
+		}
+		if pi.packageName != pj.packageName {
+			return pi.packageName < pj.packageName
+		}
+		return pi.importPathShort < pj.importPathShort
+	})
+
+	return result, nil
+}
+
+func candidateImportName(pkg *pkg) string {
+	if ImportPathToAssumedName(pkg.importPathShort) != pkg.packageName {
+		return pkg.packageName
+	}
+	return ""
+}
+
 // getAllCandidates gets all of the candidates to be imported, regardless of if they are needed.
 func getAllCandidates(filename string, env *ProcessEnv) ([]ImportFix, error) {
-	// TODO(suzmue): scan for additional candidates and filter out
-	// current package.
-
-	// Get the stdlib candidates and sort by import path.
-	var paths []string
-	for importPath := range stdlib {
-		paths = append(paths, importPath)
+	pkgs, err := getCandidatePkgs("", filename, env)
+	if err != nil {
+		return nil, err
 	}
-	sort.Strings(paths)
-
-	var imports []ImportFix
-	for _, importPath := range paths {
-		imports = append(imports, ImportFix{
+	result := make([]ImportFix, 0, len(pkgs))
+	for _, pkg := range pkgs {
+		result = append(result, ImportFix{
 			StmtInfo: ImportInfo{
-				ImportPath: importPath,
+				ImportPath: pkg.importPathShort,
+				Name:       candidateImportName(pkg),
 			},
-			IdentName: path.Base(importPath),
+			IdentName: pkg.packageName,
 			FixType:   AddImport,
 		})
 	}
-	return imports, nil
+	return result, nil
+}
+
+// A PackageExport is a package and its exports.
+type PackageExport struct {
+	Fix     *ImportFix
+	Exports []string
+}
+
+func getPackageExports(completePackage, filename string, env *ProcessEnv) ([]PackageExport, error) {
+	pkgs, err := getCandidatePkgs(completePackage, filename, env)
+	if err != nil {
+		return nil, err
+	}
+
+	results := make([]PackageExport, 0, len(pkgs))
+	for _, pkg := range pkgs {
+		fix := &ImportFix{
+			StmtInfo: ImportInfo{
+				ImportPath: pkg.importPathShort,
+				Name:       candidateImportName(pkg),
+			},
+			IdentName: pkg.packageName,
+			FixType:   AddImport,
+		}
+		var exports []string
+		if e, ok := stdlib[pkg.importPathShort]; ok {
+			exports = e
+		} else {
+			exports, err = loadExportsForPackage(context.Background(), env, completePackage, pkg)
+			if err != nil {
+				if env.Debug {
+					env.Logf("while completing %q, error loading exports from %q: %v", completePackage, pkg.importPathShort, err)
+				}
+				continue
+			}
+		}
+		sort.Strings(exports)
+		results = append(results, PackageExport{
+			Fix:     fix,
+			Exports: exports,
+		})
+	}
+
+	return results, nil
 }
 
 // ProcessEnv contains environment variables and settings that affect the use of
@@ -678,6 +783,13 @@ func (e *ProcessEnv) buildContext() *build.Context {
 	ctx := build.Default
 	ctx.GOROOT = e.GOROOT
 	ctx.GOPATH = e.GOPATH
+
+	// As of Go 1.14, build.Context has a WorkingDir field
+	// (see golang.org/issue/34860).
+	// Populate it only if present.
+	if wd := reflect.ValueOf(&ctx).Elem().FieldByName("WorkingDir"); wd.IsValid() && wd.Kind() == reflect.String {
+		wd.SetString(e.WorkingDir)
+	}
 	return &ctx
 }
 
@@ -712,9 +824,10 @@ func cmdDebugStr(cmd *exec.Cmd) string {
 
 func addStdlibCandidates(pass *pass, refs references) {
 	add := func(pkg string) {
+		exports := copyExports(stdlib[pkg])
 		pass.addCandidate(
 			&ImportInfo{ImportPath: pkg},
-			&packageInfo{name: path.Base(pkg), exports: stdlib[pkg]})
+			&packageInfo{name: path.Base(pkg), exports: exports})
 	}
 	for left := range refs {
 		if left == "rand" {
@@ -735,12 +848,15 @@ func addStdlibCandidates(pass *pass, refs references) {
 type Resolver interface {
 	// loadPackageNames loads the package names in importPaths.
 	loadPackageNames(importPaths []string, srcDir string) (map[string]string, error)
-	// scan finds (at least) the packages satisfying refs. The returned slice is unordered.
-	scan(refs references) ([]*pkg, error)
+	// scan finds (at least) the packages satisfying refs. If loadNames is true,
+	// package names will be set on the results, and dirs whose package name
+	// could not be determined will be excluded.
+	scan(refs references, loadNames bool, exclude []gopathwalk.RootType) ([]*pkg, error)
 	// loadExports returns the set of exported symbols in the package at dir.
-	// It returns an error if the package name in dir does not match expectPackage.
 	// loadExports may be called concurrently.
-	loadExports(ctx context.Context, expectPackage string, pkg *pkg) (map[string]bool, error)
+	loadExports(ctx context.Context, pkg *pkg) (string, []string, error)
+
+	ClearForNewScan()
 }
 
 // gopackagesResolver implements resolver for GOPATH and module workspaces using go/packages.
@@ -748,6 +864,8 @@ type goPackagesResolver struct {
 	env *ProcessEnv
 }
 
+func (r *goPackagesResolver) ClearForNewScan() {}
+
 func (r *goPackagesResolver) loadPackageNames(importPaths []string, srcDir string) (map[string]string, error) {
 	if len(importPaths) == 0 {
 		return nil, nil
@@ -766,13 +884,13 @@ func (r *goPackagesResolver) loadPackageNames(importPaths []string, srcDir strin
 		if _, ok := names[path]; ok {
 			continue
 		}
-		names[path] = importPathToAssumedName(path)
+		names[path] = ImportPathToAssumedName(path)
 	}
 	return names, nil
 
 }
 
-func (r *goPackagesResolver) scan(refs references) ([]*pkg, error) {
+func (r *goPackagesResolver) scan(refs references, _ bool, _ []gopathwalk.RootType) ([]*pkg, error) {
 	var loadQueries []string
 	for pkgName := range refs {
 		loadQueries = append(loadQueries, "iamashamedtousethedisabledqueryname="+pkgName)
@@ -790,33 +908,34 @@ func (r *goPackagesResolver) scan(refs references) ([]*pkg, error) {
 			dir:             filepath.Dir(goPackage.CompiledGoFiles[0]),
 			importPathShort: VendorlessPath(goPackage.PkgPath),
 			goPackage:       goPackage,
+			packageName:     goPackage.Name,
 		})
 	}
 	return scan, nil
 }
 
-func (r *goPackagesResolver) loadExports(ctx context.Context, expectPackage string, pkg *pkg) (map[string]bool, error) {
+func (r *goPackagesResolver) loadExports(ctx context.Context, pkg *pkg) (string, []string, error) {
 	if pkg.goPackage == nil {
-		return nil, fmt.Errorf("goPackage not set")
+		return "", nil, fmt.Errorf("goPackage not set")
 	}
-	exports := map[string]bool{}
+	var exports []string
 	fset := token.NewFileSet()
 	for _, fname := range pkg.goPackage.CompiledGoFiles {
 		f, err := parser.ParseFile(fset, fname, nil, 0)
 		if err != nil {
-			return nil, fmt.Errorf("parsing %s: %v", fname, err)
+			return "", nil, fmt.Errorf("parsing %s: %v", fname, err)
 		}
 		for name := range f.Scope.Objects {
 			if ast.IsExported(name) {
-				exports[name] = true
+				exports = append(exports, name)
 			}
 		}
 	}
-	return exports, nil
+	return pkg.goPackage.Name, exports, nil
 }
 
 func addExternalCandidates(pass *pass, refs references, filename string) error {
-	dirScan, err := pass.env.GetResolver().scan(refs)
+	dirScan, err := pass.env.GetResolver().scan(refs, false, nil)
 	if err != nil {
 		return err
 	}
@@ -887,7 +1006,7 @@ func notIdentifier(ch rune) bool {
 		ch >= utf8.RuneSelf && (unicode.IsLetter(ch) || unicode.IsDigit(ch)))
 }
 
-// importPathToAssumedName returns the assumed package name of an import path.
+// ImportPathToAssumedName returns the assumed package name of an import path.
 // It does this using only string parsing of the import path.
 // It picks the last element of the path that does not look like a major
 // version, and then picks the valid identifier off the start of that element.
@@ -895,7 +1014,7 @@ func notIdentifier(ch rune) bool {
 // clarity.
 // This function could be moved to a standard package and exported if we want
 // for use in other tools.
-func importPathToAssumedName(importPath string) string {
+func ImportPathToAssumedName(importPath string) string {
 	base := path.Base(importPath)
 	if strings.HasPrefix(base, "v") {
 		if _, err := strconv.Atoi(base[1:]); err == nil {
@@ -914,10 +1033,24 @@ func importPathToAssumedName(importPath string) string {
 
 // gopathResolver implements resolver for GOPATH workspaces.
 type gopathResolver struct {
-	env *ProcessEnv
+	env   *ProcessEnv
+	cache *dirInfoCache
+}
+
+func (r *gopathResolver) init() {
+	if r.cache == nil {
+		r.cache = &dirInfoCache{
+			dirs: map[string]*directoryPackageInfo{},
+		}
+	}
+}
+
+func (r *gopathResolver) ClearForNewScan() {
+	r.cache = nil
 }
 
 func (r *gopathResolver) loadPackageNames(importPaths []string, srcDir string) (map[string]string, error) {
+	r.init()
 	names := map[string]string{}
 	for _, path := range importPaths {
 		names[path] = importPathToName(r.env, path, srcDir)
@@ -1000,6 +1133,8 @@ type pkg struct {
 	goPackage       *packages.Package
 	dir             string // absolute file path to pkg directory ("/usr/lib/go/src/net/http")
 	importPathShort string // vendorless import path ("net/http", "a/b")
+	packageName     string // package name loaded from source if requested
+	relevance       int    // a weakly-defined score of how relevant a package is. 0 is most relevant.
 }
 
 type pkgDistance struct {
@@ -1043,32 +1178,74 @@ func distance(basepath, targetpath string) int {
 	return strings.Count(p, string(filepath.Separator)) + 1
 }
 
-func (r *gopathResolver) scan(_ references) ([]*pkg, error) {
-	dupCheck := make(map[string]bool)
-	var result []*pkg
-
-	var mu sync.Mutex
-
+func (r *gopathResolver) scan(_ references, loadNames bool, exclude []gopathwalk.RootType) ([]*pkg, error) {
+	r.init()
 	add := func(root gopathwalk.Root, dir string) {
-		mu.Lock()
-		defer mu.Unlock()
-
-		if _, dup := dupCheck[dir]; dup {
+		// We assume cached directories have not changed. We can skip them and their
+		// children.
+		if _, ok := r.cache.Load(dir); ok {
 			return
 		}
-		dupCheck[dir] = true
+
 		importpath := filepath.ToSlash(dir[len(root.Path)+len("/"):])
-		result = append(result, &pkg{
-			importPathShort: VendorlessPath(importpath),
-			dir:             dir,
-		})
+		info := directoryPackageInfo{
+			status:                 directoryScanned,
+			dir:                    dir,
+			rootType:               root.Type,
+			nonCanonicalImportPath: VendorlessPath(importpath),
+		}
+		r.cache.Store(dir, info)
+	}
+	roots := filterRoots(gopathwalk.SrcDirsRoots(r.env.buildContext()), exclude)
+	gopathwalk.Walk(roots, add, gopathwalk.Options{Debug: r.env.Debug, ModulesEnabled: false})
+	var result []*pkg
+	for _, dir := range r.cache.Keys() {
+		info, ok := r.cache.Load(dir)
+		if !ok {
+			continue
+		}
+		if loadNames {
+			var err error
+			info, err = r.cache.CachePackageName(info)
+			if err != nil {
+				continue
+			}
+		}
+
+		p := &pkg{
+			importPathShort: info.nonCanonicalImportPath,
+			dir:             dir,
+			relevance:       1,
+			packageName:     info.packageName,
+		}
+		if info.rootType == gopathwalk.RootGOROOT {
+			p.relevance = 0
+		}
+		result = append(result, p)
 	}
-	gopathwalk.Walk(gopathwalk.SrcDirsRoots(r.env.buildContext()), add, gopathwalk.Options{Debug: r.env.Debug, ModulesEnabled: false})
 	return result, nil
 }
 
-func (r *gopathResolver) loadExports(ctx context.Context, expectPackage string, pkg *pkg) (map[string]bool, error) {
-	return loadExportsFromFiles(ctx, r.env, expectPackage, pkg.dir)
+func filterRoots(roots []gopathwalk.Root, exclude []gopathwalk.RootType) []gopathwalk.Root {
+	var result []gopathwalk.Root
+outer:
+	for _, root := range roots {
+		for _, i := range exclude {
+			if i == root.Type {
+				continue outer
+			}
+		}
+		result = append(result, root)
+	}
+	return result
+}
+
+func (r *gopathResolver) loadExports(ctx context.Context, pkg *pkg) (string, []string, error) {
+	r.init()
+	if info, ok := r.cache.Load(pkg.dir); ok {
+		return r.cache.CacheExports(ctx, r.env, info)
+	}
+	return loadExportsFromFiles(ctx, r.env, pkg.dir)
 }
 
 // VendorlessPath returns the devendorized version of the import path ipath.
@@ -1084,13 +1261,13 @@ func VendorlessPath(ipath string) string {
 	return ipath
 }
 
-func loadExportsFromFiles(ctx context.Context, env *ProcessEnv, expectPackage string, dir string) (map[string]bool, error) {
-	exports := make(map[string]bool)
+func loadExportsFromFiles(ctx context.Context, env *ProcessEnv, dir string) (string, []string, error) {
+	var exports []string
 
 	// Look for non-test, buildable .go files which could provide exports.
 	all, err := ioutil.ReadDir(dir)
 	if err != nil {
-		return nil, err
+		return "", nil, err
 	}
 	var files []os.FileInfo
 	for _, fi := range all {
@@ -1106,47 +1283,42 @@ func loadExportsFromFiles(ctx context.Context, env *ProcessEnv, expectPackage st
 	}
 
 	if len(files) == 0 {
-		return nil, fmt.Errorf("dir %v contains no buildable, non-test .go files", dir)
+		return "", nil, fmt.Errorf("dir %v contains no buildable, non-test .go files", dir)
 	}
 
+	var pkgName string
 	fset := token.NewFileSet()
 	for _, fi := range files {
 		select {
 		case <-ctx.Done():
-			return nil, ctx.Err()
+			return "", nil, ctx.Err()
 		default:
 		}
 
 		fullFile := filepath.Join(dir, fi.Name())
 		f, err := parser.ParseFile(fset, fullFile, nil, 0)
 		if err != nil {
-			return nil, fmt.Errorf("parsing %s: %v", fullFile, err)
+			return "", nil, fmt.Errorf("parsing %s: %v", fullFile, err)
 		}
-		pkgName := f.Name.Name
-		if pkgName == "documentation" {
+		if f.Name.Name == "documentation" {
 			// Special case from go/build.ImportDir, not
 			// handled by MatchFile above.
 			continue
 		}
-		if pkgName != expectPackage {
-			return nil, fmt.Errorf("scan of dir %v is not expected package %v (actually %v)", dir, expectPackage, pkgName)
-		}
+		pkgName = f.Name.Name
 		for name := range f.Scope.Objects {
 			if ast.IsExported(name) {
-				exports[name] = true
+				exports = append(exports, name)
 			}
 		}
 	}
 
 	if env.Debug {
-		exportList := make([]string, 0, len(exports))
-		for k := range exports {
-			exportList = append(exportList, k)
-		}
-		sort.Strings(exportList)
-		env.Logf("loaded exports in dir %v (package %v): %v", dir, expectPackage, strings.Join(exportList, ", "))
+		sortedExports := append([]string(nil), exports...)
+		sort.Strings(sortedExports)
+		env.Logf("loaded exports in dir %v (package %v): %v", dir, pkgName, strings.Join(sortedExports, ", "))
 	}
-	return exports, nil
+	return pkgName, exports, nil
 }
 
 // findImport searches for a package with the given symbols.
@@ -1221,7 +1393,7 @@ func findImport(ctx context.Context, pass *pass, dirScan []*pkg, pkgName string,
 				if pass.env.Debug {
 					pass.env.Logf("loading exports in dir %s (seeking package %s)", c.pkg.dir, pkgName)
 				}
-				exports, err := pass.env.GetResolver().loadExports(ctx, pkgName, c.pkg)
+				exports, err := loadExportsForPackage(ctx, pass.env, pkgName, c.pkg)
 				if err != nil {
 					if pass.env.Debug {
 						pass.env.Logf("loading exports in dir %s (seeking package %s): %v", c.pkg.dir, pkgName, err)
@@ -1230,10 +1402,15 @@ func findImport(ctx context.Context, pass *pass, dirScan []*pkg, pkgName string,
 					return
 				}
 
+				exportsMap := make(map[string]bool, len(exports))
+				for _, sym := range exports {
+					exportsMap[sym] = true
+				}
+
 				// If it doesn't have the right
 				// symbols, send nil to mean no match.
 				for symbol := range symbols {
-					if !exports[symbol] {
+					if !exportsMap[symbol] {
 						resc <- nil
 						return
 					}
@@ -1253,6 +1430,17 @@ func findImport(ctx context.Context, pass *pass, dirScan []*pkg, pkgName string,
 	return nil, nil
 }
 
+func loadExportsForPackage(ctx context.Context, env *ProcessEnv, expectPkg string, pkg *pkg) ([]string, error) {
+	pkgName, exports, err := env.GetResolver().loadExports(ctx, pkg)
+	if err != nil {
+		return nil, err
+	}
+	if expectPkg != pkgName {
+		return nil, fmt.Errorf("dir %v is package %v, wanted %v", pkg.dir, pkgName, expectPkg)
+	}
+	return exports, err
+}
+
 // pkgIsCandidate reports whether pkg is a candidate for satisfying the
 // finding which package pkgIdent in the file named by filename is trying
 // to refer to.
@@ -1383,3 +1571,11 @@ type visitFn func(node ast.Node) ast.Visitor
 func (fn visitFn) Visit(node ast.Node) ast.Visitor {
 	return fn(node)
 }
+
+func copyExports(pkg []string) map[string]bool {
+	m := make(map[string]bool, len(pkg))
+	for _, v := range pkg {
+		m[v] = true
+	}
+	return m
+}

vendor/golang.org/x/tools/internal/imports/imports.go

@@ -83,35 +83,56 @@ func FixImports(filename string, src []byte, opt *Options) (fixes []*ImportFix,
 	return getFixes(fileSet, file, filename, opt.Env)
 }
 
-// ApplyFix will apply all of the fixes to the file and format it.
-func ApplyFixes(fixes []*ImportFix, filename string, src []byte, opt *Options) (formatted []byte, err error) {
+// ApplyFixes applies all of the fixes to the file and formats it. extraMode
+// is added in when parsing the file.
+func ApplyFixes(fixes []*ImportFix, filename string, src []byte, opt *Options, extraMode parser.Mode) (formatted []byte, err error) {
 	src, opt, err = initialize(filename, src, opt)
 	if err != nil {
 		return nil, err
 	}
 
+	// Don't use parse() -- we don't care about fragments or statement lists
+	// here, and we need to work with unparseable files.
 	fileSet := token.NewFileSet()
-	file, adjust, err := parse(fileSet, filename, src, opt)
-	if err != nil {
+	parserMode := parser.Mode(0)
+	if opt.Comments {
+		parserMode |= parser.ParseComments
+	}
+	if opt.AllErrors {
+		parserMode |= parser.AllErrors
+	}
+	parserMode |= extraMode
+
+	file, err := parser.ParseFile(fileSet, filename, src, parserMode)
+	if file == nil {
 		return nil, err
 	}
 
 	// Apply the fixes to the file.
 	apply(fileSet, file, fixes)
 
-	return formatFile(fileSet, file, src, adjust, opt)
+	return formatFile(fileSet, file, src, nil, opt)
 }
 
 // GetAllCandidates gets all of the standard library candidate packages to import in
 // sorted order on import path.
 func GetAllCandidates(filename string, opt *Options) (pkgs []ImportFix, err error) {
-	_, opt, err = initialize(filename, []byte{}, opt)
+	_, opt, err = initialize(filename, nil, opt)
 	if err != nil {
 		return nil, err
 	}
 	return getAllCandidates(filename, opt.Env)
 }
 
+// GetPackageExports returns all known packages with name pkg and their exports.
+func GetPackageExports(pkg, filename string, opt *Options) (exports []PackageExport, err error) {
+	_, opt, err = initialize(filename, nil, opt)
+	if err != nil {
+		return nil, err
+	}
+	return getPackageExports(pkg, filename, opt.Env)
+}
+
 // initialize sets the values for opt and src.
 // If they are provided, they are not changed. Otherwise opt is set to the
 // default values and src is read from the file system.

vendor/golang.org/x/tools/internal/imports/mod.go

@@ -14,10 +14,10 @@ import (
 	"strconv"
 	"strings"
 	"sync"
-	"time"
 
 	"golang.org/x/tools/internal/gopathwalk"
 	"golang.org/x/tools/internal/module"
+	"golang.org/x/tools/internal/semver"
 )
 
 // ModuleResolver implements resolver for modules using the go command as little
@@ -25,38 +25,81 @@ import (
 type ModuleResolver struct {
 	env            *ProcessEnv
 	moduleCacheDir string
+	dummyVendorMod *ModuleJSON // If vendoring is enabled, the pseudo-module that represents the /vendor directory.
 
 	Initialized   bool
 	Main          *ModuleJSON
 	ModsByModPath []*ModuleJSON // All modules, ordered by # of path components in module Path...
 	ModsByDir     []*ModuleJSON // ...or Dir.
 
-	// moduleCacheInfo stores information about the module cache.
-	moduleCacheInfo *moduleCacheInfo
+	// moduleCacheCache stores information about the module cache.
+	moduleCacheCache *dirInfoCache
+	otherCache       *dirInfoCache
 }
 
 type ModuleJSON struct {
-	Path     string           // module path
-	Version  string           // module version
-	Versions []string         // available module versions (with -versions)
-	Replace  *ModuleJSON      // replaced by this module
-	Time     *time.Time       // time version was created
-	Update   *ModuleJSON      // available update, if any (with -u)
-	Main     bool             // is this the main module?
-	Indirect bool             // is this module only an indirect dependency of main module?
-	Dir      string           // directory holding files for this module, if any
-	GoMod    string           // path to go.mod file for this module, if any
-	Error    *ModuleErrorJSON // error loading module
-}
-
-type ModuleErrorJSON struct {
-	Err string // the error itself
+	Path      string      // module path
+	Replace   *ModuleJSON // replaced by this module
+	Main      bool        // is this the main module?
+	Dir       string      // directory holding files for this module, if any
+	GoMod     string      // path to go.mod file for this module, if any
+	GoVersion string      // go version used in module
 }
 
 func (r *ModuleResolver) init() error {
 	if r.Initialized {
 		return nil
 	}
+	mainMod, vendorEnabled, err := vendorEnabled(r.env)
+	if err != nil {
+		return err
+	}
+
+	if mainMod != nil && vendorEnabled {
+		// Vendor mode is on, so all the non-Main modules are irrelevant,
+		// and we need to search /vendor for everything.
+		r.Main = mainMod
+		r.dummyVendorMod = &ModuleJSON{
+			Path: "",
+			Dir:  filepath.Join(mainMod.Dir, "vendor"),
+		}
+		r.ModsByModPath = []*ModuleJSON{mainMod, r.dummyVendorMod}
+		r.ModsByDir = []*ModuleJSON{mainMod, r.dummyVendorMod}
+	} else {
+		// Vendor mode is off, so run go list -m ... to find everything.
+		r.initAllMods()
+	}
+
+	r.moduleCacheDir = filepath.Join(filepath.SplitList(r.env.GOPATH)[0], "/pkg/mod")
+
+	sort.Slice(r.ModsByModPath, func(i, j int) bool {
+		count := func(x int) int {
+			return strings.Count(r.ModsByModPath[x].Path, "/")
+		}
+		return count(j) < count(i) // descending order
+	})
+	sort.Slice(r.ModsByDir, func(i, j int) bool {
+		count := func(x int) int {
+			return strings.Count(r.ModsByDir[x].Dir, "/")
+		}
+		return count(j) < count(i) // descending order
+	})
+
+	if r.moduleCacheCache == nil {
+		r.moduleCacheCache = &dirInfoCache{
+			dirs: map[string]*directoryPackageInfo{},
+		}
+	}
+	if r.otherCache == nil {
+		r.otherCache = &dirInfoCache{
+			dirs: map[string]*directoryPackageInfo{},
+		}
+	}
+	r.Initialized = true
+	return nil
+}
+
+func (r *ModuleResolver) initAllMods() error {
 	stdout, err := r.env.invokeGo("list", "-m", "-json", "...")
 	if err != nil {
 		return err
@@ -79,30 +122,23 @@ func (r *ModuleResolver) init() error {
 			r.Main = mod
 		}
 	}
-
-	sort.Slice(r.ModsByModPath, func(i, j int) bool {
-		count := func(x int) int {
-			return strings.Count(r.ModsByModPath[x].Path, "/")
-		}
-		return count(j) < count(i) // descending order
-	})
-	sort.Slice(r.ModsByDir, func(i, j int) bool {
-		count := func(x int) int {
-			return strings.Count(r.ModsByDir[x].Dir, "/")
-		}
-		return count(j) < count(i) // descending order
-	})
-
-	if r.moduleCacheInfo == nil {
-		r.moduleCacheInfo = &moduleCacheInfo{
-			modCacheDirInfo: make(map[string]*directoryPackageInfo),
-		}
-	}
-
-	r.Initialized = true
 	return nil
 }
 
+func (r *ModuleResolver) ClearForNewScan() {
+	r.otherCache = &dirInfoCache{
+		dirs: map[string]*directoryPackageInfo{},
+	}
+}
+
+func (r *ModuleResolver) ClearForNewMod() {
+	env := r.env
+	*r = ModuleResolver{
+		env: env,
+	}
+	r.init()
+}
+
 // findPackage returns the module and directory that contains the package at
 // the given import path, or returns nil, "" if no module is in scope.
 func (r *ModuleResolver) findPackage(importPath string) (*ModuleJSON, string) {
@@ -118,22 +154,31 @@ func (r *ModuleResolver) findPackage(importPath string) (*ModuleJSON, string) {
 			continue
 		}
 
-		if info, ok := r.moduleCacheInfo.Load(pkgDir); ok {
-			if packageScanned, err := info.reachedStatus(directoryScanned); packageScanned {
+		if info, ok := r.cacheLoad(pkgDir); ok {
+			if loaded, err := info.reachedStatus(nameLoaded); loaded {
 				if err != nil {
-					// There was some error with scanning this directory.
-					// It does not contain a valid package.
-					continue
+					continue // No package in this dir.
 				}
 				return m, pkgDir
 			}
+			if scanned, err := info.reachedStatus(directoryScanned); scanned && err != nil {
+				continue // Dir is unreadable, etc.
+			}
+			// This is slightly wrong: a directory doesn't have to have an
+			// importable package to count as a package for package-to-module
+			// resolution. package main or _test files should count but
+			// don't.
+			// TODO(heschi): fix this.
+			if _, err := r.cachePackageName(info); err == nil {
+				return m, pkgDir
+			}
 		}
 
+		// Not cached. Read the filesystem.
 		pkgFiles, err := ioutil.ReadDir(pkgDir)
 		if err != nil {
 			continue
 		}
-
 		// A module only contains a package if it has buildable go
 		// files in that directory. If not, it could be provided by an
 		// outer module. See #29736.
@@ -146,6 +191,40 @@ func (r *ModuleResolver) findPackage(importPath string) (*ModuleJSON, string) {
 	return nil, ""
 }
 
+func (r *ModuleResolver) cacheLoad(dir string) (directoryPackageInfo, bool) {
+	if info, ok := r.moduleCacheCache.Load(dir); ok {
+		return info, ok
+	}
+	return r.otherCache.Load(dir)
+}
+
+func (r *ModuleResolver) cacheStore(info directoryPackageInfo) {
+	if info.rootType == gopathwalk.RootModuleCache {
+		r.moduleCacheCache.Store(info.dir, info)
+	} else {
+		r.otherCache.Store(info.dir, info)
+	}
+}
+
+func (r *ModuleResolver) cacheKeys() []string {
+	return append(r.moduleCacheCache.Keys(), r.otherCache.Keys()...)
+}
+
+// cachePackageName caches the package name for a dir already in the cache.
+func (r *ModuleResolver) cachePackageName(info directoryPackageInfo) (directoryPackageInfo, error) {
+	if info.rootType == gopathwalk.RootModuleCache {
+		return r.moduleCacheCache.CachePackageName(info)
+	}
+	return r.otherCache.CachePackageName(info)
+}
+
+func (r *ModuleResolver) cacheExports(ctx context.Context, env *ProcessEnv, info directoryPackageInfo) (string, []string, error) {
+	if info.rootType == gopathwalk.RootModuleCache {
+		return r.moduleCacheCache.CacheExports(ctx, env, info)
+	}
+	return r.otherCache.CacheExports(ctx, env, info)
+}
+
 // findModuleByDir returns the module that contains dir, or nil if no such
 // module is in scope.
 func (r *ModuleResolver) findModuleByDir(dir string) *ModuleJSON {
@@ -184,28 +263,45 @@ func (r *ModuleResolver) dirIsNestedModule(dir string, mod *ModuleJSON) bool {
 		// so it cannot be a nested module.
 		return false
 	}
-	mf := r.findModFile(dir)
-	if mf == "" {
+	if mod != nil && mod == r.dummyVendorMod {
+		// The /vendor pseudomodule is flattened and doesn't actually count.
 		return false
 	}
-	return filepath.Dir(mf) != mod.Dir
+	modDir, _ := r.modInfo(dir)
+	if modDir == "" {
+		return false
+	}
+	return modDir != mod.Dir
 }
 
-func (r *ModuleResolver) findModFile(dir string) string {
+func (r *ModuleResolver) modInfo(dir string) (modDir string, modName string) {
+	readModName := func(modFile string) string {
+		modBytes, err := ioutil.ReadFile(modFile)
+		if err != nil {
+			return ""
+		}
+		return modulePath(modBytes)
+	}
+
 	if r.dirInModuleCache(dir) {
 		matches := modCacheRegexp.FindStringSubmatch(dir)
 		index := strings.Index(dir, matches[1]+"@"+matches[2])
-		return filepath.Join(dir[:index], matches[1]+"@"+matches[2], "go.mod")
+		modDir := filepath.Join(dir[:index], matches[1]+"@"+matches[2])
+		return modDir, readModName(filepath.Join(modDir, "go.mod"))
 	}
 	for {
+		if info, ok := r.cacheLoad(dir); ok {
+			return info.moduleDir, info.moduleName
+		}
 		f := filepath.Join(dir, "go.mod")
 		info, err := os.Stat(f)
 		if err == nil && !info.IsDir() {
-			return f
+			return dir, readModName(f)
 		}
+
 		d := filepath.Dir(dir)
 		if len(d) >= len(dir) {
-			return "" // reached top of file system, no go.mod
+			return "", "" // reached top of file system, no go.mod
 		}
 		dir = d
 	}
@@ -237,7 +333,7 @@ func (r *ModuleResolver) loadPackageNames(importPaths []string, srcDir string) (
 	return names, nil
 }
 
-func (r *ModuleResolver) scan(_ references) ([]*pkg, error) {
+func (r *ModuleResolver) scan(_ references, loadNames bool, exclude []gopathwalk.RootType) ([]*pkg, error) {
 	if err := r.init(); err != nil {
 		return nil, err
 	}
@@ -249,35 +345,30 @@ func (r *ModuleResolver) scan(_ references) ([]*pkg, error) {
 	if r.Main != nil {
 		roots = append(roots, gopathwalk.Root{r.Main.Dir, gopathwalk.RootCurrentModule})
 	}
-	if r.moduleCacheDir == "" {
-		r.moduleCacheDir = filepath.Join(filepath.SplitList(r.env.GOPATH)[0], "/pkg/mod")
-	}
-	roots = append(roots, gopathwalk.Root{r.moduleCacheDir, gopathwalk.RootModuleCache})
-
-	// Walk replace targets, just in case they're not in any of the above.
-	for _, mod := range r.ModsByModPath {
-		if mod.Replace != nil {
-			roots = append(roots, gopathwalk.Root{mod.Dir, gopathwalk.RootOther})
+	if r.dummyVendorMod != nil {
+		roots = append(roots, gopathwalk.Root{r.dummyVendorMod.Dir, gopathwalk.RootOther})
+	} else {
+		roots = append(roots, gopathwalk.Root{r.moduleCacheDir, gopathwalk.RootModuleCache})
+		// Walk replace targets, just in case they're not in any of the above.
+		for _, mod := range r.ModsByModPath {
+			if mod.Replace != nil {
+				roots = append(roots, gopathwalk.Root{mod.Dir, gopathwalk.RootOther})
+			}
 		}
 	}
 
+	roots = filterRoots(roots, exclude)
+
 	var result []*pkg
-	dupCheck := make(map[string]bool)
 	var mu sync.Mutex
 
-	// Packages in the module cache are immutable. If we have
-	// already seen this package on a previous scan of the module
-	// cache, return that result.
+	// We assume cached directories have not changed. We can skip them and their
+	// children.
 	skip := func(root gopathwalk.Root, dir string) bool {
 		mu.Lock()
 		defer mu.Unlock()
-		// If we have already processed this directory on this walk, skip it.
-		if _, dup := dupCheck[dir]; dup {
-			return true
-		}
 
-		// If we have saved this directory information, skip it.
-		info, ok := r.moduleCacheInfo.Load(dir)
+		info, ok := r.cacheLoad(dir)
 		if !ok {
 			return false
 		}
@@ -288,44 +379,19 @@ func (r *ModuleResolver) scan(_ references) ([]*pkg, error) {
 		return packageScanned
 	}
 
+	// Add anything new to the cache. We'll process everything in it below.
 	add := func(root gopathwalk.Root, dir string) {
 		mu.Lock()
 		defer mu.Unlock()
-		if _, dup := dupCheck[dir]; dup {
-			return
-		}
 
-		info, err := r.scanDirForPackage(root, dir)
-		if err != nil {
-			return
-		}
-		if root.Type == gopathwalk.RootModuleCache {
-			// Save this package information in the cache and return.
-			// Packages from the module cache are added after Walk.
-			r.moduleCacheInfo.Store(dir, info)
-			return
-		}
-
-		// Skip this package if there was an error loading package info.
-		if info.err != nil {
-			return
-		}
-
-		// The rest of this function canonicalizes the packages using the results
-		// of initializing the resolver from 'go list -m'.
-		res, err := r.canonicalize(info.nonCanonicalImportPath, info.dir, info.needsReplace)
-		if err != nil {
-			return
-		}
-
-		result = append(result, res)
+		r.cacheStore(r.scanDirForPackage(root, dir))
 	}
 
 	gopathwalk.WalkSkip(roots, add, skip, gopathwalk.Options{Debug: r.env.Debug, ModulesEnabled: true})
 
-	// Add the packages from the modules in the mod cache that were skipped.
-	for _, dir := range r.moduleCacheInfo.Keys() {
-		info, ok := r.moduleCacheInfo.Load(dir)
+	// Everything we already had, and everything new, is now in the cache.
+	for _, dir := range r.cacheKeys() {
+		info, ok := r.cacheLoad(dir)
 		if !ok {
 			continue
 		}
@@ -335,7 +401,15 @@ func (r *ModuleResolver) scan(_ references) ([]*pkg, error) {
 			continue
 		}
 
-		res, err := r.canonicalize(info.nonCanonicalImportPath, info.dir, info.needsReplace)
+		// If we want package names, make sure the cache has them.
+		if loadNames {
+			var err error
+			if info, err = r.cachePackageName(info); err != nil {
+				continue
+			}
+		}
+
+		res, err := r.canonicalize(info)
 		if err != nil {
 			continue
 		}
@@ -347,52 +421,70 @@ func (r *ModuleResolver) scan(_ references) ([]*pkg, error) {
 
 // canonicalize gets the result of canonicalizing the packages using the results
 // of initializing the resolver from 'go list -m'.
-func (r *ModuleResolver) canonicalize(importPath, dir string, needsReplace bool) (res *pkg, err error) {
-	// Check if the directory is underneath a module that's in scope.
-	if mod := r.findModuleByDir(dir); mod != nil {
-		// It is. If dir is the target of a replace directive,
-		// our guessed import path is wrong. Use the real one.
-		if mod.Dir == dir {
-			importPath = mod.Path
-		} else {
-			dirInMod := dir[len(mod.Dir)+len("/"):]
-			importPath = path.Join(mod.Path, filepath.ToSlash(dirInMod))
-		}
-	} else if needsReplace {
-		return nil, fmt.Errorf("needed this package to be in scope: %s", dir)
+func (r *ModuleResolver) canonicalize(info directoryPackageInfo) (*pkg, error) {
+	// Packages in GOROOT are already canonical, regardless of the std/cmd modules.
+	if info.rootType == gopathwalk.RootGOROOT {
+		return &pkg{
+			importPathShort: info.nonCanonicalImportPath,
+			dir:             info.dir,
+			packageName:     path.Base(info.nonCanonicalImportPath),
+			relevance:       0,
+		}, nil
 	}
 
+	importPath := info.nonCanonicalImportPath
+	relevance := 2
+	// Check if the directory is underneath a module that's in scope.
+	if mod := r.findModuleByDir(info.dir); mod != nil {
+		relevance = 1
+		// It is. If dir is the target of a replace directive,
+		// our guessed import path is wrong. Use the real one.
+		if mod.Dir == info.dir {
+			importPath = mod.Path
+		} else {
+			dirInMod := info.dir[len(mod.Dir)+len("/"):]
+			importPath = path.Join(mod.Path, filepath.ToSlash(dirInMod))
+		}
+	} else if info.needsReplace {
+		return nil, fmt.Errorf("package in %q is not valid without a replace statement", info.dir)
+	}
+
+	res := &pkg{
+		importPathShort: importPath,
+		dir:             info.dir,
+		packageName:     info.packageName, // may not be populated if the caller didn't ask for it
+		relevance:       relevance,
+	}
 	// We may have discovered a package that has a different version
 	// in scope already. Canonicalize to that one if possible.
 	if _, canonicalDir := r.findPackage(importPath); canonicalDir != "" {
-		dir = canonicalDir
+		res.dir = canonicalDir
 	}
-	return &pkg{
-		importPathShort: VendorlessPath(importPath),
-		dir:             dir,
-	}, nil
+	return res, nil
 }
 
-func (r *ModuleResolver) loadExports(ctx context.Context, expectPackage string, pkg *pkg) (map[string]bool, error) {
+func (r *ModuleResolver) loadExports(ctx context.Context, pkg *pkg) (string, []string, error) {
 	if err := r.init(); err != nil {
-		return nil, err
+		return "", nil, err
 	}
-	return loadExportsFromFiles(ctx, r.env, expectPackage, pkg.dir)
+	if info, ok := r.cacheLoad(pkg.dir); ok {
+		return r.cacheExports(ctx, r.env, info)
+	}
+	return loadExportsFromFiles(ctx, r.env, pkg.dir)
 }
 
-func (r *ModuleResolver) scanDirForPackage(root gopathwalk.Root, dir string) (directoryPackageInfo, error) {
+func (r *ModuleResolver) scanDirForPackage(root gopathwalk.Root, dir string) directoryPackageInfo {
 	subdir := ""
 	if dir != root.Path {
 		subdir = dir[len(root.Path)+len("/"):]
 	}
 	importPath := filepath.ToSlash(subdir)
 	if strings.HasPrefix(importPath, "vendor/") {
-		// Ignore vendor dirs. If -mod=vendor is on, then things
-		// should mostly just work, but when it's not vendor/
-		// is a mess. There's no easy way to tell if it's on.
-		// We can still find things in the mod cache and
-		// map them into /vendor when -mod=vendor is on.
-		return directoryPackageInfo{}, fmt.Errorf("vendor directory")
+		// Only enter vendor directories if they're explicitly requested as a root.
+		return directoryPackageInfo{
+			status: directoryScanned,
+			err:    fmt.Errorf("unwanted vendor directory"),
+		}
 	}
 	switch root.Type {
 	case gopathwalk.RootCurrentModule:
@@ -403,7 +495,7 @@ func (r *ModuleResolver) scanDirForPackage(root gopathwalk.Root, dir string) (di
 			return directoryPackageInfo{
 				status: directoryScanned,
 				err:    fmt.Errorf("invalid module cache path: %v", subdir),
-			}, nil
+			}
 		}
 		modPath, err := module.DecodePath(filepath.ToSlash(matches[1]))
 		if err != nil {
@@ -413,35 +505,34 @@ func (r *ModuleResolver) scanDirForPackage(root gopathwalk.Root, dir string) (di
 			return directoryPackageInfo{
 				status: directoryScanned,
 				err:    fmt.Errorf("decoding module cache path %q: %v", subdir, err),
-			}, nil
+			}
 		}
 		importPath = path.Join(modPath, filepath.ToSlash(matches[3]))
-	case gopathwalk.RootGOROOT:
-		importPath = subdir
 	}
 
+	modDir, modName := r.modInfo(dir)
 	result := directoryPackageInfo{
 		status:                 directoryScanned,
 		dir:                    dir,
+		rootType:               root.Type,
 		nonCanonicalImportPath: importPath,
 		needsReplace:           false,
+		moduleDir:              modDir,
+		moduleName:             modName,
 	}
 	if root.Type == gopathwalk.RootGOROOT {
 		// stdlib packages are always in scope, despite the confusing go.mod
-		return result, nil
+		return result
 	}
 	// Check that this package is not obviously impossible to import.
-	modFile := r.findModFile(dir)
-
-	modBytes, err := ioutil.ReadFile(modFile)
-	if err == nil && !strings.HasPrefix(importPath, modulePath(modBytes)) {
+	if !strings.HasPrefix(importPath, modName) {
 		// The module's declared path does not match
 		// its expected path. It probably needs a
 		// replace directive we don't have.
 		result.needsReplace = true
 	}
 
-	return result, nil
+	return result
 }
 
 // modCacheRegexp splits a path in a module cache into module, module version, and package.
@@ -490,3 +581,63 @@ func modulePath(mod []byte) string {
 	}
 	return "" // missing module path
 }
+
+var modFlagRegexp = regexp.MustCompile(`-mod[ =](\w+)`)
+
+// vendorEnabled indicates if vendoring is enabled.
+// Inspired by setDefaultBuildMod in modload/init.go
+func vendorEnabled(env *ProcessEnv) (*ModuleJSON, bool, error) {
+	mainMod, go114, err := getMainModuleAnd114(env)
+	if err != nil {
+		return nil, false, err
+	}
+	matches := modFlagRegexp.FindStringSubmatch(env.GOFLAGS)
+	var modFlag string
+	if len(matches) != 0 {
+		modFlag = matches[1]
+	}
+	if modFlag != "" {
+		// Don't override an explicit '-mod=' argument.
+		return mainMod, modFlag == "vendor", nil
+	}
+	if mainMod == nil || !go114 {
+		return mainMod, false, nil
+	}
+	// Check 1.14's automatic vendor mode.
+	if fi, err := os.Stat(filepath.Join(mainMod.Dir, "vendor")); err == nil && fi.IsDir() {
+		if mainMod.GoVersion != "" && semver.Compare("v"+mainMod.GoVersion, "v1.14") >= 0 {
+			// The Go version is at least 1.14, and a vendor directory exists.
+			// Set -mod=vendor by default.
+			return mainMod, true, nil
+		}
+	}
+	return mainMod, false, nil
+}
+
+// getMainModuleAnd114 gets the main module's information and whether the
+// go command in use is 1.14+. This is the information needed to figure out
+// if vendoring should be enabled.
+func getMainModuleAnd114(env *ProcessEnv) (*ModuleJSON, bool, error) {
+	const format = `{{.Path}}
+{{.Dir}}
+{{.GoMod}}
+{{.GoVersion}}
+{{range context.ReleaseTags}}{{if eq . "go1.14"}}{{.}}{{end}}{{end}}
+`
+	stdout, err := env.invokeGo("list", "-m", "-f", format)
+	if err != nil {
+		return nil, false, nil
+	}
+	lines := strings.Split(stdout.String(), "\n")
+	if len(lines) < 5 {
+		return nil, false, fmt.Errorf("unexpected stdout: %q", stdout)
+	}
+	mod := &ModuleJSON{
+		Path:      lines[0],
+		Dir:       lines[1],
+		GoMod:     lines[2],
+		GoVersion: lines[3],
+		Main:      true,
+	}
+	return mod, lines[4] == "go1.14", nil
+}

vendor/golang.org/x/tools/internal/imports/mod_cache.go

@@ -1,12 +1,13 @@
 package imports
 
 import (
+	"context"
+	"fmt"
 	"sync"
+
+	"golang.org/x/tools/internal/gopathwalk"
 )
 
-// ModuleResolver implements Resolver for modules using the go command as little
-// as feasible.
-//
 // To find packages to import, the resolver needs to know about all of the
 // the packages that could be imported. This includes packages that are
 // already in modules that are in (1) the current module, (2) replace targets,
@@ -30,6 +31,8 @@ type directoryPackageStatus int
 const (
 	_ directoryPackageStatus = iota
 	directoryScanned
+	nameLoaded
+	exportsLoaded
 )
 
 type directoryPackageInfo struct {
@@ -38,17 +41,30 @@ type directoryPackageInfo struct {
 	// err is non-nil when there was an error trying to reach status.
 	err error
 
-	// Set when status > directoryScanned.
+	// Set when status >= directoryScanned.
 
 	// dir is the absolute directory of this package.
-	dir string
-	// nonCanonicalImportPath is the expected import path for this package.
-	// This may not be an import path that can be used to import this package.
+	dir      string
+	rootType gopathwalk.RootType
+	// nonCanonicalImportPath is the package's expected import path. It may
+	// not actually be importable at that path.
 	nonCanonicalImportPath string
 	// needsReplace is true if the nonCanonicalImportPath does not match the
-	// the modules declared path, making it impossible to import without a
+	// module's declared path, making it impossible to import without a
 	// replace directive.
 	needsReplace bool
+
+	// Module-related information.
+	moduleDir  string // The directory that is the module root of this dir.
+	moduleName string // The module name that contains this dir.
+
+	// Set when status >= nameLoaded.
+
+	packageName string // the package name, as declared in the source.
+
+	// Set when status >= exportsLoaded.
+
+	exports []string
 }
 
 // reachedStatus returns true when info has a status at least target and any error associated with
@@ -63,8 +79,8 @@ func (info *directoryPackageInfo) reachedStatus(target directoryPackageStatus) (
 	return true, nil
 }
 
-// moduleCacheInfo is a concurrency safe map for storing information about
-// the directories in the module cache.
+// dirInfoCache is a concurrency safe map for storing information about
+// directories that may contain packages.
 //
 // The information in this cache is built incrementally. Entries are initialized in scan.
 // No new keys should be added in any other functions, as all directories containing
@@ -73,37 +89,30 @@ func (info *directoryPackageInfo) reachedStatus(target directoryPackageStatus) (
 // Other functions, including loadExports and findPackage, may update entries in this cache
 // as they discover new things about the directory.
 //
-// We do not need to protect the data in the cache for multiple writes, because it only stores
-// module cache directories, which do not change. If two competing stores take place, there will be
-// one store that wins. Although this could result in a loss of information it will not be incorrect
-// and may just result in recomputing the same result later.
+// The information in the cache is not expected to change for the cache's
+// lifetime, so there is no protection against competing writes. Users should
+// take care not to hold the cache across changes to the underlying files.
 //
 // TODO(suzmue): consider other concurrency strategies and data structures (RWLocks, sync.Map, etc)
-type moduleCacheInfo struct {
+type dirInfoCache struct {
 	mu sync.Mutex
-	// modCacheDirInfo stores information about packages in
-	// module cache directories. Keyed by absolute directory.
-	modCacheDirInfo map[string]*directoryPackageInfo
+	// dirs stores information about packages in directories, keyed by absolute path.
+	dirs map[string]*directoryPackageInfo
 }
 
 // Store stores the package info for dir.
-func (d *moduleCacheInfo) Store(dir string, info directoryPackageInfo) {
+func (d *dirInfoCache) Store(dir string, info directoryPackageInfo) {
 	d.mu.Lock()
 	defer d.mu.Unlock()
-	d.modCacheDirInfo[dir] = &directoryPackageInfo{
-		status:                 info.status,
-		err:                    info.err,
-		dir:                    info.dir,
-		nonCanonicalImportPath: info.nonCanonicalImportPath,
-		needsReplace:           info.needsReplace,
-	}
+	stored := info // defensive copy
+	d.dirs[dir] = &stored
 }
 
 // Load returns a copy of the directoryPackageInfo for absolute directory dir.
-func (d *moduleCacheInfo) Load(dir string) (directoryPackageInfo, bool) {
+func (d *dirInfoCache) Load(dir string) (directoryPackageInfo, bool) {
 	d.mu.Lock()
 	defer d.mu.Unlock()
-	info, ok := d.modCacheDirInfo[dir]
+	info, ok := d.dirs[dir]
 	if !ok {
 		return directoryPackageInfo{}, false
 	}
@@ -111,11 +120,46 @@ func (d *moduleCacheInfo) Load(dir string) (directoryPackageInfo, bool) {
 }
 
 // Keys returns the keys currently present in d.
-func (d *moduleCacheInfo) Keys() (keys []string) {
+func (d *dirInfoCache) Keys() (keys []string) {
 	d.mu.Lock()
 	defer d.mu.Unlock()
-	for key := range d.modCacheDirInfo {
+	for key := range d.dirs {
 		keys = append(keys, key)
 	}
 	return keys
 }
+
+func (d *dirInfoCache) CachePackageName(info directoryPackageInfo) (directoryPackageInfo, error) {
+	if loaded, err := info.reachedStatus(nameLoaded); loaded {
+		return info, err
+	}
+	if scanned, err := info.reachedStatus(directoryScanned); !scanned || err != nil {
+		return info, fmt.Errorf("cannot read package name, scan error: %v", err)
+	}
+	info.packageName, info.err = packageDirToName(info.dir)
+	info.status = nameLoaded
+	d.Store(info.dir, info)
+	return info, info.err
+}
+
+func (d *dirInfoCache) CacheExports(ctx context.Context, env *ProcessEnv, info directoryPackageInfo) (string, []string, error) {
+	if reached, _ := info.reachedStatus(exportsLoaded); reached {
+		return info.packageName, info.exports, info.err
+	}
+	if reached, err := info.reachedStatus(nameLoaded); reached && err != nil {
+		return "", nil, err
+	}
+	info.packageName, info.exports, info.err = loadExportsFromFiles(ctx, env, info.dir)
+	if info.err == context.Canceled {
+		return info.packageName, info.exports, info.err
+	}
+	// The cache structure wants things to proceed linearly. We can skip a
+	// step here, but only if we succeed.
+	if info.status == nameLoaded || info.err == nil {
+		info.status = exportsLoaded
+	} else {
+		info.status = nameLoaded
+	}
+	d.Store(info.dir, info)
+	return info.packageName, info.exports, info.err
+}

vendor/modules.txt

@@ -439,7 +439,7 @@ go.mongodb.org/mongo-driver/bson/bsonrw
 go.mongodb.org/mongo-driver/bson/bsontype
 go.mongodb.org/mongo-driver/bson/primitive
 go.mongodb.org/mongo-driver/x/bsonx/bsoncore
-# golang.org/x/crypto v0.0.0-20191117063200-497ca9f6d64f
+# golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413
 golang.org/x/crypto/acme
 golang.org/x/crypto/acme/autocert
 golang.org/x/crypto/argon2
@@ -508,7 +508,7 @@ golang.org/x/text/transform
 golang.org/x/text/unicode/bidi
 golang.org/x/text/unicode/norm
 golang.org/x/text/width
-# golang.org/x/tools v0.0.0-20190910221609-7f5965fd7709
+# golang.org/x/tools v0.0.0-20191213221258-04c2e8eff935
 golang.org/x/tools/go/ast/astutil
 golang.org/x/tools/go/buildutil
 golang.org/x/tools/go/gcexportdata