Update to last common x/text (#3994)

vendor/golang.org/x/text/encoding/japanese/eucjp.go

@@ -5,7 +5,6 @@
 package japanese
 
 import (
-	"errors"
 	"unicode/utf8"
 
 	"golang.org/x/text/encoding"
@@ -23,10 +22,9 @@ var eucJP = internal.Encoding{
 	identifier.EUCPkdFmtJapanese,
 }
 
-var errInvalidEUCJP = errors.New("japanese: invalid EUC-JP encoding")
-
 type eucJPDecoder struct{ transform.NopResetter }
 
+// See https://encoding.spec.whatwg.org/#euc-jp-decoder.
 func (eucJPDecoder) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
 	r, size := rune(0), 0
 loop:
@@ -37,60 +35,79 @@ loop:
 
 		case c0 == 0x8e:
 			if nSrc+1 >= len(src) {
-				err = transform.ErrShortSrc
-				break loop
+				if !atEOF {
+					err = transform.ErrShortSrc
+					break loop
+				}
+				r, size = utf8.RuneError, 1
+				break
 			}
 			c1 := src[nSrc+1]
-			if c1 < 0xa1 || 0xdf < c1 {
-				err = errInvalidEUCJP
-				break loop
+			switch {
+			case c1 < 0xa1:
+				r, size = utf8.RuneError, 1
+			case c1 > 0xdf:
+				r, size = utf8.RuneError, 2
+				if c1 == 0xff {
+					size = 1
+				}
+			default:
+				r, size = rune(c1)+(0xff61-0xa1), 2
 			}
-			r, size = rune(c1)+(0xff61-0xa1), 2
-
 		case c0 == 0x8f:
 			if nSrc+2 >= len(src) {
-				err = transform.ErrShortSrc
-				break loop
+				if !atEOF {
+					err = transform.ErrShortSrc
+					break loop
+				}
+				r, size = utf8.RuneError, 1
+				if p := nSrc + 1; p < len(src) && 0xa1 <= src[p] && src[p] < 0xfe {
+					size = 2
+				}
+				break
 			}
 			c1 := src[nSrc+1]
 			if c1 < 0xa1 || 0xfe < c1 {
-				err = errInvalidEUCJP
-				break loop
+				r, size = utf8.RuneError, 1
+				break
 			}
 			c2 := src[nSrc+2]
 			if c2 < 0xa1 || 0xfe < c2 {
-				err = errInvalidEUCJP
-				break loop
+				r, size = utf8.RuneError, 2
+				break
 			}
-			r, size = '\ufffd', 3
+			r, size = utf8.RuneError, 3
 			if i := int(c1-0xa1)*94 + int(c2-0xa1); i < len(jis0212Decode) {
 				r = rune(jis0212Decode[i])
 				if r == 0 {
-					r = '\ufffd'
+					r = utf8.RuneError
 				}
 			}
 
 		case 0xa1 <= c0 && c0 <= 0xfe:
 			if nSrc+1 >= len(src) {
-				err = transform.ErrShortSrc
-				break loop
+				if !atEOF {
+					err = transform.ErrShortSrc
+					break loop
+				}
+				r, size = utf8.RuneError, 1
+				break
 			}
 			c1 := src[nSrc+1]
 			if c1 < 0xa1 || 0xfe < c1 {
-				err = errInvalidEUCJP
-				break loop
+				r, size = utf8.RuneError, 1
+				break
 			}
-			r, size = '\ufffd', 2
+			r, size = utf8.RuneError, 2
 			if i := int(c0-0xa1)*94 + int(c1-0xa1); i < len(jis0208Decode) {
 				r = rune(jis0208Decode[i])
 				if r == 0 {
-					r = '\ufffd'
+					r = utf8.RuneError
 				}
 			}
 
 		default:
-			err = errInvalidEUCJP
-			break loop
+			r, size = utf8.RuneError, 1
 		}
 
 		if nDst+utf8.RuneLen(r) > len(dst) {
@@ -99,9 +116,6 @@ loop:
 		}
 		nDst += utf8.EncodeRune(dst[nDst:], r)
 	}
-	if atEOF && err == transform.ErrShortSrc {
-		err = errInvalidEUCJP
-	}
 	return nDst, nSrc, err
 }
 

vendor/golang.org/x/text/encoding/japanese/iso2022jp.go

@@ -5,7 +5,6 @@
 package japanese
 
 import (
-	"errors"
 	"unicode/utf8"
 
 	"golang.org/x/text/encoding"
@@ -31,8 +30,6 @@ func iso2022JPNewEncoder() transform.Transformer {
 	return new(iso2022JPEncoder)
 }
 
-var errInvalidISO2022JP = errors.New("japanese: invalid ISO-2022-JP encoding")
-
 const (
 	asciiState = iota
 	katakanaState
@@ -50,45 +47,51 @@ func (d *iso2022JPDecoder) Reset() {
 
 func (d *iso2022JPDecoder) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
 	r, size := rune(0), 0
-loop:
 	for ; nSrc < len(src); nSrc += size {
 		c0 := src[nSrc]
 		if c0 >= utf8.RuneSelf {
-			err = errInvalidISO2022JP
-			break loop
+			r, size = '\ufffd', 1
+			goto write
 		}
 
 		if c0 == asciiEsc {
 			if nSrc+2 >= len(src) {
-				err = transform.ErrShortSrc
-				break loop
+				if !atEOF {
+					return nDst, nSrc, transform.ErrShortSrc
+				}
+				// TODO: is it correct to only skip 1??
+				r, size = '\ufffd', 1
+				goto write
 			}
 			size = 3
 			c1 := src[nSrc+1]
 			c2 := src[nSrc+2]
 			switch {
-			case c1 == '$' && (c2 == '@' || c2 == 'B'):
+			case c1 == '$' && (c2 == '@' || c2 == 'B'): // 0x24 {0x40, 0x42}
 				*d = jis0208State
 				continue
-			case c1 == '$' && c2 == '(':
+			case c1 == '$' && c2 == '(': // 0x24 0x28
 				if nSrc+3 >= len(src) {
-					err = transform.ErrShortSrc
-					break loop
+					if !atEOF {
+						return nDst, nSrc, transform.ErrShortSrc
+					}
+					r, size = '\ufffd', 1
+					goto write
 				}
 				size = 4
-				if src[nSrc]+3 == 'D' {
+				if src[nSrc+3] == 'D' {
 					*d = jis0212State
 					continue
 				}
-			case c1 == '(' && (c2 == 'B' || c2 == 'J'):
+			case c1 == '(' && (c2 == 'B' || c2 == 'J'): // 0x28 {0x42, 0x4A}
 				*d = asciiState
 				continue
-			case c1 == '(' && c2 == 'I':
+			case c1 == '(' && c2 == 'I': // 0x28 0x49
 				*d = katakanaState
 				continue
 			}
-			err = errInvalidISO2022JP
-			break loop
+			r, size = '\ufffd', 1
+			goto write
 		}
 
 		switch *d {
@@ -97,8 +100,8 @@ loop:
 
 		case katakanaState:
 			if c0 < 0x21 || 0x60 <= c0 {
-				err = errInvalidISO2022JP
-				break loop
+				r, size = '\ufffd', 1
+				goto write
 			}
 			r, size = rune(c0)+(0xff61-0x21), 1
 
@@ -106,11 +109,14 @@ loop:
 			if c0 == 0x0a {
 				*d = asciiState
 				r, size = rune(c0), 1
-				break
+				goto write
 			}
 			if nSrc+1 >= len(src) {
-				err = transform.ErrShortSrc
-				break loop
+				if !atEOF {
+					return nDst, nSrc, transform.ErrShortSrc
+				}
+				r, size = '\ufffd', 1
+				goto write
 			}
 			size = 2
 			c1 := src[nSrc+1]
@@ -121,22 +127,19 @@ loop:
 				r = rune(jis0212Decode[i])
 			} else {
 				r = '\ufffd'
-				break
+				goto write
 			}
 			if r == 0 {
 				r = '\ufffd'
 			}
 		}
 
+	write:
 		if nDst+utf8.RuneLen(r) > len(dst) {
-			err = transform.ErrShortDst
-			break loop
+			return nDst, nSrc, transform.ErrShortDst
 		}
 		nDst += utf8.EncodeRune(dst[nDst:], r)
 	}
-	if atEOF && err == transform.ErrShortSrc {
-		err = errInvalidISO2022JP
-	}
 	return nDst, nSrc, err
 }
 

vendor/golang.org/x/text/encoding/japanese/maketables.go

@@ -1,161 +0,0 @@
-// Copyright 2013 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build ignore
-
-package main
-
-// This program generates tables.go:
-//	go run maketables.go | gofmt > tables.go
-
-// TODO: Emoji extensions?
-// http://www.unicode.org/faq/emoji_dingbats.html
-// http://www.unicode.org/Public/UNIDATA/EmojiSources.txt
-
-import (
-	"bufio"
-	"fmt"
-	"log"
-	"net/http"
-	"sort"
-	"strings"
-)
-
-type entry struct {
-	jisCode, table int
-}
-
-func main() {
-	fmt.Printf("// generated by go run maketables.go; DO NOT EDIT\n\n")
-	fmt.Printf("// Package japanese provides Japanese encodings such as EUC-JP and Shift JIS.\n")
-	fmt.Printf(`package japanese // import "golang.org/x/text/encoding/japanese"` + "\n\n")
-
-	reverse := [65536]entry{}
-	for i := range reverse {
-		reverse[i].table = -1
-	}
-
-	tables := []struct {
-		url  string
-		name string
-	}{
-		{"http://encoding.spec.whatwg.org/index-jis0208.txt", "0208"},
-		{"http://encoding.spec.whatwg.org/index-jis0212.txt", "0212"},
-	}
-	for i, table := range tables {
-		res, err := http.Get(table.url)
-		if err != nil {
-			log.Fatalf("%q: Get: %v", table.url, err)
-		}
-		defer res.Body.Close()
-
-		mapping := [65536]uint16{}
-
-		scanner := bufio.NewScanner(res.Body)
-		for scanner.Scan() {
-			s := strings.TrimSpace(scanner.Text())
-			if s == "" || s[0] == '#' {
-				continue
-			}
-			x, y := 0, uint16(0)
-			if _, err := fmt.Sscanf(s, "%d 0x%x", &x, &y); err != nil {
-				log.Fatalf("%q: could not parse %q", table.url, s)
-			}
-			if x < 0 || 120*94 <= x {
-				log.Fatalf("%q: JIS code %d is out of range", table.url, x)
-			}
-			mapping[x] = y
-			if reverse[y].table == -1 {
-				reverse[y] = entry{jisCode: x, table: i}
-			}
-		}
-		if err := scanner.Err(); err != nil {
-			log.Fatalf("%q: scanner error: %v", table.url, err)
-		}
-
-		fmt.Printf("// jis%sDecode is the decoding table from JIS %s code to Unicode.\n// It is defined at %s\n",
-			table.name, table.name, table.url)
-		fmt.Printf("var jis%sDecode = [...]uint16{\n", table.name)
-		for i, m := range mapping {
-			if m != 0 {
-				fmt.Printf("\t%d: 0x%04X,\n", i, m)
-			}
-		}
-		fmt.Printf("}\n\n")
-	}
-
-	// Any run of at least separation continuous zero entries in the reverse map will
-	// be a separate encode table.
-	const separation = 1024
-
-	intervals := []interval(nil)
-	low, high := -1, -1
-	for i, v := range reverse {
-		if v.table == -1 {
-			continue
-		}
-		if low < 0 {
-			low = i
-		} else if i-high >= separation {
-			if high >= 0 {
-				intervals = append(intervals, interval{low, high})
-			}
-			low = i
-		}
-		high = i + 1
-	}
-	if high >= 0 {
-		intervals = append(intervals, interval{low, high})
-	}
-	sort.Sort(byDecreasingLength(intervals))
-
-	fmt.Printf("const (\n")
-	fmt.Printf("\tjis0208    = 1\n")
-	fmt.Printf("\tjis0212    = 2\n")
-	fmt.Printf("\tcodeMask   = 0x7f\n")
-	fmt.Printf("\tcodeShift  = 7\n")
-	fmt.Printf("\ttableShift = 14\n")
-	fmt.Printf(")\n\n")
-
-	fmt.Printf("const numEncodeTables = %d\n\n", len(intervals))
-	fmt.Printf("// encodeX are the encoding tables from Unicode to JIS code,\n")
-	fmt.Printf("// sorted by decreasing length.\n")
-	for i, v := range intervals {
-		fmt.Printf("// encode%d: %5d entries for runes in [%5d, %5d).\n", i, v.len(), v.low, v.high)
-	}
-	fmt.Printf("//\n")
-	fmt.Printf("// The high two bits of the value record whether the JIS code comes from the\n")
-	fmt.Printf("// JIS0208 table (high bits == 1) or the JIS0212 table (high bits == 2).\n")
-	fmt.Printf("// The low 14 bits are two 7-bit unsigned integers j1 and j2 that form the\n")
-	fmt.Printf("// JIS code (94*j1 + j2) within that table.\n")
-	fmt.Printf("\n")
-
-	for i, v := range intervals {
-		fmt.Printf("const encode%dLow, encode%dHigh = %d, %d\n\n", i, i, v.low, v.high)
-		fmt.Printf("var encode%d = [...]uint16{\n", i)
-		for j := v.low; j < v.high; j++ {
-			x := reverse[j]
-			if x.table == -1 {
-				continue
-			}
-			fmt.Printf("\t%d - %d: jis%s<<14 | 0x%02X<<7 | 0x%02X,\n",
-				j, v.low, tables[x.table].name, x.jisCode/94, x.jisCode%94)
-		}
-		fmt.Printf("}\n\n")
-	}
-}
-
-// interval is a half-open interval [low, high).
-type interval struct {
-	low, high int
-}
-
-func (i interval) len() int { return i.high - i.low }
-
-// byDecreasingLength sorts intervals by decreasing length.
-type byDecreasingLength []interval
-
-func (b byDecreasingLength) Len() int           { return len(b) }
-func (b byDecreasingLength) Less(i, j int) bool { return b[i].len() > b[j].len() }
-func (b byDecreasingLength) Swap(i, j int)      { b[i], b[j] = b[j], b[i] }

vendor/golang.org/x/text/encoding/japanese/shiftjis.go

@@ -5,7 +5,6 @@
 package japanese
 
 import (
-	"errors"
 	"unicode/utf8"
 
 	"golang.org/x/text/encoding"
@@ -24,8 +23,6 @@ var shiftJIS = internal.Encoding{
 	identifier.ShiftJIS,
 }
 
-var errInvalidShiftJIS = errors.New("japanese: invalid Shift JIS encoding")
-
 type shiftJISDecoder struct{ transform.NopResetter }
 
 func (shiftJISDecoder) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
@@ -48,28 +45,32 @@ loop:
 			c0 = 2*c0 - 0x21
 
 			if nSrc+1 >= len(src) {
-				err = transform.ErrShortSrc
-				break loop
+				if !atEOF {
+					err = transform.ErrShortSrc
+					break loop
+				}
+				r, size = '\ufffd', 1
+				goto write
 			}
 			c1 := src[nSrc+1]
 			switch {
 			case c1 < 0x40:
-				err = errInvalidShiftJIS
-				break loop
+				r, size = '\ufffd', 1 // c1 is ASCII so output on next round
+				goto write
 			case c1 < 0x7f:
 				c0--
 				c1 -= 0x40
 			case c1 == 0x7f:
-				err = errInvalidShiftJIS
-				break loop
+				r, size = '\ufffd', 1 // c1 is ASCII so output on next round
+				goto write
 			case c1 < 0x9f:
 				c0--
 				c1 -= 0x41
 			case c1 < 0xfd:
 				c1 -= 0x9f
 			default:
-				err = errInvalidShiftJIS
-				break loop
+				r, size = '\ufffd', 2
+				goto write
 			}
 			r, size = '\ufffd', 2
 			if i := int(c0)*94 + int(c1); i < len(jis0208Decode) {
@@ -79,20 +80,19 @@ loop:
 				}
 			}
 
-		default:
-			err = errInvalidShiftJIS
-			break loop
-		}
+		case c0 == 0x80:
+			r, size = 0x80, 1
 
+		default:
+			r, size = '\ufffd', 1
+		}
+	write:
 		if nDst+utf8.RuneLen(r) > len(dst) {
 			err = transform.ErrShortDst
 			break loop
 		}
 		nDst += utf8.EncodeRune(dst[nDst:], r)
 	}
-	if atEOF && err == transform.ErrShortSrc {
-		err = errInvalidShiftJIS
-	}
 	return nDst, nSrc, err
 }