From 4ccad563d2a3559f0557bfb177bcf45144219bdf Mon Sep 17 00:00:00 2001 From: Ian Lance Taylor Date: Tue, 23 Oct 2012 04:31:11 +0000 Subject: libgo: Update to current sources. From-SVN: r192704 --- libgo/go/encoding/asn1/asn1.go | 33 +-- libgo/go/encoding/asn1/asn1_test.go | 2 +- libgo/go/encoding/asn1/common.go | 2 + libgo/go/encoding/asn1/marshal.go | 38 +++- libgo/go/encoding/asn1/marshal_test.go | 8 + libgo/go/encoding/base32/base32.go | 3 +- libgo/go/encoding/base64/base64.go | 3 +- libgo/go/encoding/csv/writer.go | 5 +- libgo/go/encoding/gob/codec_test.go | 15 +- libgo/go/encoding/gob/decode.go | 2 + libgo/go/encoding/gob/decoder.go | 37 +++- libgo/go/encoding/gob/encode.go | 23 +- libgo/go/encoding/gob/encoder.go | 6 + libgo/go/encoding/gob/encoder_test.go | 77 +++++++ libgo/go/encoding/gob/type.go | 3 + libgo/go/encoding/gob/type_test.go | 61 ++++++ libgo/go/encoding/json/decode.go | 161 +++++++------- libgo/go/encoding/json/decode_test.go | 378 +++++++++++++++++++++++++++------ libgo/go/encoding/json/encode.go | 255 +++++++++++++++++----- libgo/go/encoding/json/stream.go | 6 +- libgo/go/encoding/json/stream_test.go | 22 ++ libgo/go/encoding/xml/marshal.go | 36 ++-- libgo/go/encoding/xml/marshal_test.go | 56 ++++- libgo/go/encoding/xml/read.go | 18 +- libgo/go/encoding/xml/typeinfo.go | 27 ++- libgo/go/encoding/xml/xml.go | 51 ++++- libgo/go/encoding/xml/xml_test.go | 84 +++++++- 27 files changed, 1134 insertions(+), 278 deletions(-) (limited to 'libgo/go/encoding') diff --git a/libgo/go/encoding/asn1/asn1.go b/libgo/go/encoding/asn1/asn1.go index ac2b5f8daa6..cac9d64b5e9 100644 --- a/libgo/go/encoding/asn1/asn1.go +++ b/libgo/go/encoding/asn1/asn1.go @@ -77,15 +77,15 @@ func parseInt64(bytes []byte) (ret int64, err error) { // parseInt treats the given bytes as a big-endian, signed integer and returns // the result. -func parseInt(bytes []byte) (int, error) { +func parseInt32(bytes []byte) (int32, error) { ret64, err := parseInt64(bytes) if err != nil { return 0, err } - if ret64 != int64(int(ret64)) { + if ret64 != int64(int32(ret64)) { return 0, StructuralError{"integer too large"} } - return int(ret64), nil + return int32(ret64), nil } var bigOne = big.NewInt(1) @@ -670,7 +670,7 @@ func parseField(v reflect.Value, bytes []byte, initOffset int, params fieldParam err = err1 return case enumeratedType: - parsedInt, err1 := parseInt(innerBytes) + parsedInt, err1 := parseInt32(innerBytes) if err1 == nil { v.SetInt(int64(parsedInt)) } @@ -692,19 +692,20 @@ func parseField(v reflect.Value, bytes []byte, initOffset int, params fieldParam } err = err1 return - case reflect.Int, reflect.Int32: - parsedInt, err1 := parseInt(innerBytes) - if err1 == nil { - val.SetInt(int64(parsedInt)) - } - err = err1 - return - case reflect.Int64: - parsedInt, err1 := parseInt64(innerBytes) - if err1 == nil { - val.SetInt(parsedInt) + case reflect.Int, reflect.Int32, reflect.Int64: + if val.Type().Size() == 4 { + parsedInt, err1 := parseInt32(innerBytes) + if err1 == nil { + val.SetInt(int64(parsedInt)) + } + err = err1 + } else { + parsedInt, err1 := parseInt64(innerBytes) + if err1 == nil { + val.SetInt(parsedInt) + } + err = err1 } - err = err1 return // TODO(dfc) Add support for the remaining integer types case reflect.Struct: diff --git a/libgo/go/encoding/asn1/asn1_test.go b/libgo/go/encoding/asn1/asn1_test.go index eb848bdb4af..cabdf03b482 100644 --- a/libgo/go/encoding/asn1/asn1_test.go +++ b/libgo/go/encoding/asn1/asn1_test.go @@ -64,7 +64,7 @@ var int32TestData = []int32Test{ func TestParseInt32(t *testing.T) { for i, test := range int32TestData { - ret, err := parseInt(test.in) + ret, err := parseInt32(test.in) if (err == nil) != test.ok { t.Errorf("#%d: Incorrect error result (did fail? %v, expected: %v)", i, err == nil, test.ok) } diff --git a/libgo/go/encoding/asn1/common.go b/libgo/go/encoding/asn1/common.go index 03856bc55c5..33a117ece19 100644 --- a/libgo/go/encoding/asn1/common.go +++ b/libgo/go/encoding/asn1/common.go @@ -98,6 +98,8 @@ func parseFieldParameters(str string) (ret fieldParameters) { ret.stringType = tagIA5String case part == "printable": ret.stringType = tagPrintableString + case part == "utf8": + ret.stringType = tagUTF8String case strings.HasPrefix(part, "default:"): i, err := strconv.ParseInt(part[8:], 10, 64) if err == nil { diff --git a/libgo/go/encoding/asn1/marshal.go b/libgo/go/encoding/asn1/marshal.go index 163bca575de..3fd6be8ad92 100644 --- a/libgo/go/encoding/asn1/marshal.go +++ b/libgo/go/encoding/asn1/marshal.go @@ -6,11 +6,13 @@ package asn1 import ( "bytes" + "errors" "fmt" "io" "math/big" "reflect" "time" + "unicode/utf8" ) // A forkableWriter is an in-memory buffer that can be @@ -280,6 +282,11 @@ func marshalIA5String(out *forkableWriter, s string) (err error) { return } +func marshalUTF8String(out *forkableWriter, s string) (err error) { + _, err = out.Write([]byte(s)) + return +} + func marshalTwoDigits(out *forkableWriter, v int) (err error) { err = out.WriteByte(byte('0' + (v/10)%10)) if err != nil { @@ -446,10 +453,13 @@ func marshalBody(out *forkableWriter, value reflect.Value, params fieldParameter } return case reflect.String: - if params.stringType == tagIA5String { + switch params.stringType { + case tagIA5String: return marshalIA5String(out, v.String()) - } else { + case tagPrintableString: return marshalPrintableString(out, v.String()) + default: + return marshalUTF8String(out, v.String()) } return } @@ -492,11 +502,27 @@ func marshalField(out *forkableWriter, v reflect.Value, params fieldParameters) } class := classUniversal - if params.stringType != 0 { - if tag != tagPrintableString { - return StructuralError{"Explicit string type given to non-string member"} + if params.stringType != 0 && tag != tagPrintableString { + return StructuralError{"Explicit string type given to non-string member"} + } + + if tag == tagPrintableString { + if params.stringType == 0 { + // This is a string without an explicit string type. We'll use + // a PrintableString if the character set in the string is + // sufficiently limited, otherwise we'll use a UTF8String. + for _, r := range v.String() { + if r >= utf8.RuneSelf || !isPrintable(byte(r)) { + if !utf8.ValidString(v.String()) { + return errors.New("asn1: string not valid UTF-8") + } + tag = tagUTF8String + break + } + } + } else { + tag = params.stringType } - tag = params.stringType } if params.set { diff --git a/libgo/go/encoding/asn1/marshal_test.go b/libgo/go/encoding/asn1/marshal_test.go index f43bcae681a..c203533a917 100644 --- a/libgo/go/encoding/asn1/marshal_test.go +++ b/libgo/go/encoding/asn1/marshal_test.go @@ -122,6 +122,7 @@ var marshalTests = []marshalTest{ {testSET([]int{10}), "310302010a"}, {omitEmptyTest{[]string{}}, "3000"}, {omitEmptyTest{[]string{"1"}}, "30053003130131"}, + {"Σ", "0c02cea3"}, } func TestMarshal(t *testing.T) { @@ -137,3 +138,10 @@ func TestMarshal(t *testing.T) { } } } + +func TestInvalidUTF8(t *testing.T) { + _, err := Marshal(string([]byte{0xff, 0xff})) + if err == nil { + t.Errorf("invalid UTF8 string was accepted") + } +} diff --git a/libgo/go/encoding/base32/base32.go b/libgo/go/encoding/base32/base32.go index 71da6e22b12..dbefc48fa37 100644 --- a/libgo/go/encoding/base32/base32.go +++ b/libgo/go/encoding/base32/base32.go @@ -237,7 +237,6 @@ func (enc *Encoding) decode(dst, src []byte) (n int, end bool, err error) { dlen := 8 // do the top bytes contain any data? - dbufloop: for j := 0; j < 8; { if len(src) == 0 { return n, false, CorruptInputError(len(osrc) - len(src) - j) @@ -258,7 +257,7 @@ func (enc *Encoding) decode(dst, src []byte) (n int, end bool, err error) { } dlen = j end = true - break dbufloop + break } dbuf[j] = enc.decodeMap[in] if dbuf[j] == 0xFF { diff --git a/libgo/go/encoding/base64/base64.go b/libgo/go/encoding/base64/base64.go index 0b842f06610..e66672a1c93 100644 --- a/libgo/go/encoding/base64/base64.go +++ b/libgo/go/encoding/base64/base64.go @@ -216,7 +216,6 @@ func (enc *Encoding) decode(dst, src []byte) (n int, end bool, err error) { var dbuf [4]byte dlen := 4 - dbufloop: for j := 0; j < 4; { if len(src) == 0 { return n, false, CorruptInputError(len(osrc) - len(src) - j) @@ -240,7 +239,7 @@ func (enc *Encoding) decode(dst, src []byte) (n int, end bool, err error) { } dlen = j end = true - break dbufloop + break } dbuf[j] = enc.decodeMap[in] if dbuf[j] == 0xFF { diff --git a/libgo/go/encoding/csv/writer.go b/libgo/go/encoding/csv/writer.go index c4dcba5668a..324944cc829 100644 --- a/libgo/go/encoding/csv/writer.go +++ b/libgo/go/encoding/csv/writer.go @@ -101,11 +101,10 @@ func (w *Writer) WriteAll(records [][]string) (err error) { for _, record := range records { err = w.Write(record) if err != nil { - break + return err } } - w.Flush() - return nil + return w.w.Flush() } // fieldNeedsQuotes returns true if our field must be enclosed in quotes. diff --git a/libgo/go/encoding/gob/codec_test.go b/libgo/go/encoding/gob/codec_test.go index ebcbb78ebe6..482212b7467 100644 --- a/libgo/go/encoding/gob/codec_test.go +++ b/libgo/go/encoding/gob/codec_test.go @@ -7,6 +7,7 @@ package gob import ( "bytes" "errors" + "flag" "math" "math/rand" "reflect" @@ -16,6 +17,8 @@ import ( "unsafe" ) +var doFuzzTests = flag.Bool("gob.fuzz", false, "run the fuzz tests, which are large and very slow") + // Guarantee encoding format by comparing some encodings to hand-written values type EncodeT struct { x uint64 @@ -1434,7 +1437,8 @@ func encFuzzDec(rng *rand.Rand, in interface{}) error { // This does some "fuzz testing" by attempting to decode a sequence of random bytes. func TestFuzz(t *testing.T) { - if testing.Short() { + if !*doFuzzTests { + t.Logf("disabled; run with -gob.fuzz to enable") return } @@ -1453,11 +1457,16 @@ func TestFuzz(t *testing.T) { } func TestFuzzRegressions(t *testing.T) { + if !*doFuzzTests { + t.Logf("disabled; run with -gob.fuzz to enable") + return + } + // An instance triggering a type name of length ~102 GB. testFuzz(t, 1328492090837718000, 100, new(float32)) // An instance triggering a type name of 1.6 GB. - // Commented out because it takes 5m to run. - //testFuzz(t, 1330522872628565000, 100, new(int)) + // Note: can take several minutes to run. + testFuzz(t, 1330522872628565000, 100, new(int)) } func testFuzz(t *testing.T, seed int64, n int, input ...interface{}) { diff --git a/libgo/go/encoding/gob/decode.go b/libgo/go/encoding/gob/decode.go index 8690b35d714..900c69ddb47 100644 --- a/libgo/go/encoding/gob/decode.go +++ b/libgo/go/encoding/gob/decode.go @@ -717,7 +717,9 @@ func (dec *Decoder) decodeInterface(ityp reflect.Type, state *decoderState, p ui errorf("name too long (%d bytes): %.20q...", len(name), name) } // The concrete type must be registered. + registerLock.RLock() typ, ok := nameToConcreteType[name] + registerLock.RUnlock() if !ok { errorf("name not registered for interface: %q", name) } diff --git a/libgo/go/encoding/gob/decoder.go b/libgo/go/encoding/gob/decoder.go index c5c7d3fdb10..04f706ca540 100644 --- a/libgo/go/encoding/gob/decoder.go +++ b/libgo/go/encoding/gob/decoder.go @@ -87,21 +87,38 @@ func (dec *Decoder) recvMessage() bool { // readMessage reads the next nbytes bytes from the input. func (dec *Decoder) readMessage(nbytes int) { - // Allocate the buffer. - if cap(dec.tmp) < nbytes { - dec.tmp = make([]byte, nbytes+100) // room to grow + // Allocate the dec.tmp buffer, up to 10KB. + const maxBuf = 10 * 1024 + nTmp := nbytes + if nTmp > maxBuf { + nTmp = maxBuf } - dec.tmp = dec.tmp[:nbytes] + if cap(dec.tmp) < nTmp { + nAlloc := nTmp + 100 // A little extra for growth. + if nAlloc > maxBuf { + nAlloc = maxBuf + } + dec.tmp = make([]byte, nAlloc) + } + dec.tmp = dec.tmp[:nTmp] // Read the data - _, dec.err = io.ReadFull(dec.r, dec.tmp) - if dec.err != nil { - if dec.err == io.EOF { - dec.err = io.ErrUnexpectedEOF + dec.buf.Grow(nbytes) + for nbytes > 0 { + if nbytes < nTmp { + dec.tmp = dec.tmp[:nbytes] } - return + var nRead int + nRead, dec.err = io.ReadFull(dec.r, dec.tmp) + if dec.err != nil { + if dec.err == io.EOF { + dec.err = io.ErrUnexpectedEOF + } + return + } + dec.buf.Write(dec.tmp) + nbytes -= nRead } - dec.buf.Write(dec.tmp) } // toInt turns an encoded uint64 into an int, according to the marshaling rules. diff --git a/libgo/go/encoding/gob/encode.go b/libgo/go/encoding/gob/encode.go index 168e08b137a..ea37a6cbd58 100644 --- a/libgo/go/encoding/gob/encode.go +++ b/libgo/go/encoding/gob/encode.go @@ -426,6 +426,12 @@ func (enc *Encoder) encodeMap(b *bytes.Buffer, mv reflect.Value, keyOp, elemOp e // by the concrete value. A nil value gets sent as the empty string for the name, // followed by no value. func (enc *Encoder) encodeInterface(b *bytes.Buffer, iv reflect.Value) { + // Gobs can encode nil interface values but not typed interface + // values holding nil pointers, since nil pointers point to no value. + elem := iv.Elem() + if elem.Kind() == reflect.Ptr && elem.IsNil() { + errorf("gob: cannot encode nil pointer of type %s inside interface", iv.Elem().Type()) + } state := enc.newEncoderState(b) state.fieldnum = -1 state.sendZero = true @@ -435,7 +441,9 @@ func (enc *Encoder) encodeInterface(b *bytes.Buffer, iv reflect.Value) { } ut := userType(iv.Elem().Type()) + registerLock.RLock() name, ok := concreteTypeToName[ut.base] + registerLock.RUnlock() if !ok { errorf("type not registered for interface: %s", ut.base) } @@ -454,7 +462,7 @@ func (enc *Encoder) encodeInterface(b *bytes.Buffer, iv reflect.Value) { enc.pushWriter(b) data := new(bytes.Buffer) data.Write(spaceForLength) - enc.encode(data, iv.Elem(), ut) + enc.encode(data, elem, ut) if enc.err != nil { error_(enc.err) } @@ -698,9 +706,20 @@ func (enc *Encoder) getEncEngine(ut *userTypeInfo) *encEngine { error_(err1) } if info.encoder == nil { - // mark this engine as underway before compiling to handle recursive types. + // Assign the encEngine now, so recursive types work correctly. But... info.encoder = new(encEngine) + // ... if we fail to complete building the engine, don't cache the half-built machine. + // Doing this here means we won't cache a type that is itself OK but + // that contains a nested type that won't compile. The result is consistent + // error behavior when Encode is called multiple times on the top-level type. + ok := false + defer func() { + if !ok { + info.encoder = nil + } + }() info.encoder = enc.compileEnc(ut) + ok = true } return info.encoder } diff --git a/libgo/go/encoding/gob/encoder.go b/libgo/go/encoding/gob/encoder.go index a15b5a1f9a1..51444bb5269 100644 --- a/libgo/go/encoding/gob/encoder.go +++ b/libgo/go/encoding/gob/encoder.go @@ -218,6 +218,12 @@ func (enc *Encoder) sendTypeId(state *encoderState, ut *userTypeInfo) { // EncodeValue transmits the data item represented by the reflection value, // guaranteeing that all necessary type information has been transmitted first. func (enc *Encoder) EncodeValue(value reflect.Value) error { + // Gobs contain values. They cannot represent nil pointers, which + // have no value to encode. + if value.Kind() == reflect.Ptr && value.IsNil() { + panic("gob: cannot encode nil pointer of type " + value.Type().String()) + } + // Make sure we're single-threaded through here, so multiple // goroutines can share an encoder. enc.mutex.Lock() diff --git a/libgo/go/encoding/gob/encoder_test.go b/libgo/go/encoding/gob/encoder_test.go index db824d99917..b684772c691 100644 --- a/libgo/go/encoding/gob/encoder_test.go +++ b/libgo/go/encoding/gob/encoder_test.go @@ -737,6 +737,83 @@ func TestPtrToMapOfMap(t *testing.T) { } } +// A top-level nil pointer generates a panic with a helpful string-valued message. +func TestTopLevelNilPointer(t *testing.T) { + errMsg := topLevelNilPanic(t) + if errMsg == "" { + t.Fatal("top-level nil pointer did not panic") + } + if !strings.Contains(errMsg, "nil pointer") { + t.Fatal("expected nil pointer error, got:", errMsg) + } +} + +func topLevelNilPanic(t *testing.T) (panicErr string) { + defer func() { + e := recover() + if err, ok := e.(string); ok { + panicErr = err + } + }() + var ip *int + buf := new(bytes.Buffer) + if err := NewEncoder(buf).Encode(ip); err != nil { + t.Fatal("error in encode:", err) + } + return +} + +func TestNilPointerInsideInterface(t *testing.T) { + var ip *int + si := struct { + I interface{} + }{ + I: ip, + } + buf := new(bytes.Buffer) + err := NewEncoder(buf).Encode(si) + if err == nil { + t.Fatal("expected error, got none") + } + errMsg := err.Error() + if !strings.Contains(errMsg, "nil pointer") || !strings.Contains(errMsg, "interface") { + t.Fatal("expected error about nil pointer and interface, got:", errMsg) + } +} + +type Bug4Public struct { + Name string + Secret Bug4Secret +} + +type Bug4Secret struct { + a int // error: no exported fields. +} + +// Test that a failed compilation doesn't leave around an executable encoder. +// Issue 3273. +func TestMutipleEncodingsOfBadType(t *testing.T) { + x := Bug4Public{ + Name: "name", + Secret: Bug4Secret{1}, + } + buf := new(bytes.Buffer) + enc := NewEncoder(buf) + err := enc.Encode(x) + if err == nil { + t.Fatal("first encoding: expected error") + } + buf.Reset() + enc = NewEncoder(buf) + err = enc.Encode(x) + if err == nil { + t.Fatal("second encoding: expected error") + } + if !strings.Contains(err.Error(), "no exported fields") { + t.Errorf("expected error about no exported fields; got %v", err) + } +} + // There was an error check comparing the length of the input with the // length of the slice being decoded. It was wrong because the next // thing in the input might be a type definition, which would lead to diff --git a/libgo/go/encoding/gob/type.go b/libgo/go/encoding/gob/type.go index a8ee2fa4a5a..ea0db4eac45 100644 --- a/libgo/go/encoding/gob/type.go +++ b/libgo/go/encoding/gob/type.go @@ -712,6 +712,7 @@ type GobDecoder interface { } var ( + registerLock sync.RWMutex nameToConcreteType = make(map[string]reflect.Type) concreteTypeToName = make(map[reflect.Type]string) ) @@ -723,6 +724,8 @@ func RegisterName(name string, value interface{}) { // reserved for nil panic("attempt to register empty name") } + registerLock.Lock() + defer registerLock.Unlock() ut := userType(reflect.TypeOf(value)) // Check for incompatible duplicates. The name must refer to the // same user type, and vice versa. diff --git a/libgo/go/encoding/gob/type_test.go b/libgo/go/encoding/gob/type_test.go index 42bdb4cf7bb..e230d22d431 100644 --- a/libgo/go/encoding/gob/type_test.go +++ b/libgo/go/encoding/gob/type_test.go @@ -5,6 +5,7 @@ package gob import ( + "bytes" "reflect" "testing" ) @@ -159,3 +160,63 @@ func TestRegistration(t *testing.T) { Register(new(T)) Register(new(T)) } + +type N1 struct{} +type N2 struct{} + +// See comment in type.go/Register. +func TestRegistrationNaming(t *testing.T) { + testCases := []struct { + t interface{} + name string + }{ + {&N1{}, "*gob.N1"}, + {N2{}, "encoding/gob.N2"}, + } + + for _, tc := range testCases { + Register(tc.t) + + tct := reflect.TypeOf(tc.t) + registerLock.RLock() + ct := nameToConcreteType[tc.name] + registerLock.RUnlock() + if ct != tct { + t.Errorf("nameToConcreteType[%q] = %v, want %v", tc.name, ct, tct) + } + // concreteTypeToName is keyed off the base type. + if tct.Kind() == reflect.Ptr { + tct = tct.Elem() + } + if n := concreteTypeToName[tct]; n != tc.name { + t.Errorf("concreteTypeToName[%v] got %v, want %v", tct, n, tc.name) + } + } +} + +func TestStressParallel(t *testing.T) { + type T2 struct{ A int } + c := make(chan bool) + const N = 10 + for i := 0; i < N; i++ { + go func() { + p := new(T2) + Register(p) + b := new(bytes.Buffer) + enc := NewEncoder(b) + err := enc.Encode(p) + if err != nil { + t.Error("encoder fail:", err) + } + dec := NewDecoder(b) + err = dec.Decode(p) + if err != nil { + t.Error("decoder fail:", err) + } + c <- true + }() + } + for i := 0; i < N; i++ { + <-c + } +} diff --git a/libgo/go/encoding/json/decode.go b/libgo/go/encoding/json/decode.go index d61f8870646..47e3d89aa35 100644 --- a/libgo/go/encoding/json/decode.go +++ b/libgo/go/encoding/json/decode.go @@ -137,6 +137,22 @@ func (d *decodeState) unmarshal(v interface{}) (err error) { return d.savedError } +// A Number represents a JSON number literal. +type Number string + +// String returns the literal text of the number. +func (n Number) String() string { return string(n) } + +// Float64 returns the number as a float64. +func (n Number) Float64() (float64, error) { + return strconv.ParseFloat(string(n), 64) +} + +// Int64 returns the number as an int64. +func (n Number) Int64() (int64, error) { + return strconv.ParseInt(string(n), 10, 64) +} + // decodeState represents the state while decoding a JSON value. type decodeState struct { data []byte @@ -145,6 +161,7 @@ type decodeState struct { nextscan scanner // for calls to nextValue savedError error tempstr string // scratch space to avoid some allocations + useNumber bool } // errPhase is used for errors that should not happen unless @@ -265,47 +282,32 @@ func (d *decodeState) indirect(v reflect.Value, decodingNull bool) (Unmarshaler, v = v.Addr() } for { - var isUnmarshaler bool - if v.Type().NumMethod() > 0 { - // Remember that this is an unmarshaler, - // but wait to return it until after allocating - // the pointer (if necessary). - _, isUnmarshaler = v.Interface().(Unmarshaler) - } - // Load value from interface, but only if the result will be // usefully addressable. - if iv := v; iv.Kind() == reflect.Interface && !iv.IsNil() { - e := iv.Elem() + if v.Kind() == reflect.Interface && !v.IsNil() { + e := v.Elem() if e.Kind() == reflect.Ptr && !e.IsNil() && (!decodingNull || e.Elem().Kind() == reflect.Ptr) { v = e continue } } - pv := v - if pv.Kind() != reflect.Ptr { + if v.Kind() != reflect.Ptr { break } - if pv.Elem().Kind() != reflect.Ptr && decodingNull && pv.CanSet() { - return nil, pv + if v.Elem().Kind() != reflect.Ptr && decodingNull && v.CanSet() { + break } - if pv.IsNil() { - pv.Set(reflect.New(pv.Type().Elem())) + if v.IsNil() { + v.Set(reflect.New(v.Type().Elem())) } - if isUnmarshaler { - // Using v.Interface().(Unmarshaler) - // here means that we have to use a pointer - // as the struct field. We cannot use a value inside - // a pointer to a struct, because in that case - // v.Interface() is the value (x.f) not the pointer (&x.f). - // This is an unfortunate consequence of reflect. - // An alternative would be to look up the - // UnmarshalJSON method and return a FuncValue. - return v.Interface().(Unmarshaler), reflect.Value{} + if v.Type().NumMethod() > 0 { + if unmarshaler, ok := v.Interface().(Unmarshaler); ok { + return unmarshaler, reflect.Value{} + } } - v = pv.Elem() + v = v.Elem() } return nil, v } @@ -491,51 +493,39 @@ func (d *decodeState) object(v reflect.Value) { } subv = mapElem } else { - var f reflect.StructField - var ok bool - st := sv.Type() - for i := 0; i < sv.NumField(); i++ { - sf := st.Field(i) - tag := sf.Tag.Get("json") - if tag == "-" { - // Pretend this field doesn't exist. - continue + var f *field + fields := cachedTypeFields(sv.Type()) + for i := range fields { + ff := &fields[i] + if ff.name == key { + f = ff + break } - if sf.Anonymous { - // Pretend this field doesn't exist, - // so that we can do a good job with - // these in a later version. - continue - } - // First, tag match - tagName, _ := parseTag(tag) - if tagName == key { - f = sf - ok = true - break // no better match possible - } - // Second, exact field name match - if sf.Name == key { - f = sf - ok = true - } - // Third, case-insensitive field name match, - // but only if a better match hasn't already been seen - if !ok && strings.EqualFold(sf.Name, key) { - f = sf - ok = true + if f == nil && strings.EqualFold(ff.name, key) { + f = ff } } - - // Extract value; name must be exported. - if ok { - if f.PkgPath != "" { - d.saveError(&UnmarshalFieldError{key, st, f}) - } else { - subv = sv.FieldByIndex(f.Index) + if f != nil { + subv = sv + destring = f.quoted + for _, i := range f.index { + if subv.Kind() == reflect.Ptr { + if subv.IsNil() { + subv.Set(reflect.New(subv.Type().Elem())) + } + subv = subv.Elem() + } + subv = subv.Field(i) + } + } else { + // To give a good error, a quick scan for unexported fields in top level. + st := sv.Type() + for i := 0; i < st.NumField(); i++ { + f := st.Field(i) + if f.PkgPath != "" && strings.EqualFold(f.Name, key) { + d.saveError(&UnmarshalFieldError{key, st, f}) + } } - _, opts := parseTag(f.Tag.Get("json")) - destring = opts.Contains("string") } } @@ -586,6 +576,21 @@ func (d *decodeState) literal(v reflect.Value) { d.literalStore(d.data[start:d.off], v, false) } +// convertNumber converts the number literal s to a float64 or a Number +// depending on the setting of d.useNumber. +func (d *decodeState) convertNumber(s string) (interface{}, error) { + if d.useNumber { + return Number(s), nil + } + f, err := strconv.ParseFloat(s, 64) + if err != nil { + return nil, &UnmarshalTypeError{"number " + s, reflect.TypeOf(0.0)} + } + return f, nil +} + +var numberType = reflect.TypeOf(Number("")) + // literalStore decodes a literal stored in item into v. // // fromQuoted indicates whether this literal came from unwrapping a @@ -674,15 +679,19 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool s := string(item) switch v.Kind() { default: + if v.Kind() == reflect.String && v.Type() == numberType { + v.SetString(s) + break + } if fromQuoted { d.error(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) } else { d.error(&UnmarshalTypeError{"number", v.Type()}) } case reflect.Interface: - n, err := strconv.ParseFloat(s, 64) + n, err := d.convertNumber(s) if err != nil { - d.saveError(&UnmarshalTypeError{"number " + s, v.Type()}) + d.saveError(err) break } v.Set(reflect.ValueOf(n)) @@ -836,9 +845,9 @@ func (d *decodeState) literalInterface() interface{} { if c != '-' && (c < '0' || c > '9') { d.error(errPhase) } - n, err := strconv.ParseFloat(string(item), 64) + n, err := d.convertNumber(string(item)) if err != nil { - d.saveError(&UnmarshalTypeError{"number " + string(item), reflect.TypeOf(0.0)}) + d.saveError(err) } return n } @@ -979,11 +988,3 @@ func unquoteBytes(s []byte) (t []byte, ok bool) { } return b[0:w], true } - -// The following is issue 3069. - -// BUG(rsc): This package ignores anonymous (embedded) struct fields -// during encoding and decoding. A future version may assign meaning -// to them. To force an anonymous field to be ignored in all future -// versions of this package, use an explicit `json:"-"` tag in the struct -// definition. diff --git a/libgo/go/encoding/json/decode_test.go b/libgo/go/encoding/json/decode_test.go index 6fac22c4a35..f2da141b8f9 100644 --- a/libgo/go/encoding/json/decode_test.go +++ b/libgo/go/encoding/json/decode_test.go @@ -7,6 +7,7 @@ package json import ( "bytes" "fmt" + "image" "reflect" "strings" "testing" @@ -18,6 +19,32 @@ type T struct { Z int `json:"-"` } +type U struct { + Alphabet string `json:"alpha"` +} + +type V struct { + F1 interface{} + F2 int32 + F3 Number +} + +// ifaceNumAsFloat64/ifaceNumAsNumber are used to test unmarshalling with and +// without UseNumber +var ifaceNumAsFloat64 = map[string]interface{}{ + "k1": float64(1), + "k2": "s", + "k3": []interface{}{float64(1), float64(2.0), float64(3e-3)}, + "k4": map[string]interface{}{"kk1": "s", "kk2": float64(2)}, +} + +var ifaceNumAsNumber = map[string]interface{}{ + "k1": Number("1"), + "k2": "s", + "k3": []interface{}{Number("1"), Number("2.0"), Number("3e-3")}, + "k4": map[string]interface{}{"kk1": "s", "kk2": Number("2")}, +} + type tx struct { x int } @@ -48,55 +75,237 @@ var ( umstruct = ustruct{unmarshaler{true}} ) +// Test data structures for anonymous fields. + +type Point struct { + Z int +} + +type Top struct { + Level0 int + Embed0 + *Embed0a + *Embed0b `json:"e,omitempty"` // treated as named + Embed0c `json:"-"` // ignored + Loop + Embed0p // has Point with X, Y, used + Embed0q // has Point with Z, used +} + +type Embed0 struct { + Level1a int // overridden by Embed0a's Level1a with json tag + Level1b int // used because Embed0a's Level1b is renamed + Level1c int // used because Embed0a's Level1c is ignored + Level1d int // annihilated by Embed0a's Level1d + Level1e int `json:"x"` // annihilated by Embed0a.Level1e +} + +type Embed0a struct { + Level1a int `json:"Level1a,omitempty"` + Level1b int `json:"LEVEL1B,omitempty"` + Level1c int `json:"-"` + Level1d int // annihilated by Embed0's Level1d + Level1f int `json:"x"` // annihilated by Embed0's Level1e +} + +type Embed0b Embed0 + +type Embed0c Embed0 + +type Embed0p struct { + image.Point +} + +type Embed0q struct { + Point +} + +type Loop struct { + Loop1 int `json:",omitempty"` + Loop2 int `json:",omitempty"` + *Loop +} + +// From reflect test: +// The X in S6 and S7 annihilate, but they also block the X in S8.S9. +type S5 struct { + S6 + S7 + S8 +} + +type S6 struct { + X int +} + +type S7 S6 + +type S8 struct { + S9 +} + +type S9 struct { + X int + Y int +} + +// From reflect test: +// The X in S11.S6 and S12.S6 annihilate, but they also block the X in S13.S8.S9. +type S10 struct { + S11 + S12 + S13 +} + +type S11 struct { + S6 +} + +type S12 struct { + S6 +} + +type S13 struct { + S8 +} + type unmarshalTest struct { - in string - ptr interface{} - out interface{} - err error + in string + ptr interface{} + out interface{} + err error + useNumber bool +} + +type Ambig struct { + // Given "hello", the first match should win. + First int `json:"HELLO"` + Second int `json:"Hello"` } var unmarshalTests = []unmarshalTest{ // basic types - {`true`, new(bool), true, nil}, - {`1`, new(int), 1, nil}, - {`1.2`, new(float64), 1.2, nil}, - {`-5`, new(int16), int16(-5), nil}, - {`"a\u1234"`, new(string), "a\u1234", nil}, - {`"http:\/\/"`, new(string), "http://", nil}, - {`"g-clef: \uD834\uDD1E"`, new(string), "g-clef: \U0001D11E", nil}, - {`"invalid: \uD834x\uDD1E"`, new(string), "invalid: \uFFFDx\uFFFD", nil}, - {"null", new(interface{}), nil, nil}, - {`{"X": [1,2,3], "Y": 4}`, new(T), T{Y: 4}, &UnmarshalTypeError{"array", reflect.TypeOf("")}}, - {`{"x": 1}`, new(tx), tx{}, &UnmarshalFieldError{"x", txType, txType.Field(0)}}, + {in: `true`, ptr: new(bool), out: true}, + {in: `1`, ptr: new(int), out: 1}, + {in: `1.2`, ptr: new(float64), out: 1.2}, + {in: `-5`, ptr: new(int16), out: int16(-5)}, + {in: `2`, ptr: new(Number), out: Number("2"), useNumber: true}, + {in: `2`, ptr: new(Number), out: Number("2")}, + {in: `2`, ptr: new(interface{}), out: float64(2.0)}, + {in: `2`, ptr: new(interface{}), out: Number("2"), useNumber: true}, + {in: `"a\u1234"`, ptr: new(string), out: "a\u1234"}, + {in: `"http:\/\/"`, ptr: new(string), out: "http://"}, + {in: `"g-clef: \uD834\uDD1E"`, ptr: new(string), out: "g-clef: \U0001D11E"}, + {in: `"invalid: \uD834x\uDD1E"`, ptr: new(string), out: "invalid: \uFFFDx\uFFFD"}, + {in: "null", ptr: new(interface{}), out: nil}, + {in: `{"X": [1,2,3], "Y": 4}`, ptr: new(T), out: T{Y: 4}, err: &UnmarshalTypeError{"array", reflect.TypeOf("")}}, + {in: `{"x": 1}`, ptr: new(tx), out: tx{}, err: &UnmarshalFieldError{"x", txType, txType.Field(0)}}, + {in: `{"F1":1,"F2":2,"F3":3}`, ptr: new(V), out: V{F1: float64(1), F2: int32(2), F3: Number("3")}}, + {in: `{"F1":1,"F2":2,"F3":3}`, ptr: new(V), out: V{F1: Number("1"), F2: int32(2), F3: Number("3")}, useNumber: true}, + {in: `{"k1":1,"k2":"s","k3":[1,2.0,3e-3],"k4":{"kk1":"s","kk2":2}}`, ptr: new(interface{}), out: ifaceNumAsFloat64}, + {in: `{"k1":1,"k2":"s","k3":[1,2.0,3e-3],"k4":{"kk1":"s","kk2":2}}`, ptr: new(interface{}), out: ifaceNumAsNumber, useNumber: true}, // Z has a "-" tag. - {`{"Y": 1, "Z": 2}`, new(T), T{Y: 1}, nil}, + {in: `{"Y": 1, "Z": 2}`, ptr: new(T), out: T{Y: 1}}, + + {in: `{"alpha": "abc", "alphabet": "xyz"}`, ptr: new(U), out: U{Alphabet: "abc"}}, + {in: `{"alpha": "abc"}`, ptr: new(U), out: U{Alphabet: "abc"}}, + {in: `{"alphabet": "xyz"}`, ptr: new(U), out: U{}}, // syntax errors - {`{"X": "foo", "Y"}`, nil, nil, &SyntaxError{"invalid character '}' after object key", 17}}, - {`[1, 2, 3+]`, nil, nil, &SyntaxError{"invalid character '+' after array element", 9}}, + {in: `{"X": "foo", "Y"}`, err: &SyntaxError{"invalid character '}' after object key", 17}}, + {in: `[1, 2, 3+]`, err: &SyntaxError{"invalid character '+' after array element", 9}}, + {in: `{"X":12x}`, err: &SyntaxError{"invalid character 'x' after object key:value pair", 8}, useNumber: true}, // array tests - {`[1, 2, 3]`, new([3]int), [3]int{1, 2, 3}, nil}, - {`[1, 2, 3]`, new([1]int), [1]int{1}, nil}, - {`[1, 2, 3]`, new([5]int), [5]int{1, 2, 3, 0, 0}, nil}, + {in: `[1, 2, 3]`, ptr: new([3]int), out: [3]int{1, 2, 3}}, + {in: `[1, 2, 3]`, ptr: new([1]int), out: [1]int{1}}, + {in: `[1, 2, 3]`, ptr: new([5]int), out: [5]int{1, 2, 3, 0, 0}}, // composite tests - {allValueIndent, new(All), allValue, nil}, - {allValueCompact, new(All), allValue, nil}, - {allValueIndent, new(*All), &allValue, nil}, - {allValueCompact, new(*All), &allValue, nil}, - {pallValueIndent, new(All), pallValue, nil}, - {pallValueCompact, new(All), pallValue, nil}, - {pallValueIndent, new(*All), &pallValue, nil}, - {pallValueCompact, new(*All), &pallValue, nil}, + {in: allValueIndent, ptr: new(All), out: allValue}, + {in: allValueCompact, ptr: new(All), out: allValue}, + {in: allValueIndent, ptr: new(*All), out: &allValue}, + {in: allValueCompact, ptr: new(*All), out: &allValue}, + {in: pallValueIndent, ptr: new(All), out: pallValue}, + {in: pallValueCompact, ptr: new(All), out: pallValue}, + {in: pallValueIndent, ptr: new(*All), out: &pallValue}, + {in: pallValueCompact, ptr: new(*All), out: &pallValue}, // unmarshal interface test - {`{"T":false}`, &um0, umtrue, nil}, // use "false" so test will fail if custom unmarshaler is not called - {`{"T":false}`, &ump, &umtrue, nil}, - {`[{"T":false}]`, &umslice, umslice, nil}, - {`[{"T":false}]`, &umslicep, &umslice, nil}, - {`{"M":{"T":false}}`, &umstruct, umstruct, nil}, + {in: `{"T":false}`, ptr: &um0, out: umtrue}, // use "false" so test will fail if custom unmarshaler is not called + {in: `{"T":false}`, ptr: &ump, out: &umtrue}, + {in: `[{"T":false}]`, ptr: &umslice, out: umslice}, + {in: `[{"T":false}]`, ptr: &umslicep, out: &umslice}, + {in: `{"M":{"T":false}}`, ptr: &umstruct, out: umstruct}, + + { + in: `{ + "Level0": 1, + "Level1b": 2, + "Level1c": 3, + "x": 4, + "Level1a": 5, + "LEVEL1B": 6, + "e": { + "Level1a": 8, + "Level1b": 9, + "Level1c": 10, + "Level1d": 11, + "x": 12 + }, + "Loop1": 13, + "Loop2": 14, + "X": 15, + "Y": 16, + "Z": 17 + }`, + ptr: new(Top), + out: Top{ + Level0: 1, + Embed0: Embed0{ + Level1b: 2, + Level1c: 3, + }, + Embed0a: &Embed0a{ + Level1a: 5, + Level1b: 6, + }, + Embed0b: &Embed0b{ + Level1a: 8, + Level1b: 9, + Level1c: 10, + Level1d: 11, + Level1e: 12, + }, + Loop: Loop{ + Loop1: 13, + Loop2: 14, + }, + Embed0p: Embed0p{ + Point: image.Point{X: 15, Y: 16}, + }, + Embed0q: Embed0q{ + Point: Point{Z: 17}, + }, + }, + }, + { + in: `{"hello": 1}`, + ptr: new(Ambig), + out: Ambig{First: 1}, + }, + + { + in: `{"X": 1,"Y":2}`, + ptr: new(S5), + out: S5{S8: S8{S9: S9{Y: 2}}}, + }, + { + in: `{"X": 1,"Y":2}`, + ptr: new(S10), + out: S10{S13: S13{S8: S8{S9: S9{Y: 2}}}}, + }, } func TestMarshal(t *testing.T) { @@ -135,6 +344,18 @@ func TestMarshalBadUTF8(t *testing.T) { } } +func TestMarshalNumberZeroVal(t *testing.T) { + var n Number + out, err := Marshal(n) + if err != nil { + t.Fatal(err) + } + outStr := string(out) + if outStr != "0" { + t.Fatalf("Invalid zero val for Number: %q", outStr) + } +} + func TestUnmarshal(t *testing.T) { for i, tt := range unmarshalTests { var scan scanner @@ -150,7 +371,11 @@ func TestUnmarshal(t *testing.T) { } // v = new(right-type) v := reflect.New(reflect.TypeOf(tt.ptr).Elem()) - if err := Unmarshal([]byte(in), v.Interface()); !reflect.DeepEqual(err, tt.err) { + dec := NewDecoder(bytes.NewBuffer(in)) + if tt.useNumber { + dec.UseNumber() + } + if err := dec.Decode(v.Interface()); !reflect.DeepEqual(err, tt.err) { t.Errorf("#%d: %v want %v", i, err, tt.err) continue } @@ -162,6 +387,28 @@ func TestUnmarshal(t *testing.T) { println(string(data)) continue } + + // Check round trip. + if tt.err == nil { + enc, err := Marshal(v.Interface()) + if err != nil { + t.Errorf("#%d: error re-marshaling: %v", i, err) + continue + } + vv := reflect.New(reflect.TypeOf(tt.ptr).Elem()) + dec = NewDecoder(bytes.NewBuffer(enc)) + if tt.useNumber { + dec.UseNumber() + } + if err := dec.Decode(vv.Interface()); err != nil { + t.Errorf("#%d: error re-unmarshaling: %v", i, err) + continue + } + if !reflect.DeepEqual(v.Elem().Interface(), vv.Elem().Interface()) { + t.Errorf("#%d: mismatch\nhave: %#+v\nwant: %#+v", i, v.Elem().Interface(), vv.Elem().Interface()) + continue + } + } } } @@ -182,6 +429,38 @@ func TestUnmarshalMarshal(t *testing.T) { } } +var numberTests = []struct { + in string + i int64 + intErr string + f float64 + floatErr string +}{ + {in: "-1.23e1", intErr: "strconv.ParseInt: parsing \"-1.23e1\": invalid syntax", f: -1.23e1}, + {in: "-12", i: -12, f: -12.0}, + {in: "1e1000", intErr: "strconv.ParseInt: parsing \"1e1000\": invalid syntax", floatErr: "strconv.ParseFloat: parsing \"1e1000\": value out of range"}, +} + +// Independent of Decode, basic coverage of the accessors in Number +func TestNumberAccessors(t *testing.T) { + for _, tt := range numberTests { + n := Number(tt.in) + if s := n.String(); s != tt.in { + t.Errorf("Number(%q).String() is %q", tt.in, s) + } + if i, err := n.Int64(); err == nil && tt.intErr == "" && i != tt.i { + t.Errorf("Number(%q).Int64() is %d", tt.in, i) + } else if (err == nil && tt.intErr != "") || (err != nil && err.Error() != tt.intErr) { + t.Errorf("Number(%q).Int64() wanted error %q but got: %v", tt.in, tt.intErr, err) + } + if f, err := n.Float64(); err == nil && tt.floatErr == "" && f != tt.f { + t.Errorf("Number(%q).Float64() is %g", tt.in, f) + } else if (err == nil && tt.floatErr != "") || (err != nil && err.Error() != tt.floatErr) { + t.Errorf("Number(%q).Float64() wanted error %q but got: %v", tt.in, tt.floatErr, err) + } + } +} + func TestLargeByteSlice(t *testing.T) { s0 := make([]byte, 2000) for i := range s0 { @@ -610,35 +889,6 @@ func TestRefUnmarshal(t *testing.T) { } } -// Test that anonymous fields are ignored. -// We may assign meaning to them later. -func TestAnonymous(t *testing.T) { - type S struct { - T - N int - } - - data, err := Marshal(new(S)) - if err != nil { - t.Fatalf("Marshal: %v", err) - } - want := `{"N":0}` - if string(data) != want { - t.Fatalf("Marshal = %#q, want %#q", string(data), want) - } - - var s S - if err := Unmarshal([]byte(`{"T": 1, "T": {"Y": 1}, "N": 2}`), &s); err != nil { - t.Fatalf("Unmarshal: %v", err) - } - if s.N != 2 { - t.Fatal("Unmarshal: did not set N") - } - if s.T.Y != 0 { - t.Fatal("Unmarshal: did set T.Y") - } -} - // Test that the empty string doesn't panic decoding when ,string is specified // Issue 3450 func TestEmptyString(t *testing.T) { diff --git a/libgo/go/encoding/json/encode.go b/libgo/go/encoding/json/encode.go index d2c1c4424c7..c8535ef79d6 100644 --- a/libgo/go/encoding/json/encode.go +++ b/libgo/go/encoding/json/encode.go @@ -36,7 +36,7 @@ import ( // // Boolean values encode as JSON booleans. // -// Floating point and integer values encode as JSON numbers. +// Floating point, integer, and Number values encode as JSON numbers. // // String values encode as JSON strings, with each invalid UTF-8 sequence // replaced by the encoding of the Unicode replacement character U+FFFD. @@ -84,6 +84,16 @@ import ( // only Unicode letters, digits, dollar signs, percent signs, hyphens, // underscores and slashes. // +// Anonymous struct fields are usually marshaled as if their inner exported fields +// were fields in the outer struct, subject to the usual Go visibility rules. +// An anonymous struct field with a name given in its JSON tag is treated as +// having that name instead of as anonymous. +// +// Handling of anonymous struct fields is new in Go 1.1. +// Prior to Go 1.1, anonymous struct fields were ignored. To force ignoring of +// an anonymous struct field in both current and earlier versions, give the field +// a JSON tag of "-". +// // Map values encode as JSON objects. // The map's key type must be string; the object keys are used directly // as map keys. @@ -312,6 +322,14 @@ func (e *encodeState) reflectValueQuoted(v reflect.Value, quoted bool) { e.Write(b) } case reflect.String: + if v.Type() == numberType { + numStr := v.String() + if numStr == "" { + numStr = "0" // Number's zero-val + } + e.WriteString(numStr) + break + } if quoted { sb, err := Marshal(v.String()) if err != nil { @@ -325,9 +343,9 @@ func (e *encodeState) reflectValueQuoted(v reflect.Value, quoted bool) { case reflect.Struct: e.WriteByte('{') first := true - for _, ef := range encodeFields(v.Type()) { - fieldValue := v.Field(ef.i) - if ef.omitEmpty && isEmptyValue(fieldValue) { + for _, f := range cachedTypeFields(v.Type()) { + fv := fieldByIndex(v, f.index) + if !fv.IsValid() || f.omitEmpty && isEmptyValue(fv) { continue } if first { @@ -335,9 +353,9 @@ func (e *encodeState) reflectValueQuoted(v reflect.Value, quoted bool) { } else { e.WriteByte(',') } - e.string(ef.tag) + e.string(f.name) e.WriteByte(':') - e.reflectValueQuoted(fieldValue, ef.quoted) + e.reflectValueQuoted(fv, f.quoted) } e.WriteByte('}') @@ -432,6 +450,19 @@ func isValidTag(s string) bool { return true } +func fieldByIndex(v reflect.Value, index []int) reflect.Value { + for _, i := range index { + if v.Kind() == reflect.Ptr { + if v.IsNil() { + return reflect.Value{} + } + v = v.Elem() + } + v = v.Field(i) + } + return v +} + // stringValues is a slice of reflect.Value holding *reflect.StringValue. // It implements the methods to sort by string. type stringValues []reflect.Value @@ -490,67 +521,183 @@ func (e *encodeState) string(s string) (int, error) { return e.Len() - len0, nil } -// encodeField contains information about how to encode a field of a -// struct. -type encodeField struct { - i int // field index in struct - tag string - quoted bool +// A field represents a single field found in a struct. +type field struct { + name string + tag bool + index []int + typ reflect.Type omitEmpty bool + quoted bool } -var ( - typeCacheLock sync.RWMutex - encodeFieldsCache = make(map[reflect.Type][]encodeField) -) +// byName sorts field by name, breaking ties with depth, +// then breaking ties with "name came from json tag", then +// breaking ties with index sequence. +type byName []field -// encodeFields returns a slice of encodeField for a given -// struct type. -func encodeFields(t reflect.Type) []encodeField { - typeCacheLock.RLock() - fs, ok := encodeFieldsCache[t] - typeCacheLock.RUnlock() - if ok { - return fs - } +func (x byName) Len() int { return len(x) } + +func (x byName) Swap(i, j int) { x[i], x[j] = x[j], x[i] } - typeCacheLock.Lock() - defer typeCacheLock.Unlock() - fs, ok = encodeFieldsCache[t] - if ok { - return fs +func (x byName) Less(i, j int) bool { + if x[i].name != x[j].name { + return x[i].name < x[j].name } + if len(x[i].index) != len(x[j].index) { + return len(x[i].index) < len(x[j].index) + } + if x[i].tag != x[j].tag { + return x[i].tag + } + return byIndex(x).Less(i, j) +} - v := reflect.Zero(t) - n := v.NumField() - for i := 0; i < n; i++ { - f := t.Field(i) - if f.PkgPath != "" { - continue +// byIndex sorts field by index sequence. +type byIndex []field + +func (x byIndex) Len() int { return len(x) } + +func (x byIndex) Swap(i, j int) { x[i], x[j] = x[j], x[i] } + +func (x byIndex) Less(i, j int) bool { + for k, xik := range x[i].index { + if k >= len(x[j].index) { + return false } - if f.Anonymous { - // We want to do a better job with these later, - // so for now pretend they don't exist. - continue + if xik != x[j].index[k] { + return xik < x[j].index[k] } - var ef encodeField - ef.i = i - ef.tag = f.Name + } + return len(x[i].index) < len(x[j].index) +} + +// typeFields returns a list of fields that JSON should recognize for the given type. +// The algorithm is breadth-first search over the set of structs to include - the top struct +// and then any reachable anonymous structs. +func typeFields(t reflect.Type) []field { + // Anonymous fields to explore at the current level and the next. + current := []field{} + next := []field{{typ: t}} + + // Count of queued names for current level and the next. + count := map[reflect.Type]int{} + nextCount := map[reflect.Type]int{} + + // Types already visited at an earlier level. + visited := map[reflect.Type]bool{} + + // Fields found. + var fields []field + + for len(next) > 0 { + current, next = next, current[:0] + count, nextCount = nextCount, map[reflect.Type]int{} - tv := f.Tag.Get("json") - if tv != "" { - if tv == "-" { + for _, f := range current { + if visited[f.typ] { continue } - name, opts := parseTag(tv) - if isValidTag(name) { - ef.tag = name + visited[f.typ] = true + + // Scan f.typ for fields to include. + for i := 0; i < f.typ.NumField(); i++ { + sf := f.typ.Field(i) + if sf.PkgPath != "" { // unexported + continue + } + tag := sf.Tag.Get("json") + if tag == "-" { + continue + } + name, opts := parseTag(tag) + if !isValidTag(name) { + name = "" + } + index := make([]int, len(f.index)+1) + copy(index, f.index) + index[len(f.index)] = i + // Record found field and index sequence. + if name != "" || !sf.Anonymous { + tagged := name != "" + if name == "" { + name = sf.Name + } + fields = append(fields, field{name, tagged, index, sf.Type, + opts.Contains("omitempty"), opts.Contains("string")}) + if count[f.typ] > 1 { + // If there were multiple instances, add a second, + // so that the annihilation code will see a duplicate. + // It only cares about the distinction between 1 or 2, + // so don't bother generating any more copies. + fields = append(fields, fields[len(fields)-1]) + } + continue + } + + // Record new anonymous struct to explore in next round. + ft := sf.Type + if ft.Name() == "" { + // Must be pointer. + ft = ft.Elem() + } + nextCount[ft]++ + if nextCount[ft] == 1 { + next = append(next, field{name: ft.Name(), index: index, typ: ft}) + } } - ef.omitEmpty = opts.Contains("omitempty") - ef.quoted = opts.Contains("string") } - fs = append(fs, ef) } - encodeFieldsCache[t] = fs - return fs + + sort.Sort(byName(fields)) + + // Remove fields with annihilating name collisions + // and also fields shadowed by fields with explicit JSON tags. + name := "" + out := fields[:0] + for _, f := range fields { + if f.name != name { + name = f.name + out = append(out, f) + continue + } + if n := len(out); n > 0 && out[n-1].name == name && (!out[n-1].tag || f.tag) { + out = out[:n-1] + } + } + fields = out + + sort.Sort(byIndex(fields)) + + return fields +} + +var fieldCache struct { + sync.RWMutex + m map[reflect.Type][]field +} + +// cachedTypeFields is like typeFields but uses a cache to avoid repeated work. +func cachedTypeFields(t reflect.Type) []field { + fieldCache.RLock() + f := fieldCache.m[t] + fieldCache.RUnlock() + if f != nil { + return f + } + + // Compute fields without lock. + // Might duplicate effort but won't hold other computations back. + f = typeFields(t) + if f == nil { + f = []field{} + } + + fieldCache.Lock() + if fieldCache.m == nil { + fieldCache.m = map[reflect.Type][]field{} + } + fieldCache.m[t] = f + fieldCache.Unlock() + return f } diff --git a/libgo/go/encoding/json/stream.go b/libgo/go/encoding/json/stream.go index 7d1cc5f119c..9592467d25b 100644 --- a/libgo/go/encoding/json/stream.go +++ b/libgo/go/encoding/json/stream.go @@ -26,6 +26,10 @@ func NewDecoder(r io.Reader) *Decoder { return &Decoder{r: r} } +// UseNumber causes the Decoder to unmarshal a number into an interface{} as a +// Number instead of as a float64. +func (dec *Decoder) UseNumber() { dec.d.useNumber = true } + // Decode reads the next JSON-encoded value from its // input and stores it in the value pointed to by v. // @@ -74,7 +78,7 @@ Input: // scanEnd is delayed one byte. // We might block trying to get that byte from src, // so instead invent a space byte. - if v == scanEndObject && dec.scan.step(&dec.scan, ' ') == scanEnd { + if (v == scanEndObject || v == scanEndArray) && dec.scan.step(&dec.scan, ' ') == scanEnd { scanp += i + 1 break Input } diff --git a/libgo/go/encoding/json/stream_test.go b/libgo/go/encoding/json/stream_test.go index ce5a7e6d656..4d66f556767 100644 --- a/libgo/go/encoding/json/stream_test.go +++ b/libgo/go/encoding/json/stream_test.go @@ -6,6 +6,7 @@ package json import ( "bytes" + "net" "reflect" "testing" ) @@ -145,3 +146,24 @@ func TestNullRawMessage(t *testing.T) { t.Fatalf("Marshal: have %#q want %#q", b, msg) } } + +var blockingTests = []string{ + `{"x": 1}`, + `[1, 2, 3]`, +} + +func TestBlocking(t *testing.T) { + for _, enc := range blockingTests { + r, w := net.Pipe() + go w.Write([]byte(enc)) + var val interface{} + + // If Decode reads beyond what w.Write writes above, + // it will block, and the test will deadlock. + if err := NewDecoder(r).Decode(&val); err != nil { + t.Errorf("decoding %s: %v", enc, err) + } + r.Close() + w.Close() + } +} diff --git a/libgo/go/encoding/xml/marshal.go b/libgo/go/encoding/xml/marshal.go index 6c3170bdda3..8592a0c15cb 100644 --- a/libgo/go/encoding/xml/marshal.go +++ b/libgo/go/encoding/xml/marshal.go @@ -57,8 +57,8 @@ const ( // if the field value is empty. The empty values are false, 0, any // nil pointer or interface value, and any array, slice, map, or // string of length zero. -// - a non-pointer anonymous struct field is handled as if the -// fields of its value were part of the outer struct. +// - an anonymous struct field is handled as if the fields of its +// value were part of the outer struct. // // If a field uses a tag "a>b>c", then the element c will be nested inside // parent elements a and b. Fields that appear next to each other that name @@ -83,9 +83,7 @@ func MarshalIndent(v interface{}, prefix, indent string) ([]byte, error) { enc := NewEncoder(&b) enc.prefix = prefix enc.indent = indent - err := enc.marshalValue(reflect.ValueOf(v), nil) - enc.Flush() - if err != nil { + if err := enc.Encode(v); err != nil { return nil, err } return b.Bytes(), nil @@ -107,8 +105,10 @@ func NewEncoder(w io.Writer) *Encoder { // of Go values to XML. func (enc *Encoder) Encode(v interface{}) error { err := enc.marshalValue(reflect.ValueOf(v), nil) - enc.Flush() - return err + if err != nil { + return err + } + return enc.Flush() } type printer struct { @@ -164,7 +164,7 @@ func (p *printer) marshalValue(val reflect.Value, finfo *fieldInfo) error { xmlname := tinfo.xmlname if xmlname.name != "" { xmlns, name = xmlname.xmlns, xmlname.name - } else if v, ok := val.FieldByIndex(xmlname.idx).Interface().(Name); ok && v.Local != "" { + } else if v, ok := xmlname.value(val).Interface().(Name); ok && v.Local != "" { xmlns, name = v.Space, v.Local } } @@ -195,7 +195,7 @@ func (p *printer) marshalValue(val reflect.Value, finfo *fieldInfo) error { if finfo.flags&fAttr == 0 { continue } - fv := val.FieldByIndex(finfo.idx) + fv := finfo.value(val) if finfo.flags&fOmitEmpty != 0 && isEmptyValue(fv) { continue } @@ -224,7 +224,7 @@ func (p *printer) marshalValue(val reflect.Value, finfo *fieldInfo) error { p.WriteString(name) p.WriteByte('>') - return nil + return p.cachedWriteError() } var timeType = reflect.TypeOf(time.Time{}) @@ -260,15 +260,15 @@ func (p *printer) marshalSimple(typ reflect.Type, val reflect.Value) error { default: return &UnsupportedTypeError{typ} } - return nil + return p.cachedWriteError() } var ddBytes = []byte("--") func (p *printer) marshalStruct(tinfo *typeInfo, val reflect.Value) error { if val.Type() == timeType { - p.WriteString(val.Interface().(time.Time).Format(time.RFC3339Nano)) - return nil + _, err := p.WriteString(val.Interface().(time.Time).Format(time.RFC3339Nano)) + return err } s := parentStack{printer: p} for i := range tinfo.fields { @@ -276,7 +276,7 @@ func (p *printer) marshalStruct(tinfo *typeInfo, val reflect.Value) error { if finfo.flags&(fAttr|fAny) != 0 { continue } - vf := val.FieldByIndex(finfo.idx) + vf := finfo.value(val) switch finfo.flags & fMode { case fCharData: switch vf.Kind() { @@ -353,7 +353,13 @@ func (p *printer) marshalStruct(tinfo *typeInfo, val reflect.Value) error { } } s.trim(nil) - return nil + return p.cachedWriteError() +} + +// return the bufio Writer's cached write error +func (p *printer) cachedWriteError() error { + _, err := p.Write(nil) + return err } func (p *printer) writeIndent(depthDelta int) { diff --git a/libgo/go/encoding/xml/marshal_test.go b/libgo/go/encoding/xml/marshal_test.go index b6978a1e65b..e729a247af4 100644 --- a/libgo/go/encoding/xml/marshal_test.go +++ b/libgo/go/encoding/xml/marshal_test.go @@ -5,6 +5,9 @@ package xml import ( + "bytes" + "errors" + "io" "reflect" "strconv" "strings" @@ -108,7 +111,7 @@ type EmbedA struct { type EmbedB struct { FieldB string - EmbedC + *EmbedC } type EmbedC struct { @@ -493,7 +496,7 @@ var marshalTests = []struct { }, EmbedB: EmbedB{ FieldB: "A.B.B", - EmbedC: EmbedC{ + EmbedC: &EmbedC{ FieldA1: "A.B.C.A1", FieldA2: "A.B.C.A2", FieldB: "", // Shadowed by A.B.B @@ -779,6 +782,55 @@ func TestUnmarshal(t *testing.T) { } } +type limitedBytesWriter struct { + w io.Writer + remain int // until writes fail +} + +func (lw *limitedBytesWriter) Write(p []byte) (n int, err error) { + if lw.remain <= 0 { + println("error") + return 0, errors.New("write limit hit") + } + if len(p) > lw.remain { + p = p[:lw.remain] + n, _ = lw.w.Write(p) + lw.remain = 0 + return n, errors.New("write limit hit") + } + n, err = lw.w.Write(p) + lw.remain -= n + return n, err +} + +func TestMarshalWriteErrors(t *testing.T) { + var buf bytes.Buffer + const writeCap = 1024 + w := &limitedBytesWriter{&buf, writeCap} + enc := NewEncoder(w) + var err error + var i int + const n = 4000 + for i = 1; i <= n; i++ { + err = enc.Encode(&Passenger{ + Name: []string{"Alice", "Bob"}, + Weight: 5, + }) + if err != nil { + break + } + } + if err == nil { + t.Error("expected an error") + } + if i == n { + t.Errorf("expected to fail before the end") + } + if buf.Len() != writeCap { + t.Errorf("buf.Len() = %d; want %d", buf.Len(), writeCap) + } +} + func BenchmarkMarshal(b *testing.B) { for i := 0; i < b.N; i++ { Marshal(atomValue) diff --git a/libgo/go/encoding/xml/read.go b/libgo/go/encoding/xml/read.go index c2168242091..0e6761d66ad 100644 --- a/libgo/go/encoding/xml/read.go +++ b/libgo/go/encoding/xml/read.go @@ -81,8 +81,8 @@ import ( // of the above rules and the struct has a field with tag ",any", // unmarshal maps the sub-element to that struct field. // -// * A non-pointer anonymous struct field is handled as if the -// fields of its value were part of the outer struct. +// * An anonymous struct field is handled as if the fields of its +// value were part of the outer struct. // // * A struct field with tag "-" is never unmarshalled into. // @@ -248,7 +248,7 @@ func (p *Decoder) unmarshal(val reflect.Value, start *StartElement) error { } return UnmarshalError(e) } - fv := sv.FieldByIndex(finfo.idx) + fv := finfo.value(sv) if _, ok := fv.Interface().(Name); ok { fv.Set(reflect.ValueOf(start.Name)) } @@ -260,7 +260,7 @@ func (p *Decoder) unmarshal(val reflect.Value, start *StartElement) error { finfo := &tinfo.fields[i] switch finfo.flags & fMode { case fAttr: - strv := sv.FieldByIndex(finfo.idx) + strv := finfo.value(sv) // Look for attribute. for _, a := range start.Attr { if a.Name.Local == finfo.name { @@ -271,22 +271,22 @@ func (p *Decoder) unmarshal(val reflect.Value, start *StartElement) error { case fCharData: if !saveData.IsValid() { - saveData = sv.FieldByIndex(finfo.idx) + saveData = finfo.value(sv) } case fComment: if !saveComment.IsValid() { - saveComment = sv.FieldByIndex(finfo.idx) + saveComment = finfo.value(sv) } case fAny: if !saveAny.IsValid() { - saveAny = sv.FieldByIndex(finfo.idx) + saveAny = finfo.value(sv) } case fInnerXml: if !saveXML.IsValid() { - saveXML = sv.FieldByIndex(finfo.idx) + saveXML = finfo.value(sv) if p.saved == nil { saveXMLIndex = 0 p.saved = new(bytes.Buffer) @@ -461,7 +461,7 @@ Loop: } if len(finfo.parents) == len(parents) && finfo.name == start.Name.Local { // It's a perfect match, unmarshal the field. - return true, p.unmarshal(sv.FieldByIndex(finfo.idx), start) + return true, p.unmarshal(finfo.value(sv), start) } if len(finfo.parents) > len(parents) && finfo.parents[len(parents)] == start.Name.Local { // It's a prefix for the field. Break and recurse diff --git a/libgo/go/encoding/xml/typeinfo.go b/libgo/go/encoding/xml/typeinfo.go index 8e2e4508b10..970d1701932 100644 --- a/libgo/go/encoding/xml/typeinfo.go +++ b/libgo/go/encoding/xml/typeinfo.go @@ -66,10 +66,14 @@ func getTypeInfo(typ reflect.Type) (*typeInfo, error) { // For embedded structs, embed its fields. if f.Anonymous { - if f.Type.Kind() != reflect.Struct { + t := f.Type + if t.Kind() == reflect.Ptr { + t = t.Elem() + } + if t.Kind() != reflect.Struct { continue } - inner, err := getTypeInfo(f.Type) + inner, err := getTypeInfo(t) if err != nil { return nil, err } @@ -327,3 +331,22 @@ type TagPathError struct { func (e *TagPathError) Error() string { return fmt.Sprintf("%s field %q with tag %q conflicts with field %q with tag %q", e.Struct, e.Field1, e.Tag1, e.Field2, e.Tag2) } + +// value returns v's field value corresponding to finfo. +// It's equivalent to v.FieldByIndex(finfo.idx), but initializes +// and dereferences pointers as necessary. +func (finfo *fieldInfo) value(v reflect.Value) reflect.Value { + for i, x := range finfo.idx { + if i > 0 { + t := v.Type() + if t.Kind() == reflect.Ptr && t.Elem().Kind() == reflect.Struct { + if v.IsNil() { + v.Set(reflect.New(v.Type().Elem())) + } + v = v.Elem() + } + } + v = v.Field(x) + } + return v +} diff --git a/libgo/go/encoding/xml/xml.go b/libgo/go/encoding/xml/xml.go index 5066f5c0106..fbd2208e334 100644 --- a/libgo/go/encoding/xml/xml.go +++ b/libgo/go/encoding/xml/xml.go @@ -584,6 +584,7 @@ func (d *Decoder) RawToken() (Token, error) { if inquote == 0 && b == '>' && depth == 0 { break } + HandleB: d.buf.WriteByte(b) switch { case b == inquote: @@ -599,7 +600,35 @@ func (d *Decoder) RawToken() (Token, error) { depth-- case b == '<' && inquote == 0: - depth++ + // Look for ]> +]> + --> --> []> +` + +var directivesWithCommentsTokens = []Token{ + CharData("\n"), + Directive(`DOCTYPE []`), + CharData("\n"), + Directive(`DOCTYPE []`), + CharData("\n"), + Directive(`DOCTYPE []`), + CharData("\n"), +} + +func TestDirectivesWithComments(t *testing.T) { + d := NewDecoder(strings.NewReader(directivesWithCommentsInput)) + + for i, want := range directivesWithCommentsTokens { + have, err := d.Token() + if err != nil { + t.Fatalf("token %d: unexpected error: %s", i, err) + } + if !reflect.DeepEqual(have, want) { + t.Errorf("token %d = %#v want %#v", i, have, want) + } + } +} -- cgit v1.2.3