1 // Go support for Protocol Buffers - Google's data interchange format
3 // Copyright 2016 The Go Authors. All rights reserved.
4 // https://github.com/golang/protobuf
6 // Redistribution and use in source and binary forms, with or without
7 // modification, are permitted provided that the following conditions are
10 // * Redistributions of source code must retain the above copyright
11 // notice, this list of conditions and the following disclaimer.
12 // * Redistributions in binary form must reproduce the above
13 // copyright notice, this list of conditions and the following disclaimer
14 // in the documentation and/or other materials provided with the
16 // * Neither the name of Google Inc. nor the names of its
17 // contributors may be used to endorse or promote products derived from
18 // this software without specific prior written permission.
20 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
47 // Unmarshal is the entry point from the generated .pb.go files.
48 // This function is not intended to be used by non-generated code.
49 // This function is not subject to any compatibility guarantee.
50 // msg contains a pointer to a protocol buffer struct.
51 // b is the data to be unmarshaled into the protocol buffer.
52 // a is a pointer to a place to store cached unmarshal information.
53 func (a *InternalMessageInfo) Unmarshal(msg Message, b []byte) error {
54 // Load the unmarshal information for this message type.
55 // The atomic load ensures memory consistency.
56 u := atomicLoadUnmarshalInfo(&a.unmarshal)
58 // Slow path: find unmarshal info for msg, update a with it.
59 u = getUnmarshalInfo(reflect.TypeOf(msg).Elem())
60 atomicStoreUnmarshalInfo(&a.unmarshal, u)
62 // Then do the unmarshaling.
63 err := u.unmarshal(toPointer(&msg), b)
67 type unmarshalInfo struct {
68 typ reflect.Type // type of the protobuf struct
70 // 0 = only typ field is initialized
71 // 1 = completely initialized
73 lock sync.Mutex // prevents double initialization
74 dense []unmarshalFieldInfo // fields indexed by tag #
75 sparse map[uint64]unmarshalFieldInfo // fields indexed by tag #
76 reqFields []string // names of required fields
77 reqMask uint64 // 1<<len(reqFields)-1
78 unrecognized field // offset of []byte to put unrecognized data (or invalidField if we should throw it away)
79 extensions field // offset of extensions field (of type proto.XXX_InternalExtensions), or invalidField if it does not exist
80 oldExtensions field // offset of old-form extensions field (of type map[int]Extension)
81 extensionRanges []ExtensionRange // if non-nil, implies extensions field is valid
82 isMessageSet bool // if true, implies extensions field is valid
85 // An unmarshaler takes a stream of bytes and a pointer to a field of a message.
86 // It decodes the field, stores it at f, and returns the unused bytes.
87 // w is the wire encoding.
88 // b is the data after the tag and wire encoding have been read.
89 type unmarshaler func(b []byte, f pointer, w int) ([]byte, error)
91 type unmarshalFieldInfo struct {
92 // location of the field in the proto message structure.
95 // function to unmarshal the data for the field.
98 // if a required field, contains a single set bit at this field's index in the required field list.
101 name string // name of the field, for error reporting
105 unmarshalInfoMap = map[reflect.Type]*unmarshalInfo{}
106 unmarshalInfoLock sync.Mutex
109 // getUnmarshalInfo returns the data structure which can be
110 // subsequently used to unmarshal a message of the given type.
111 // t is the type of the message (note: not pointer to message).
112 func getUnmarshalInfo(t reflect.Type) *unmarshalInfo {
113 // It would be correct to return a new unmarshalInfo
114 // unconditionally. We would end up allocating one
115 // per occurrence of that type as a message or submessage.
116 // We use a cache here just to reduce memory usage.
117 unmarshalInfoLock.Lock()
118 defer unmarshalInfoLock.Unlock()
119 u := unmarshalInfoMap[t]
121 u = &unmarshalInfo{typ: t}
122 // Note: we just set the type here. The rest of the fields
123 // will be initialized on first use.
124 unmarshalInfoMap[t] = u
129 // unmarshal does the main work of unmarshaling a message.
130 // u provides type information used to unmarshal the message.
131 // m is a pointer to a protocol buffer message.
132 // b is a byte stream to unmarshal into m.
133 // This is top routine used when recursively unmarshaling submessages.
134 func (u *unmarshalInfo) unmarshal(m pointer, b []byte) error {
135 if atomic.LoadInt32(&u.initialized) == 0 {
136 u.computeUnmarshalInfo()
139 return unmarshalMessageSet(b, m.offset(u.extensions).toExtensions())
141 var reqMask uint64 // bitmask of required fields we've seen.
144 // Read tag and wire type.
145 // Special case 1 and 2 byte varints.
150 } else if len(b) >= 2 && b[1] < 128 {
151 x = uint64(b[0]&0x7f) + uint64(b[1])<<7
155 x, n = decodeVarint(b)
157 return io.ErrUnexpectedEOF
164 // Dispatch on the tag to one of the unmarshal* functions below.
165 var f unmarshalFieldInfo
166 if tag < uint64(len(u.dense)) {
171 if fn := f.unmarshal; fn != nil {
173 b, err = fn(b, m.offset(f.field), wire)
178 if r, ok := err.(*RequiredNotSetError); ok {
179 // Remember this error, but keep parsing. We need to produce
180 // a full parse even if a required field is missing.
187 if err != errInternalBadWireType {
188 if err == errInvalidUTF8 {
190 fullName := revProtoTypes[reflect.PtrTo(u.typ)] + "." + f.name
191 errLater = &invalidUTF8Error{fullName}
197 // Fragments with bad wire type are treated as unknown fields.
201 if !u.unrecognized.IsValid() {
202 // Don't keep unrecognized data; just skip it.
204 b, err = skipField(b, wire)
210 // Keep unrecognized data around.
211 // maybe in extensions, maybe in the unrecognized field.
212 z := m.offset(u.unrecognized).toBytes()
213 var emap map[int32]Extension
215 for _, r := range u.extensionRanges {
216 if uint64(r.Start) <= tag && tag <= uint64(r.End) {
217 if u.extensions.IsValid() {
218 mp := m.offset(u.extensions).toExtensions()
219 emap = mp.extensionsWrite()
224 if u.oldExtensions.IsValid() {
225 p := m.offset(u.oldExtensions).toOldExtensions()
228 emap = map[int32]Extension{}
235 panic("no extensions field available")
239 // Use wire type to skip data.
242 b, err = skipField(b, wire)
246 *z = encodeVarint(*z, tag<<3|uint64(wire))
247 *z = append(*z, b0[:len(b0)-len(b)]...)
253 if reqMask != u.reqMask && errLater == nil {
254 // A required field of this message is missing.
255 for _, n := range u.reqFields {
257 errLater = &RequiredNotSetError{n}
265 // computeUnmarshalInfo fills in u with information for use
266 // in unmarshaling protocol buffers of type u.typ.
267 func (u *unmarshalInfo) computeUnmarshalInfo() {
269 defer u.lock.Unlock()
270 if u.initialized != 0 {
276 // Set up the "not found" value for the unrecognized byte buffer.
277 // This is the default for proto3.
278 u.unrecognized = invalidField
279 u.extensions = invalidField
280 u.oldExtensions = invalidField
282 // List of the generated type and offset for each oneof field.
283 type oneofField struct {
284 ityp reflect.Type // interface type of oneof field
285 field field // offset in containing message
287 var oneofFields []oneofField
289 for i := 0; i < n; i++ {
291 if f.Name == "XXX_unrecognized" {
292 // The byte slice used to hold unrecognized input is special.
293 if f.Type != reflect.TypeOf(([]byte)(nil)) {
294 panic("bad type for XXX_unrecognized field: " + f.Type.Name())
296 u.unrecognized = toField(&f)
299 if f.Name == "XXX_InternalExtensions" {
301 if f.Type != reflect.TypeOf(XXX_InternalExtensions{}) {
302 panic("bad type for XXX_InternalExtensions field: " + f.Type.Name())
304 u.extensions = toField(&f)
305 if f.Tag.Get("protobuf_messageset") == "1" {
306 u.isMessageSet = true
310 if f.Name == "XXX_extensions" {
311 // An older form of the extensions field.
312 if f.Type != reflect.TypeOf((map[int32]Extension)(nil)) {
313 panic("bad type for XXX_extensions field: " + f.Type.Name())
315 u.oldExtensions = toField(&f)
318 if f.Name == "XXX_NoUnkeyedLiteral" || f.Name == "XXX_sizecache" {
322 oneof := f.Tag.Get("protobuf_oneof")
324 oneofFields = append(oneofFields, oneofField{f.Type, toField(&f)})
325 // The rest of oneof processing happens below.
329 tags := f.Tag.Get("protobuf")
330 tagArray := strings.Split(tags, ",")
331 if len(tagArray) < 2 {
332 panic("protobuf tag not enough fields in " + t.Name() + "." + f.Name + ": " + tags)
334 tag, err := strconv.Atoi(tagArray[1])
336 panic("protobuf tag field not an integer: " + tagArray[1])
340 for _, tag := range tagArray[3:] {
341 if strings.HasPrefix(tag, "name=") {
346 // Extract unmarshaling function from the field (its type and tags).
347 unmarshal := fieldUnmarshaler(&f)
351 if tagArray[2] == "req" {
352 bit := len(u.reqFields)
353 u.reqFields = append(u.reqFields, name)
354 reqMask = uint64(1) << uint(bit)
355 // TODO: if we have more than 64 required fields, we end up
356 // not verifying that all required fields are present.
357 // Fix this, perhaps using a count of required fields?
360 // Store the info in the correct slot in the message.
361 u.setTag(tag, toField(&f), unmarshal, reqMask, name)
364 // Find any types associated with oneof fields.
365 var oneofImplementers []interface{}
366 switch m := reflect.Zero(reflect.PtrTo(t)).Interface().(type) {
367 case oneofFuncsIface:
368 _, _, _, oneofImplementers = m.XXX_OneofFuncs()
369 case oneofWrappersIface:
370 oneofImplementers = m.XXX_OneofWrappers()
372 for _, v := range oneofImplementers {
373 tptr := reflect.TypeOf(v) // *Msg_X
374 typ := tptr.Elem() // Msg_X
376 f := typ.Field(0) // oneof implementers have one field
377 baseUnmarshal := fieldUnmarshaler(&f)
378 tags := strings.Split(f.Tag.Get("protobuf"), ",")
379 fieldNum, err := strconv.Atoi(tags[1])
381 panic("protobuf tag field not an integer: " + tags[1])
384 for _, tag := range tags {
385 if strings.HasPrefix(tag, "name=") {
386 name = strings.TrimPrefix(tag, "name=")
391 // Find the oneof field that this struct implements.
392 // Might take O(n^2) to process all of the oneofs, but who cares.
393 for _, of := range oneofFields {
394 if tptr.Implements(of.ityp) {
395 // We have found the corresponding interface for this struct.
396 // That lets us know where this struct should be stored
397 // when we encounter it during unmarshaling.
398 unmarshal := makeUnmarshalOneof(typ, of.ityp, baseUnmarshal)
399 u.setTag(fieldNum, of.field, unmarshal, 0, name)
405 // Get extension ranges, if any.
406 fn := reflect.Zero(reflect.PtrTo(t)).MethodByName("ExtensionRangeArray")
408 if !u.extensions.IsValid() && !u.oldExtensions.IsValid() {
409 panic("a message with extensions, but no extensions field in " + t.Name())
411 u.extensionRanges = fn.Call(nil)[0].Interface().([]ExtensionRange)
414 // Explicitly disallow tag 0. This will ensure we flag an error
415 // when decoding a buffer of all zeros. Without this code, we
416 // would decode and skip an all-zero buffer of even length.
417 // [0 0] is [tag=0/wiretype=varint varint-encoded-0].
418 u.setTag(0, zeroField, func(b []byte, f pointer, w int) ([]byte, error) {
419 return nil, fmt.Errorf("proto: %s: illegal tag 0 (wire type %d)", t, w)
422 // Set mask for required field check.
423 u.reqMask = uint64(1)<<uint(len(u.reqFields)) - 1
425 atomic.StoreInt32(&u.initialized, 1)
428 // setTag stores the unmarshal information for the given tag.
429 // tag = tag # for field
430 // field/unmarshal = unmarshal info for that field.
431 // reqMask = if required, bitmask for field position in required field list. 0 otherwise.
432 // name = short name of the field.
433 func (u *unmarshalInfo) setTag(tag int, field field, unmarshal unmarshaler, reqMask uint64, name string) {
434 i := unmarshalFieldInfo{field: field, unmarshal: unmarshal, reqMask: reqMask, name: name}
435 n := u.typ.NumField()
436 if tag >= 0 && (tag < 16 || tag < 2*n) { // TODO: what are the right numbers here?
437 for len(u.dense) <= tag {
438 u.dense = append(u.dense, unmarshalFieldInfo{})
444 u.sparse = map[uint64]unmarshalFieldInfo{}
446 u.sparse[uint64(tag)] = i
449 // fieldUnmarshaler returns an unmarshaler for the given field.
450 func fieldUnmarshaler(f *reflect.StructField) unmarshaler {
451 if f.Type.Kind() == reflect.Map {
452 return makeUnmarshalMap(f)
454 return typeUnmarshaler(f.Type, f.Tag.Get("protobuf"))
457 // typeUnmarshaler returns an unmarshaler for the given field type / field tag pair.
458 func typeUnmarshaler(t reflect.Type, tags string) unmarshaler {
459 tagArray := strings.Split(tags, ",")
460 encoding := tagArray[0]
464 for _, tag := range tagArray[3:] {
465 if strings.HasPrefix(tag, "name=") {
472 validateUTF8 = validateUTF8 && proto3
474 // Figure out packaging (pointer, slice, or both)
477 if t.Kind() == reflect.Slice && t.Elem().Kind() != reflect.Uint8 {
481 if t.Kind() == reflect.Ptr {
486 // We'll never have both pointer and slice for basic types.
487 if pointer && slice && t.Kind() != reflect.Struct {
488 panic("both pointer and slice for basic type in " + t.Name())
494 return unmarshalBoolPtr
497 return unmarshalBoolSlice
499 return unmarshalBoolValue
504 return unmarshalFixedS32Ptr
507 return unmarshalFixedS32Slice
509 return unmarshalFixedS32Value
511 // this could be int32 or enum
513 return unmarshalInt32Ptr
516 return unmarshalInt32Slice
518 return unmarshalInt32Value
521 return unmarshalSint32Ptr
524 return unmarshalSint32Slice
526 return unmarshalSint32Value
532 return unmarshalFixedS64Ptr
535 return unmarshalFixedS64Slice
537 return unmarshalFixedS64Value
540 return unmarshalInt64Ptr
543 return unmarshalInt64Slice
545 return unmarshalInt64Value
548 return unmarshalSint64Ptr
551 return unmarshalSint64Slice
553 return unmarshalSint64Value
559 return unmarshalFixed32Ptr
562 return unmarshalFixed32Slice
564 return unmarshalFixed32Value
567 return unmarshalUint32Ptr
570 return unmarshalUint32Slice
572 return unmarshalUint32Value
578 return unmarshalFixed64Ptr
581 return unmarshalFixed64Slice
583 return unmarshalFixed64Value
586 return unmarshalUint64Ptr
589 return unmarshalUint64Slice
591 return unmarshalUint64Value
593 case reflect.Float32:
595 return unmarshalFloat32Ptr
598 return unmarshalFloat32Slice
600 return unmarshalFloat32Value
601 case reflect.Float64:
603 return unmarshalFloat64Ptr
606 return unmarshalFloat64Slice
608 return unmarshalFloat64Value
610 panic("map type in typeUnmarshaler in " + t.Name())
613 panic("bad pointer in slice case in " + t.Name())
616 return unmarshalBytesSlice
618 return unmarshalBytesValue
622 return unmarshalUTF8StringPtr
625 return unmarshalUTF8StringSlice
627 return unmarshalUTF8StringValue
630 return unmarshalStringPtr
633 return unmarshalStringSlice
635 return unmarshalStringValue
637 // message or group field
639 panic(fmt.Sprintf("message/group field %s:%s without pointer", t, encoding))
644 return makeUnmarshalMessageSlicePtr(getUnmarshalInfo(t), name)
646 return makeUnmarshalMessagePtr(getUnmarshalInfo(t), name)
649 return makeUnmarshalGroupSlicePtr(getUnmarshalInfo(t), name)
651 return makeUnmarshalGroupPtr(getUnmarshalInfo(t), name)
654 panic(fmt.Sprintf("unmarshaler not found type:%s encoding:%s", t, encoding))
657 // Below are all the unmarshalers for individual fields of various types.
659 func unmarshalInt64Value(b []byte, f pointer, w int) ([]byte, error) {
661 return b, errInternalBadWireType
663 x, n := decodeVarint(b)
665 return nil, io.ErrUnexpectedEOF
673 func unmarshalInt64Ptr(b []byte, f pointer, w int) ([]byte, error) {
675 return b, errInternalBadWireType
677 x, n := decodeVarint(b)
679 return nil, io.ErrUnexpectedEOF
687 func unmarshalInt64Slice(b []byte, f pointer, w int) ([]byte, error) {
688 if w == WireBytes { // packed
689 x, n := decodeVarint(b)
691 return nil, io.ErrUnexpectedEOF
694 if x > uint64(len(b)) {
695 return nil, io.ErrUnexpectedEOF
700 x, n = decodeVarint(b)
702 return nil, io.ErrUnexpectedEOF
706 s := f.toInt64Slice()
712 return b, errInternalBadWireType
714 x, n := decodeVarint(b)
716 return nil, io.ErrUnexpectedEOF
720 s := f.toInt64Slice()
725 func unmarshalSint64Value(b []byte, f pointer, w int) ([]byte, error) {
727 return b, errInternalBadWireType
729 x, n := decodeVarint(b)
731 return nil, io.ErrUnexpectedEOF
734 v := int64(x>>1) ^ int64(x)<<63>>63
739 func unmarshalSint64Ptr(b []byte, f pointer, w int) ([]byte, error) {
741 return b, errInternalBadWireType
743 x, n := decodeVarint(b)
745 return nil, io.ErrUnexpectedEOF
748 v := int64(x>>1) ^ int64(x)<<63>>63
753 func unmarshalSint64Slice(b []byte, f pointer, w int) ([]byte, error) {
754 if w == WireBytes { // packed
755 x, n := decodeVarint(b)
757 return nil, io.ErrUnexpectedEOF
760 if x > uint64(len(b)) {
761 return nil, io.ErrUnexpectedEOF
766 x, n = decodeVarint(b)
768 return nil, io.ErrUnexpectedEOF
771 v := int64(x>>1) ^ int64(x)<<63>>63
772 s := f.toInt64Slice()
778 return b, errInternalBadWireType
780 x, n := decodeVarint(b)
782 return nil, io.ErrUnexpectedEOF
785 v := int64(x>>1) ^ int64(x)<<63>>63
786 s := f.toInt64Slice()
791 func unmarshalUint64Value(b []byte, f pointer, w int) ([]byte, error) {
793 return b, errInternalBadWireType
795 x, n := decodeVarint(b)
797 return nil, io.ErrUnexpectedEOF
805 func unmarshalUint64Ptr(b []byte, f pointer, w int) ([]byte, error) {
807 return b, errInternalBadWireType
809 x, n := decodeVarint(b)
811 return nil, io.ErrUnexpectedEOF
815 *f.toUint64Ptr() = &v
819 func unmarshalUint64Slice(b []byte, f pointer, w int) ([]byte, error) {
820 if w == WireBytes { // packed
821 x, n := decodeVarint(b)
823 return nil, io.ErrUnexpectedEOF
826 if x > uint64(len(b)) {
827 return nil, io.ErrUnexpectedEOF
832 x, n = decodeVarint(b)
834 return nil, io.ErrUnexpectedEOF
838 s := f.toUint64Slice()
844 return b, errInternalBadWireType
846 x, n := decodeVarint(b)
848 return nil, io.ErrUnexpectedEOF
852 s := f.toUint64Slice()
857 func unmarshalInt32Value(b []byte, f pointer, w int) ([]byte, error) {
859 return b, errInternalBadWireType
861 x, n := decodeVarint(b)
863 return nil, io.ErrUnexpectedEOF
871 func unmarshalInt32Ptr(b []byte, f pointer, w int) ([]byte, error) {
873 return b, errInternalBadWireType
875 x, n := decodeVarint(b)
877 return nil, io.ErrUnexpectedEOF
885 func unmarshalInt32Slice(b []byte, f pointer, w int) ([]byte, error) {
886 if w == WireBytes { // packed
887 x, n := decodeVarint(b)
889 return nil, io.ErrUnexpectedEOF
892 if x > uint64(len(b)) {
893 return nil, io.ErrUnexpectedEOF
898 x, n = decodeVarint(b)
900 return nil, io.ErrUnexpectedEOF
904 f.appendInt32Slice(v)
909 return b, errInternalBadWireType
911 x, n := decodeVarint(b)
913 return nil, io.ErrUnexpectedEOF
917 f.appendInt32Slice(v)
921 func unmarshalSint32Value(b []byte, f pointer, w int) ([]byte, error) {
923 return b, errInternalBadWireType
925 x, n := decodeVarint(b)
927 return nil, io.ErrUnexpectedEOF
930 v := int32(x>>1) ^ int32(x)<<31>>31
935 func unmarshalSint32Ptr(b []byte, f pointer, w int) ([]byte, error) {
937 return b, errInternalBadWireType
939 x, n := decodeVarint(b)
941 return nil, io.ErrUnexpectedEOF
944 v := int32(x>>1) ^ int32(x)<<31>>31
949 func unmarshalSint32Slice(b []byte, f pointer, w int) ([]byte, error) {
950 if w == WireBytes { // packed
951 x, n := decodeVarint(b)
953 return nil, io.ErrUnexpectedEOF
956 if x > uint64(len(b)) {
957 return nil, io.ErrUnexpectedEOF
962 x, n = decodeVarint(b)
964 return nil, io.ErrUnexpectedEOF
967 v := int32(x>>1) ^ int32(x)<<31>>31
968 f.appendInt32Slice(v)
973 return b, errInternalBadWireType
975 x, n := decodeVarint(b)
977 return nil, io.ErrUnexpectedEOF
980 v := int32(x>>1) ^ int32(x)<<31>>31
981 f.appendInt32Slice(v)
985 func unmarshalUint32Value(b []byte, f pointer, w int) ([]byte, error) {
987 return b, errInternalBadWireType
989 x, n := decodeVarint(b)
991 return nil, io.ErrUnexpectedEOF
999 func unmarshalUint32Ptr(b []byte, f pointer, w int) ([]byte, error) {
1000 if w != WireVarint {
1001 return b, errInternalBadWireType
1003 x, n := decodeVarint(b)
1005 return nil, io.ErrUnexpectedEOF
1009 *f.toUint32Ptr() = &v
1013 func unmarshalUint32Slice(b []byte, f pointer, w int) ([]byte, error) {
1014 if w == WireBytes { // packed
1015 x, n := decodeVarint(b)
1017 return nil, io.ErrUnexpectedEOF
1020 if x > uint64(len(b)) {
1021 return nil, io.ErrUnexpectedEOF
1026 x, n = decodeVarint(b)
1028 return nil, io.ErrUnexpectedEOF
1032 s := f.toUint32Slice()
1037 if w != WireVarint {
1038 return b, errInternalBadWireType
1040 x, n := decodeVarint(b)
1042 return nil, io.ErrUnexpectedEOF
1046 s := f.toUint32Slice()
1051 func unmarshalFixed64Value(b []byte, f pointer, w int) ([]byte, error) {
1052 if w != WireFixed64 {
1053 return b, errInternalBadWireType
1056 return nil, io.ErrUnexpectedEOF
1058 v := uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56
1063 func unmarshalFixed64Ptr(b []byte, f pointer, w int) ([]byte, error) {
1064 if w != WireFixed64 {
1065 return b, errInternalBadWireType
1068 return nil, io.ErrUnexpectedEOF
1070 v := uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56
1071 *f.toUint64Ptr() = &v
1075 func unmarshalFixed64Slice(b []byte, f pointer, w int) ([]byte, error) {
1076 if w == WireBytes { // packed
1077 x, n := decodeVarint(b)
1079 return nil, io.ErrUnexpectedEOF
1082 if x > uint64(len(b)) {
1083 return nil, io.ErrUnexpectedEOF
1089 return nil, io.ErrUnexpectedEOF
1091 v := uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56
1092 s := f.toUint64Slice()
1098 if w != WireFixed64 {
1099 return b, errInternalBadWireType
1102 return nil, io.ErrUnexpectedEOF
1104 v := uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56
1105 s := f.toUint64Slice()
1110 func unmarshalFixedS64Value(b []byte, f pointer, w int) ([]byte, error) {
1111 if w != WireFixed64 {
1112 return b, errInternalBadWireType
1115 return nil, io.ErrUnexpectedEOF
1117 v := int64(b[0]) | int64(b[1])<<8 | int64(b[2])<<16 | int64(b[3])<<24 | int64(b[4])<<32 | int64(b[5])<<40 | int64(b[6])<<48 | int64(b[7])<<56
1122 func unmarshalFixedS64Ptr(b []byte, f pointer, w int) ([]byte, error) {
1123 if w != WireFixed64 {
1124 return b, errInternalBadWireType
1127 return nil, io.ErrUnexpectedEOF
1129 v := int64(b[0]) | int64(b[1])<<8 | int64(b[2])<<16 | int64(b[3])<<24 | int64(b[4])<<32 | int64(b[5])<<40 | int64(b[6])<<48 | int64(b[7])<<56
1130 *f.toInt64Ptr() = &v
1134 func unmarshalFixedS64Slice(b []byte, f pointer, w int) ([]byte, error) {
1135 if w == WireBytes { // packed
1136 x, n := decodeVarint(b)
1138 return nil, io.ErrUnexpectedEOF
1141 if x > uint64(len(b)) {
1142 return nil, io.ErrUnexpectedEOF
1148 return nil, io.ErrUnexpectedEOF
1150 v := int64(b[0]) | int64(b[1])<<8 | int64(b[2])<<16 | int64(b[3])<<24 | int64(b[4])<<32 | int64(b[5])<<40 | int64(b[6])<<48 | int64(b[7])<<56
1151 s := f.toInt64Slice()
1157 if w != WireFixed64 {
1158 return b, errInternalBadWireType
1161 return nil, io.ErrUnexpectedEOF
1163 v := int64(b[0]) | int64(b[1])<<8 | int64(b[2])<<16 | int64(b[3])<<24 | int64(b[4])<<32 | int64(b[5])<<40 | int64(b[6])<<48 | int64(b[7])<<56
1164 s := f.toInt64Slice()
1169 func unmarshalFixed32Value(b []byte, f pointer, w int) ([]byte, error) {
1170 if w != WireFixed32 {
1171 return b, errInternalBadWireType
1174 return nil, io.ErrUnexpectedEOF
1176 v := uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24
1181 func unmarshalFixed32Ptr(b []byte, f pointer, w int) ([]byte, error) {
1182 if w != WireFixed32 {
1183 return b, errInternalBadWireType
1186 return nil, io.ErrUnexpectedEOF
1188 v := uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24
1189 *f.toUint32Ptr() = &v
1193 func unmarshalFixed32Slice(b []byte, f pointer, w int) ([]byte, error) {
1194 if w == WireBytes { // packed
1195 x, n := decodeVarint(b)
1197 return nil, io.ErrUnexpectedEOF
1200 if x > uint64(len(b)) {
1201 return nil, io.ErrUnexpectedEOF
1207 return nil, io.ErrUnexpectedEOF
1209 v := uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24
1210 s := f.toUint32Slice()
1216 if w != WireFixed32 {
1217 return b, errInternalBadWireType
1220 return nil, io.ErrUnexpectedEOF
1222 v := uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24
1223 s := f.toUint32Slice()
1228 func unmarshalFixedS32Value(b []byte, f pointer, w int) ([]byte, error) {
1229 if w != WireFixed32 {
1230 return b, errInternalBadWireType
1233 return nil, io.ErrUnexpectedEOF
1235 v := int32(b[0]) | int32(b[1])<<8 | int32(b[2])<<16 | int32(b[3])<<24
1240 func unmarshalFixedS32Ptr(b []byte, f pointer, w int) ([]byte, error) {
1241 if w != WireFixed32 {
1242 return b, errInternalBadWireType
1245 return nil, io.ErrUnexpectedEOF
1247 v := int32(b[0]) | int32(b[1])<<8 | int32(b[2])<<16 | int32(b[3])<<24
1252 func unmarshalFixedS32Slice(b []byte, f pointer, w int) ([]byte, error) {
1253 if w == WireBytes { // packed
1254 x, n := decodeVarint(b)
1256 return nil, io.ErrUnexpectedEOF
1259 if x > uint64(len(b)) {
1260 return nil, io.ErrUnexpectedEOF
1266 return nil, io.ErrUnexpectedEOF
1268 v := int32(b[0]) | int32(b[1])<<8 | int32(b[2])<<16 | int32(b[3])<<24
1269 f.appendInt32Slice(v)
1274 if w != WireFixed32 {
1275 return b, errInternalBadWireType
1278 return nil, io.ErrUnexpectedEOF
1280 v := int32(b[0]) | int32(b[1])<<8 | int32(b[2])<<16 | int32(b[3])<<24
1281 f.appendInt32Slice(v)
1285 func unmarshalBoolValue(b []byte, f pointer, w int) ([]byte, error) {
1286 if w != WireVarint {
1287 return b, errInternalBadWireType
1289 // Note: any length varint is allowed, even though any sane
1290 // encoder will use one byte.
1291 // See https://github.com/golang/protobuf/issues/76
1292 x, n := decodeVarint(b)
1294 return nil, io.ErrUnexpectedEOF
1296 // TODO: check if x>1? Tests seem to indicate no.
1302 func unmarshalBoolPtr(b []byte, f pointer, w int) ([]byte, error) {
1303 if w != WireVarint {
1304 return b, errInternalBadWireType
1306 x, n := decodeVarint(b)
1308 return nil, io.ErrUnexpectedEOF
1315 func unmarshalBoolSlice(b []byte, f pointer, w int) ([]byte, error) {
1316 if w == WireBytes { // packed
1317 x, n := decodeVarint(b)
1319 return nil, io.ErrUnexpectedEOF
1322 if x > uint64(len(b)) {
1323 return nil, io.ErrUnexpectedEOF
1328 x, n = decodeVarint(b)
1330 return nil, io.ErrUnexpectedEOF
1333 s := f.toBoolSlice()
1339 if w != WireVarint {
1340 return b, errInternalBadWireType
1342 x, n := decodeVarint(b)
1344 return nil, io.ErrUnexpectedEOF
1347 s := f.toBoolSlice()
1352 func unmarshalFloat64Value(b []byte, f pointer, w int) ([]byte, error) {
1353 if w != WireFixed64 {
1354 return b, errInternalBadWireType
1357 return nil, io.ErrUnexpectedEOF
1359 v := math.Float64frombits(uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56)
1364 func unmarshalFloat64Ptr(b []byte, f pointer, w int) ([]byte, error) {
1365 if w != WireFixed64 {
1366 return b, errInternalBadWireType
1369 return nil, io.ErrUnexpectedEOF
1371 v := math.Float64frombits(uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56)
1372 *f.toFloat64Ptr() = &v
1376 func unmarshalFloat64Slice(b []byte, f pointer, w int) ([]byte, error) {
1377 if w == WireBytes { // packed
1378 x, n := decodeVarint(b)
1380 return nil, io.ErrUnexpectedEOF
1383 if x > uint64(len(b)) {
1384 return nil, io.ErrUnexpectedEOF
1390 return nil, io.ErrUnexpectedEOF
1392 v := math.Float64frombits(uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56)
1393 s := f.toFloat64Slice()
1399 if w != WireFixed64 {
1400 return b, errInternalBadWireType
1403 return nil, io.ErrUnexpectedEOF
1405 v := math.Float64frombits(uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56)
1406 s := f.toFloat64Slice()
1411 func unmarshalFloat32Value(b []byte, f pointer, w int) ([]byte, error) {
1412 if w != WireFixed32 {
1413 return b, errInternalBadWireType
1416 return nil, io.ErrUnexpectedEOF
1418 v := math.Float32frombits(uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24)
1423 func unmarshalFloat32Ptr(b []byte, f pointer, w int) ([]byte, error) {
1424 if w != WireFixed32 {
1425 return b, errInternalBadWireType
1428 return nil, io.ErrUnexpectedEOF
1430 v := math.Float32frombits(uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24)
1431 *f.toFloat32Ptr() = &v
1435 func unmarshalFloat32Slice(b []byte, f pointer, w int) ([]byte, error) {
1436 if w == WireBytes { // packed
1437 x, n := decodeVarint(b)
1439 return nil, io.ErrUnexpectedEOF
1442 if x > uint64(len(b)) {
1443 return nil, io.ErrUnexpectedEOF
1449 return nil, io.ErrUnexpectedEOF
1451 v := math.Float32frombits(uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24)
1452 s := f.toFloat32Slice()
1458 if w != WireFixed32 {
1459 return b, errInternalBadWireType
1462 return nil, io.ErrUnexpectedEOF
1464 v := math.Float32frombits(uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24)
1465 s := f.toFloat32Slice()
1470 func unmarshalStringValue(b []byte, f pointer, w int) ([]byte, error) {
1472 return b, errInternalBadWireType
1474 x, n := decodeVarint(b)
1476 return nil, io.ErrUnexpectedEOF
1479 if x > uint64(len(b)) {
1480 return nil, io.ErrUnexpectedEOF
1487 func unmarshalStringPtr(b []byte, f pointer, w int) ([]byte, error) {
1489 return b, errInternalBadWireType
1491 x, n := decodeVarint(b)
1493 return nil, io.ErrUnexpectedEOF
1496 if x > uint64(len(b)) {
1497 return nil, io.ErrUnexpectedEOF
1500 *f.toStringPtr() = &v
1504 func unmarshalStringSlice(b []byte, f pointer, w int) ([]byte, error) {
1506 return b, errInternalBadWireType
1508 x, n := decodeVarint(b)
1510 return nil, io.ErrUnexpectedEOF
1513 if x > uint64(len(b)) {
1514 return nil, io.ErrUnexpectedEOF
1517 s := f.toStringSlice()
1522 func unmarshalUTF8StringValue(b []byte, f pointer, w int) ([]byte, error) {
1524 return b, errInternalBadWireType
1526 x, n := decodeVarint(b)
1528 return nil, io.ErrUnexpectedEOF
1531 if x > uint64(len(b)) {
1532 return nil, io.ErrUnexpectedEOF
1536 if !utf8.ValidString(v) {
1537 return b[x:], errInvalidUTF8
1542 func unmarshalUTF8StringPtr(b []byte, f pointer, w int) ([]byte, error) {
1544 return b, errInternalBadWireType
1546 x, n := decodeVarint(b)
1548 return nil, io.ErrUnexpectedEOF
1551 if x > uint64(len(b)) {
1552 return nil, io.ErrUnexpectedEOF
1555 *f.toStringPtr() = &v
1556 if !utf8.ValidString(v) {
1557 return b[x:], errInvalidUTF8
1562 func unmarshalUTF8StringSlice(b []byte, f pointer, w int) ([]byte, error) {
1564 return b, errInternalBadWireType
1566 x, n := decodeVarint(b)
1568 return nil, io.ErrUnexpectedEOF
1571 if x > uint64(len(b)) {
1572 return nil, io.ErrUnexpectedEOF
1575 s := f.toStringSlice()
1577 if !utf8.ValidString(v) {
1578 return b[x:], errInvalidUTF8
1583 var emptyBuf [0]byte
1585 func unmarshalBytesValue(b []byte, f pointer, w int) ([]byte, error) {
1587 return b, errInternalBadWireType
1589 x, n := decodeVarint(b)
1591 return nil, io.ErrUnexpectedEOF
1594 if x > uint64(len(b)) {
1595 return nil, io.ErrUnexpectedEOF
1597 // The use of append here is a trick which avoids the zeroing
1598 // that would be required if we used a make/copy pair.
1599 // We append to emptyBuf instead of nil because we want
1600 // a non-nil result even when the length is 0.
1601 v := append(emptyBuf[:], b[:x]...)
1606 func unmarshalBytesSlice(b []byte, f pointer, w int) ([]byte, error) {
1608 return b, errInternalBadWireType
1610 x, n := decodeVarint(b)
1612 return nil, io.ErrUnexpectedEOF
1615 if x > uint64(len(b)) {
1616 return nil, io.ErrUnexpectedEOF
1618 v := append(emptyBuf[:], b[:x]...)
1619 s := f.toBytesSlice()
1624 func makeUnmarshalMessagePtr(sub *unmarshalInfo, name string) unmarshaler {
1625 return func(b []byte, f pointer, w int) ([]byte, error) {
1627 return b, errInternalBadWireType
1629 x, n := decodeVarint(b)
1631 return nil, io.ErrUnexpectedEOF
1634 if x > uint64(len(b)) {
1635 return nil, io.ErrUnexpectedEOF
1637 // First read the message field to see if something is there.
1638 // The semantics of multiple submessages are weird. Instead of
1639 // the last one winning (as it is for all other fields), multiple
1640 // submessages are merged.
1643 v = valToPointer(reflect.New(sub.typ))
1646 err := sub.unmarshal(v, b[:x])
1648 if r, ok := err.(*RequiredNotSetError); ok {
1649 r.field = name + "." + r.field
1658 func makeUnmarshalMessageSlicePtr(sub *unmarshalInfo, name string) unmarshaler {
1659 return func(b []byte, f pointer, w int) ([]byte, error) {
1661 return b, errInternalBadWireType
1663 x, n := decodeVarint(b)
1665 return nil, io.ErrUnexpectedEOF
1668 if x > uint64(len(b)) {
1669 return nil, io.ErrUnexpectedEOF
1671 v := valToPointer(reflect.New(sub.typ))
1672 err := sub.unmarshal(v, b[:x])
1674 if r, ok := err.(*RequiredNotSetError); ok {
1675 r.field = name + "." + r.field
1685 func makeUnmarshalGroupPtr(sub *unmarshalInfo, name string) unmarshaler {
1686 return func(b []byte, f pointer, w int) ([]byte, error) {
1687 if w != WireStartGroup {
1688 return b, errInternalBadWireType
1690 x, y := findEndGroup(b)
1692 return nil, io.ErrUnexpectedEOF
1696 v = valToPointer(reflect.New(sub.typ))
1699 err := sub.unmarshal(v, b[:x])
1701 if r, ok := err.(*RequiredNotSetError); ok {
1702 r.field = name + "." + r.field
1711 func makeUnmarshalGroupSlicePtr(sub *unmarshalInfo, name string) unmarshaler {
1712 return func(b []byte, f pointer, w int) ([]byte, error) {
1713 if w != WireStartGroup {
1714 return b, errInternalBadWireType
1716 x, y := findEndGroup(b)
1718 return nil, io.ErrUnexpectedEOF
1720 v := valToPointer(reflect.New(sub.typ))
1721 err := sub.unmarshal(v, b[:x])
1723 if r, ok := err.(*RequiredNotSetError); ok {
1724 r.field = name + "." + r.field
1734 func makeUnmarshalMap(f *reflect.StructField) unmarshaler {
1738 unmarshalKey := typeUnmarshaler(kt, f.Tag.Get("protobuf_key"))
1739 unmarshalVal := typeUnmarshaler(vt, f.Tag.Get("protobuf_val"))
1740 return func(b []byte, f pointer, w int) ([]byte, error) {
1741 // The map entry is a submessage. Figure out how big it is.
1743 return nil, fmt.Errorf("proto: bad wiretype for map field: got %d want %d", w, WireBytes)
1745 x, n := decodeVarint(b)
1747 return nil, io.ErrUnexpectedEOF
1750 if x > uint64(len(b)) {
1751 return nil, io.ErrUnexpectedEOF
1753 r := b[x:] // unused data to return
1754 b = b[:x] // data for map entry
1756 // Note: we could use #keys * #values ~= 200 functions
1757 // to do map decoding without reflection. Probably not worth it.
1758 // Maps will be somewhat slow. Oh well.
1760 // Read key and value from data.
1762 k := reflect.New(kt)
1763 v := reflect.New(vt)
1765 x, n := decodeVarint(b)
1767 return nil, io.ErrUnexpectedEOF
1775 b, err = unmarshalKey(b, valToPointer(k), wire)
1777 b, err = unmarshalVal(b, valToPointer(v), wire)
1779 err = errInternalBadWireType // skip unknown tag
1782 if nerr.Merge(err) {
1785 if err != errInternalBadWireType {
1789 // Skip past unknown fields.
1790 b, err = skipField(b, wire)
1796 // Get map, allocate if needed.
1797 m := f.asPointerTo(t).Elem() // an addressable map[K]T
1799 m.Set(reflect.MakeMap(t))
1803 m.SetMapIndex(k.Elem(), v.Elem())
1809 // makeUnmarshalOneof makes an unmarshaler for oneof fields.
1817 // typ is the type of the concrete entry for a oneof case (e.g. Msg_X).
1818 // ityp is the interface type of the oneof field (e.g. isMsg_F).
1819 // unmarshal is the unmarshaler for the base type of the oneof case (e.g. int64).
1820 // Note that this function will be called once for each case in the oneof.
1821 func makeUnmarshalOneof(typ, ityp reflect.Type, unmarshal unmarshaler) unmarshaler {
1823 field0 := toField(&sf)
1824 return func(b []byte, f pointer, w int) ([]byte, error) {
1825 // Allocate holder for value.
1826 v := reflect.New(typ)
1828 // Unmarshal data into holder.
1829 // We unmarshal into the first field of the holder object.
1832 b, err = unmarshal(b, valToPointer(v).offset(field0), w)
1833 if !nerr.Merge(err) {
1837 // Write pointer to holder into target field.
1838 f.asPointerTo(ityp).Elem().Set(v)
1844 // Error used by decode internally.
1845 var errInternalBadWireType = errors.New("proto: internal error: bad wiretype")
1847 // skipField skips past a field of type wire and returns the remaining bytes.
1848 func skipField(b []byte, wire int) ([]byte, error) {
1851 _, k := decodeVarint(b)
1853 return b, io.ErrUnexpectedEOF
1858 return b, io.ErrUnexpectedEOF
1863 return b, io.ErrUnexpectedEOF
1867 m, k := decodeVarint(b)
1868 if k == 0 || uint64(len(b)-k) < m {
1869 return b, io.ErrUnexpectedEOF
1872 case WireStartGroup:
1873 _, i := findEndGroup(b)
1875 return b, io.ErrUnexpectedEOF
1879 return b, fmt.Errorf("proto: can't skip unknown wire type %d", wire)
1884 // findEndGroup finds the index of the next EndGroup tag.
1885 // Groups may be nested, so the "next" EndGroup tag is the first
1886 // unpaired EndGroup.
1887 // findEndGroup returns the indexes of the start and end of the EndGroup tag.
1888 // Returns (-1,-1) if it can't find one.
1889 func findEndGroup(b []byte) (int, int) {
1893 x, n := decodeVarint(b[i:])
1901 _, k := decodeVarint(b[i:])
1917 m, k := decodeVarint(b[i:])
1922 if uint64(len(b)-i) < m {
1926 case WireStartGroup:
1939 // encodeVarint appends a varint-encoded integer to b and returns the result.
1940 func encodeVarint(b []byte, x uint64) []byte {
1942 b = append(b, byte(x&0x7f|0x80))
1945 return append(b, byte(x))
1948 // decodeVarint reads a varint-encoded integer from b.
1949 // Returns the decoded integer and the number of bytes read.
1950 // If there is an error, it returns 0,0.
1951 func decodeVarint(b []byte) (uint64, int) {