encode.go 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346
  1. // Copyright 2019 The Go Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style
  3. // license that can be found in the LICENSE file.
  4. package proto
  5. import (
  6. "sort"
  7. "google.golang.org/protobuf/encoding/protowire"
  8. "google.golang.org/protobuf/internal/encoding/messageset"
  9. "google.golang.org/protobuf/internal/fieldsort"
  10. "google.golang.org/protobuf/internal/mapsort"
  11. "google.golang.org/protobuf/internal/pragma"
  12. "google.golang.org/protobuf/reflect/protoreflect"
  13. "google.golang.org/protobuf/runtime/protoiface"
  14. )
  15. // MarshalOptions configures the marshaler.
  16. //
  17. // Example usage:
  18. // b, err := MarshalOptions{Deterministic: true}.Marshal(m)
  19. type MarshalOptions struct {
  20. pragma.NoUnkeyedLiterals
  21. // AllowPartial allows messages that have missing required fields to marshal
  22. // without returning an error. If AllowPartial is false (the default),
  23. // Marshal will return an error if there are any missing required fields.
  24. AllowPartial bool
  25. // Deterministic controls whether the same message will always be
  26. // serialized to the same bytes within the same binary.
  27. //
  28. // Setting this option guarantees that repeated serialization of
  29. // the same message will return the same bytes, and that different
  30. // processes of the same binary (which may be executing on different
  31. // machines) will serialize equal messages to the same bytes.
  32. // It has no effect on the resulting size of the encoded message compared
  33. // to a non-deterministic marshal.
  34. //
  35. // Note that the deterministic serialization is NOT canonical across
  36. // languages. It is not guaranteed to remain stable over time. It is
  37. // unstable across different builds with schema changes due to unknown
  38. // fields. Users who need canonical serialization (e.g., persistent
  39. // storage in a canonical form, fingerprinting, etc.) must define
  40. // their own canonicalization specification and implement their own
  41. // serializer rather than relying on this API.
  42. //
  43. // If deterministic serialization is requested, map entries will be
  44. // sorted by keys in lexographical order. This is an implementation
  45. // detail and subject to change.
  46. Deterministic bool
  47. // UseCachedSize indicates that the result of a previous Size call
  48. // may be reused.
  49. //
  50. // Setting this option asserts that:
  51. //
  52. // 1. Size has previously been called on this message with identical
  53. // options (except for UseCachedSize itself).
  54. //
  55. // 2. The message and all its submessages have not changed in any
  56. // way since the Size call.
  57. //
  58. // If either of these invariants is violated,
  59. // the results are undefined and may include panics or corrupted output.
  60. //
  61. // Implementations MAY take this option into account to provide
  62. // better performance, but there is no guarantee that they will do so.
  63. // There is absolutely no guarantee that Size followed by Marshal with
  64. // UseCachedSize set will perform equivalently to Marshal alone.
  65. UseCachedSize bool
  66. }
  67. // Marshal returns the wire-format encoding of m.
  68. func Marshal(m Message) ([]byte, error) {
  69. // Treat nil message interface as an empty message; nothing to output.
  70. if m == nil {
  71. return nil, nil
  72. }
  73. out, err := MarshalOptions{}.marshal(nil, m.ProtoReflect())
  74. if len(out.Buf) == 0 && err == nil {
  75. out.Buf = emptyBytesForMessage(m)
  76. }
  77. return out.Buf, err
  78. }
  79. // Marshal returns the wire-format encoding of m.
  80. func (o MarshalOptions) Marshal(m Message) ([]byte, error) {
  81. // Treat nil message interface as an empty message; nothing to output.
  82. if m == nil {
  83. return nil, nil
  84. }
  85. out, err := o.marshal(nil, m.ProtoReflect())
  86. if len(out.Buf) == 0 && err == nil {
  87. out.Buf = emptyBytesForMessage(m)
  88. }
  89. return out.Buf, err
  90. }
  91. // emptyBytesForMessage returns a nil buffer if and only if m is invalid,
  92. // otherwise it returns a non-nil empty buffer.
  93. //
  94. // This is to assist the edge-case where user-code does the following:
  95. // m1.OptionalBytes, _ = proto.Marshal(m2)
  96. // where they expect the proto2 "optional_bytes" field to be populated
  97. // if any only if m2 is a valid message.
  98. func emptyBytesForMessage(m Message) []byte {
  99. if m == nil || !m.ProtoReflect().IsValid() {
  100. return nil
  101. }
  102. return emptyBuf[:]
  103. }
  104. // MarshalAppend appends the wire-format encoding of m to b,
  105. // returning the result.
  106. func (o MarshalOptions) MarshalAppend(b []byte, m Message) ([]byte, error) {
  107. // Treat nil message interface as an empty message; nothing to append.
  108. if m == nil {
  109. return b, nil
  110. }
  111. out, err := o.marshal(b, m.ProtoReflect())
  112. return out.Buf, err
  113. }
  114. // MarshalState returns the wire-format encoding of a message.
  115. //
  116. // This method permits fine-grained control over the marshaler.
  117. // Most users should use Marshal instead.
  118. func (o MarshalOptions) MarshalState(in protoiface.MarshalInput) (protoiface.MarshalOutput, error) {
  119. return o.marshal(in.Buf, in.Message)
  120. }
  121. // marshal is a centralized function that all marshal operations go through.
  122. // For profiling purposes, avoid changing the name of this function or
  123. // introducing other code paths for marshal that do not go through this.
  124. func (o MarshalOptions) marshal(b []byte, m protoreflect.Message) (out protoiface.MarshalOutput, err error) {
  125. allowPartial := o.AllowPartial
  126. o.AllowPartial = true
  127. if methods := protoMethods(m); methods != nil && methods.Marshal != nil &&
  128. !(o.Deterministic && methods.Flags&protoiface.SupportMarshalDeterministic == 0) {
  129. in := protoiface.MarshalInput{
  130. Message: m,
  131. Buf: b,
  132. }
  133. if o.Deterministic {
  134. in.Flags |= protoiface.MarshalDeterministic
  135. }
  136. if o.UseCachedSize {
  137. in.Flags |= protoiface.MarshalUseCachedSize
  138. }
  139. if methods.Size != nil {
  140. sout := methods.Size(protoiface.SizeInput{
  141. Message: m,
  142. Flags: in.Flags,
  143. })
  144. if cap(b) < len(b)+sout.Size {
  145. in.Buf = make([]byte, len(b), growcap(cap(b), len(b)+sout.Size))
  146. copy(in.Buf, b)
  147. }
  148. in.Flags |= protoiface.MarshalUseCachedSize
  149. }
  150. out, err = methods.Marshal(in)
  151. } else {
  152. out.Buf, err = o.marshalMessageSlow(b, m)
  153. }
  154. if err != nil {
  155. return out, err
  156. }
  157. if allowPartial {
  158. return out, nil
  159. }
  160. return out, checkInitialized(m)
  161. }
  162. func (o MarshalOptions) marshalMessage(b []byte, m protoreflect.Message) ([]byte, error) {
  163. out, err := o.marshal(b, m)
  164. return out.Buf, err
  165. }
  166. // growcap scales up the capacity of a slice.
  167. //
  168. // Given a slice with a current capacity of oldcap and a desired
  169. // capacity of wantcap, growcap returns a new capacity >= wantcap.
  170. //
  171. // The algorithm is mostly identical to the one used by append as of Go 1.14.
  172. func growcap(oldcap, wantcap int) (newcap int) {
  173. if wantcap > oldcap*2 {
  174. newcap = wantcap
  175. } else if oldcap < 1024 {
  176. // The Go 1.14 runtime takes this case when len(s) < 1024,
  177. // not when cap(s) < 1024. The difference doesn't seem
  178. // significant here.
  179. newcap = oldcap * 2
  180. } else {
  181. newcap = oldcap
  182. for 0 < newcap && newcap < wantcap {
  183. newcap += newcap / 4
  184. }
  185. if newcap <= 0 {
  186. newcap = wantcap
  187. }
  188. }
  189. return newcap
  190. }
  191. func (o MarshalOptions) marshalMessageSlow(b []byte, m protoreflect.Message) ([]byte, error) {
  192. if messageset.IsMessageSet(m.Descriptor()) {
  193. return o.marshalMessageSet(b, m)
  194. }
  195. // There are many choices for what order we visit fields in. The default one here
  196. // is chosen for reasonable efficiency and simplicity given the protoreflect API.
  197. // It is not deterministic, since Message.Range does not return fields in any
  198. // defined order.
  199. //
  200. // When using deterministic serialization, we sort the known fields.
  201. var err error
  202. o.rangeFields(m, func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool {
  203. b, err = o.marshalField(b, fd, v)
  204. return err == nil
  205. })
  206. if err != nil {
  207. return b, err
  208. }
  209. b = append(b, m.GetUnknown()...)
  210. return b, nil
  211. }
  212. // rangeFields visits fields in a defined order when deterministic serialization is enabled.
  213. func (o MarshalOptions) rangeFields(m protoreflect.Message, f func(protoreflect.FieldDescriptor, protoreflect.Value) bool) {
  214. if !o.Deterministic {
  215. m.Range(f)
  216. return
  217. }
  218. var fds []protoreflect.FieldDescriptor
  219. m.Range(func(fd protoreflect.FieldDescriptor, _ protoreflect.Value) bool {
  220. fds = append(fds, fd)
  221. return true
  222. })
  223. sort.Slice(fds, func(a, b int) bool {
  224. return fieldsort.Less(fds[a], fds[b])
  225. })
  226. for _, fd := range fds {
  227. if !f(fd, m.Get(fd)) {
  228. break
  229. }
  230. }
  231. }
  232. func (o MarshalOptions) marshalField(b []byte, fd protoreflect.FieldDescriptor, value protoreflect.Value) ([]byte, error) {
  233. switch {
  234. case fd.IsList():
  235. return o.marshalList(b, fd, value.List())
  236. case fd.IsMap():
  237. return o.marshalMap(b, fd, value.Map())
  238. default:
  239. b = protowire.AppendTag(b, fd.Number(), wireTypes[fd.Kind()])
  240. return o.marshalSingular(b, fd, value)
  241. }
  242. }
  243. func (o MarshalOptions) marshalList(b []byte, fd protoreflect.FieldDescriptor, list protoreflect.List) ([]byte, error) {
  244. if fd.IsPacked() && list.Len() > 0 {
  245. b = protowire.AppendTag(b, fd.Number(), protowire.BytesType)
  246. b, pos := appendSpeculativeLength(b)
  247. for i, llen := 0, list.Len(); i < llen; i++ {
  248. var err error
  249. b, err = o.marshalSingular(b, fd, list.Get(i))
  250. if err != nil {
  251. return b, err
  252. }
  253. }
  254. b = finishSpeculativeLength(b, pos)
  255. return b, nil
  256. }
  257. kind := fd.Kind()
  258. for i, llen := 0, list.Len(); i < llen; i++ {
  259. var err error
  260. b = protowire.AppendTag(b, fd.Number(), wireTypes[kind])
  261. b, err = o.marshalSingular(b, fd, list.Get(i))
  262. if err != nil {
  263. return b, err
  264. }
  265. }
  266. return b, nil
  267. }
  268. func (o MarshalOptions) marshalMap(b []byte, fd protoreflect.FieldDescriptor, mapv protoreflect.Map) ([]byte, error) {
  269. keyf := fd.MapKey()
  270. valf := fd.MapValue()
  271. var err error
  272. o.rangeMap(mapv, keyf.Kind(), func(key protoreflect.MapKey, value protoreflect.Value) bool {
  273. b = protowire.AppendTag(b, fd.Number(), protowire.BytesType)
  274. var pos int
  275. b, pos = appendSpeculativeLength(b)
  276. b, err = o.marshalField(b, keyf, key.Value())
  277. if err != nil {
  278. return false
  279. }
  280. b, err = o.marshalField(b, valf, value)
  281. if err != nil {
  282. return false
  283. }
  284. b = finishSpeculativeLength(b, pos)
  285. return true
  286. })
  287. return b, err
  288. }
  289. func (o MarshalOptions) rangeMap(mapv protoreflect.Map, kind protoreflect.Kind, f func(protoreflect.MapKey, protoreflect.Value) bool) {
  290. if !o.Deterministic {
  291. mapv.Range(f)
  292. return
  293. }
  294. mapsort.Range(mapv, kind, f)
  295. }
  296. // When encoding length-prefixed fields, we speculatively set aside some number of bytes
  297. // for the length, encode the data, and then encode the length (shifting the data if necessary
  298. // to make room).
  299. const speculativeLength = 1
  300. func appendSpeculativeLength(b []byte) ([]byte, int) {
  301. pos := len(b)
  302. b = append(b, "\x00\x00\x00\x00"[:speculativeLength]...)
  303. return b, pos
  304. }
  305. func finishSpeculativeLength(b []byte, pos int) []byte {
  306. mlen := len(b) - pos - speculativeLength
  307. msiz := protowire.SizeVarint(uint64(mlen))
  308. if msiz != speculativeLength {
  309. for i := 0; i < msiz-speculativeLength; i++ {
  310. b = append(b, 0)
  311. }
  312. copy(b[pos+msiz:], b[pos+speculativeLength:])
  313. b = b[:pos+msiz+mlen]
  314. }
  315. protowire.AppendVarint(b[:pos], uint64(mlen))
  316. return b
  317. }