wire.go 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538
  1. // Copyright 2018 The Go Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style
  3. // license that can be found in the LICENSE file.
  4. // Package protowire parses and formats the raw wire encoding.
  5. // See https://developers.google.com/protocol-buffers/docs/encoding.
  6. //
  7. // For marshaling and unmarshaling entire protobuf messages,
  8. // use the "google.golang.org/protobuf/proto" package instead.
  9. package protowire
  10. import (
  11. "io"
  12. "math"
  13. "math/bits"
  14. "google.golang.org/protobuf/internal/errors"
  15. )
  16. // Number represents the field number.
  17. type Number int32
  18. const (
  19. MinValidNumber Number = 1
  20. FirstReservedNumber Number = 19000
  21. LastReservedNumber Number = 19999
  22. MaxValidNumber Number = 1<<29 - 1
  23. )
  24. // IsValid reports whether the field number is semantically valid.
  25. //
  26. // Note that while numbers within the reserved range are semantically invalid,
  27. // they are syntactically valid in the wire format.
  28. // Implementations may treat records with reserved field numbers as unknown.
  29. func (n Number) IsValid() bool {
  30. return MinValidNumber <= n && n < FirstReservedNumber || LastReservedNumber < n && n <= MaxValidNumber
  31. }
  32. // Type represents the wire type.
  33. type Type int8
  34. const (
  35. VarintType Type = 0
  36. Fixed32Type Type = 5
  37. Fixed64Type Type = 1
  38. BytesType Type = 2
  39. StartGroupType Type = 3
  40. EndGroupType Type = 4
  41. )
  42. const (
  43. _ = -iota
  44. errCodeTruncated
  45. errCodeFieldNumber
  46. errCodeOverflow
  47. errCodeReserved
  48. errCodeEndGroup
  49. )
  50. var (
  51. errFieldNumber = errors.New("invalid field number")
  52. errOverflow = errors.New("variable length integer overflow")
  53. errReserved = errors.New("cannot parse reserved wire type")
  54. errEndGroup = errors.New("mismatching end group marker")
  55. errParse = errors.New("parse error")
  56. )
  57. // ParseError converts an error code into an error value.
  58. // This returns nil if n is a non-negative number.
  59. func ParseError(n int) error {
  60. if n >= 0 {
  61. return nil
  62. }
  63. switch n {
  64. case errCodeTruncated:
  65. return io.ErrUnexpectedEOF
  66. case errCodeFieldNumber:
  67. return errFieldNumber
  68. case errCodeOverflow:
  69. return errOverflow
  70. case errCodeReserved:
  71. return errReserved
  72. case errCodeEndGroup:
  73. return errEndGroup
  74. default:
  75. return errParse
  76. }
  77. }
  78. // ConsumeField parses an entire field record (both tag and value) and returns
  79. // the field number, the wire type, and the total length.
  80. // This returns a negative length upon an error (see ParseError).
  81. //
  82. // The total length includes the tag header and the end group marker (if the
  83. // field is a group).
  84. func ConsumeField(b []byte) (Number, Type, int) {
  85. num, typ, n := ConsumeTag(b)
  86. if n < 0 {
  87. return 0, 0, n // forward error code
  88. }
  89. m := ConsumeFieldValue(num, typ, b[n:])
  90. if m < 0 {
  91. return 0, 0, m // forward error code
  92. }
  93. return num, typ, n + m
  94. }
  95. // ConsumeFieldValue parses a field value and returns its length.
  96. // This assumes that the field Number and wire Type have already been parsed.
  97. // This returns a negative length upon an error (see ParseError).
  98. //
  99. // When parsing a group, the length includes the end group marker and
  100. // the end group is verified to match the starting field number.
  101. func ConsumeFieldValue(num Number, typ Type, b []byte) (n int) {
  102. switch typ {
  103. case VarintType:
  104. _, n = ConsumeVarint(b)
  105. return n
  106. case Fixed32Type:
  107. _, n = ConsumeFixed32(b)
  108. return n
  109. case Fixed64Type:
  110. _, n = ConsumeFixed64(b)
  111. return n
  112. case BytesType:
  113. _, n = ConsumeBytes(b)
  114. return n
  115. case StartGroupType:
  116. n0 := len(b)
  117. for {
  118. num2, typ2, n := ConsumeTag(b)
  119. if n < 0 {
  120. return n // forward error code
  121. }
  122. b = b[n:]
  123. if typ2 == EndGroupType {
  124. if num != num2 {
  125. return errCodeEndGroup
  126. }
  127. return n0 - len(b)
  128. }
  129. n = ConsumeFieldValue(num2, typ2, b)
  130. if n < 0 {
  131. return n // forward error code
  132. }
  133. b = b[n:]
  134. }
  135. case EndGroupType:
  136. return errCodeEndGroup
  137. default:
  138. return errCodeReserved
  139. }
  140. }
  141. // AppendTag encodes num and typ as a varint-encoded tag and appends it to b.
  142. func AppendTag(b []byte, num Number, typ Type) []byte {
  143. return AppendVarint(b, EncodeTag(num, typ))
  144. }
  145. // ConsumeTag parses b as a varint-encoded tag, reporting its length.
  146. // This returns a negative length upon an error (see ParseError).
  147. func ConsumeTag(b []byte) (Number, Type, int) {
  148. v, n := ConsumeVarint(b)
  149. if n < 0 {
  150. return 0, 0, n // forward error code
  151. }
  152. num, typ := DecodeTag(v)
  153. if num < MinValidNumber {
  154. return 0, 0, errCodeFieldNumber
  155. }
  156. return num, typ, n
  157. }
  158. func SizeTag(num Number) int {
  159. return SizeVarint(EncodeTag(num, 0)) // wire type has no effect on size
  160. }
  161. // AppendVarint appends v to b as a varint-encoded uint64.
  162. func AppendVarint(b []byte, v uint64) []byte {
  163. switch {
  164. case v < 1<<7:
  165. b = append(b, byte(v))
  166. case v < 1<<14:
  167. b = append(b,
  168. byte((v>>0)&0x7f|0x80),
  169. byte(v>>7))
  170. case v < 1<<21:
  171. b = append(b,
  172. byte((v>>0)&0x7f|0x80),
  173. byte((v>>7)&0x7f|0x80),
  174. byte(v>>14))
  175. case v < 1<<28:
  176. b = append(b,
  177. byte((v>>0)&0x7f|0x80),
  178. byte((v>>7)&0x7f|0x80),
  179. byte((v>>14)&0x7f|0x80),
  180. byte(v>>21))
  181. case v < 1<<35:
  182. b = append(b,
  183. byte((v>>0)&0x7f|0x80),
  184. byte((v>>7)&0x7f|0x80),
  185. byte((v>>14)&0x7f|0x80),
  186. byte((v>>21)&0x7f|0x80),
  187. byte(v>>28))
  188. case v < 1<<42:
  189. b = append(b,
  190. byte((v>>0)&0x7f|0x80),
  191. byte((v>>7)&0x7f|0x80),
  192. byte((v>>14)&0x7f|0x80),
  193. byte((v>>21)&0x7f|0x80),
  194. byte((v>>28)&0x7f|0x80),
  195. byte(v>>35))
  196. case v < 1<<49:
  197. b = append(b,
  198. byte((v>>0)&0x7f|0x80),
  199. byte((v>>7)&0x7f|0x80),
  200. byte((v>>14)&0x7f|0x80),
  201. byte((v>>21)&0x7f|0x80),
  202. byte((v>>28)&0x7f|0x80),
  203. byte((v>>35)&0x7f|0x80),
  204. byte(v>>42))
  205. case v < 1<<56:
  206. b = append(b,
  207. byte((v>>0)&0x7f|0x80),
  208. byte((v>>7)&0x7f|0x80),
  209. byte((v>>14)&0x7f|0x80),
  210. byte((v>>21)&0x7f|0x80),
  211. byte((v>>28)&0x7f|0x80),
  212. byte((v>>35)&0x7f|0x80),
  213. byte((v>>42)&0x7f|0x80),
  214. byte(v>>49))
  215. case v < 1<<63:
  216. b = append(b,
  217. byte((v>>0)&0x7f|0x80),
  218. byte((v>>7)&0x7f|0x80),
  219. byte((v>>14)&0x7f|0x80),
  220. byte((v>>21)&0x7f|0x80),
  221. byte((v>>28)&0x7f|0x80),
  222. byte((v>>35)&0x7f|0x80),
  223. byte((v>>42)&0x7f|0x80),
  224. byte((v>>49)&0x7f|0x80),
  225. byte(v>>56))
  226. default:
  227. b = append(b,
  228. byte((v>>0)&0x7f|0x80),
  229. byte((v>>7)&0x7f|0x80),
  230. byte((v>>14)&0x7f|0x80),
  231. byte((v>>21)&0x7f|0x80),
  232. byte((v>>28)&0x7f|0x80),
  233. byte((v>>35)&0x7f|0x80),
  234. byte((v>>42)&0x7f|0x80),
  235. byte((v>>49)&0x7f|0x80),
  236. byte((v>>56)&0x7f|0x80),
  237. 1)
  238. }
  239. return b
  240. }
  241. // ConsumeVarint parses b as a varint-encoded uint64, reporting its length.
  242. // This returns a negative length upon an error (see ParseError).
  243. func ConsumeVarint(b []byte) (v uint64, n int) {
  244. var y uint64
  245. if len(b) <= 0 {
  246. return 0, errCodeTruncated
  247. }
  248. v = uint64(b[0])
  249. if v < 0x80 {
  250. return v, 1
  251. }
  252. v -= 0x80
  253. if len(b) <= 1 {
  254. return 0, errCodeTruncated
  255. }
  256. y = uint64(b[1])
  257. v += y << 7
  258. if y < 0x80 {
  259. return v, 2
  260. }
  261. v -= 0x80 << 7
  262. if len(b) <= 2 {
  263. return 0, errCodeTruncated
  264. }
  265. y = uint64(b[2])
  266. v += y << 14
  267. if y < 0x80 {
  268. return v, 3
  269. }
  270. v -= 0x80 << 14
  271. if len(b) <= 3 {
  272. return 0, errCodeTruncated
  273. }
  274. y = uint64(b[3])
  275. v += y << 21
  276. if y < 0x80 {
  277. return v, 4
  278. }
  279. v -= 0x80 << 21
  280. if len(b) <= 4 {
  281. return 0, errCodeTruncated
  282. }
  283. y = uint64(b[4])
  284. v += y << 28
  285. if y < 0x80 {
  286. return v, 5
  287. }
  288. v -= 0x80 << 28
  289. if len(b) <= 5 {
  290. return 0, errCodeTruncated
  291. }
  292. y = uint64(b[5])
  293. v += y << 35
  294. if y < 0x80 {
  295. return v, 6
  296. }
  297. v -= 0x80 << 35
  298. if len(b) <= 6 {
  299. return 0, errCodeTruncated
  300. }
  301. y = uint64(b[6])
  302. v += y << 42
  303. if y < 0x80 {
  304. return v, 7
  305. }
  306. v -= 0x80 << 42
  307. if len(b) <= 7 {
  308. return 0, errCodeTruncated
  309. }
  310. y = uint64(b[7])
  311. v += y << 49
  312. if y < 0x80 {
  313. return v, 8
  314. }
  315. v -= 0x80 << 49
  316. if len(b) <= 8 {
  317. return 0, errCodeTruncated
  318. }
  319. y = uint64(b[8])
  320. v += y << 56
  321. if y < 0x80 {
  322. return v, 9
  323. }
  324. v -= 0x80 << 56
  325. if len(b) <= 9 {
  326. return 0, errCodeTruncated
  327. }
  328. y = uint64(b[9])
  329. v += y << 63
  330. if y < 2 {
  331. return v, 10
  332. }
  333. return 0, errCodeOverflow
  334. }
  335. // SizeVarint returns the encoded size of a varint.
  336. // The size is guaranteed to be within 1 and 10, inclusive.
  337. func SizeVarint(v uint64) int {
  338. // This computes 1 + (bits.Len64(v)-1)/7.
  339. // 9/64 is a good enough approximation of 1/7
  340. return int(9*uint32(bits.Len64(v))+64) / 64
  341. }
  342. // AppendFixed32 appends v to b as a little-endian uint32.
  343. func AppendFixed32(b []byte, v uint32) []byte {
  344. return append(b,
  345. byte(v>>0),
  346. byte(v>>8),
  347. byte(v>>16),
  348. byte(v>>24))
  349. }
  350. // ConsumeFixed32 parses b as a little-endian uint32, reporting its length.
  351. // This returns a negative length upon an error (see ParseError).
  352. func ConsumeFixed32(b []byte) (v uint32, n int) {
  353. if len(b) < 4 {
  354. return 0, errCodeTruncated
  355. }
  356. v = uint32(b[0])<<0 | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24
  357. return v, 4
  358. }
  359. // SizeFixed32 returns the encoded size of a fixed32; which is always 4.
  360. func SizeFixed32() int {
  361. return 4
  362. }
  363. // AppendFixed64 appends v to b as a little-endian uint64.
  364. func AppendFixed64(b []byte, v uint64) []byte {
  365. return append(b,
  366. byte(v>>0),
  367. byte(v>>8),
  368. byte(v>>16),
  369. byte(v>>24),
  370. byte(v>>32),
  371. byte(v>>40),
  372. byte(v>>48),
  373. byte(v>>56))
  374. }
  375. // ConsumeFixed64 parses b as a little-endian uint64, reporting its length.
  376. // This returns a negative length upon an error (see ParseError).
  377. func ConsumeFixed64(b []byte) (v uint64, n int) {
  378. if len(b) < 8 {
  379. return 0, errCodeTruncated
  380. }
  381. v = uint64(b[0])<<0 | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56
  382. return v, 8
  383. }
  384. // SizeFixed64 returns the encoded size of a fixed64; which is always 8.
  385. func SizeFixed64() int {
  386. return 8
  387. }
  388. // AppendBytes appends v to b as a length-prefixed bytes value.
  389. func AppendBytes(b []byte, v []byte) []byte {
  390. return append(AppendVarint(b, uint64(len(v))), v...)
  391. }
  392. // ConsumeBytes parses b as a length-prefixed bytes value, reporting its length.
  393. // This returns a negative length upon an error (see ParseError).
  394. func ConsumeBytes(b []byte) (v []byte, n int) {
  395. m, n := ConsumeVarint(b)
  396. if n < 0 {
  397. return nil, n // forward error code
  398. }
  399. if m > uint64(len(b[n:])) {
  400. return nil, errCodeTruncated
  401. }
  402. return b[n:][:m], n + int(m)
  403. }
  404. // SizeBytes returns the encoded size of a length-prefixed bytes value,
  405. // given only the length.
  406. func SizeBytes(n int) int {
  407. return SizeVarint(uint64(n)) + n
  408. }
  409. // AppendString appends v to b as a length-prefixed bytes value.
  410. func AppendString(b []byte, v string) []byte {
  411. return append(AppendVarint(b, uint64(len(v))), v...)
  412. }
  413. // ConsumeString parses b as a length-prefixed bytes value, reporting its length.
  414. // This returns a negative length upon an error (see ParseError).
  415. func ConsumeString(b []byte) (v string, n int) {
  416. bb, n := ConsumeBytes(b)
  417. return string(bb), n
  418. }
  419. // AppendGroup appends v to b as group value, with a trailing end group marker.
  420. // The value v must not contain the end marker.
  421. func AppendGroup(b []byte, num Number, v []byte) []byte {
  422. return AppendVarint(append(b, v...), EncodeTag(num, EndGroupType))
  423. }
  424. // ConsumeGroup parses b as a group value until the trailing end group marker,
  425. // and verifies that the end marker matches the provided num. The value v
  426. // does not contain the end marker, while the length does contain the end marker.
  427. // This returns a negative length upon an error (see ParseError).
  428. func ConsumeGroup(num Number, b []byte) (v []byte, n int) {
  429. n = ConsumeFieldValue(num, StartGroupType, b)
  430. if n < 0 {
  431. return nil, n // forward error code
  432. }
  433. b = b[:n]
  434. // Truncate off end group marker, but need to handle denormalized varints.
  435. // Assuming end marker is never 0 (which is always the case since
  436. // EndGroupType is non-zero), we can truncate all trailing bytes where the
  437. // lower 7 bits are all zero (implying that the varint is denormalized).
  438. for len(b) > 0 && b[len(b)-1]&0x7f == 0 {
  439. b = b[:len(b)-1]
  440. }
  441. b = b[:len(b)-SizeTag(num)]
  442. return b, n
  443. }
  444. // SizeGroup returns the encoded size of a group, given only the length.
  445. func SizeGroup(num Number, n int) int {
  446. return n + SizeTag(num)
  447. }
  448. // DecodeTag decodes the field Number and wire Type from its unified form.
  449. // The Number is -1 if the decoded field number overflows int32.
  450. // Other than overflow, this does not check for field number validity.
  451. func DecodeTag(x uint64) (Number, Type) {
  452. // NOTE: MessageSet allows for larger field numbers than normal.
  453. if x>>3 > uint64(math.MaxInt32) {
  454. return -1, 0
  455. }
  456. return Number(x >> 3), Type(x & 7)
  457. }
  458. // EncodeTag encodes the field Number and wire Type into its unified form.
  459. func EncodeTag(num Number, typ Type) uint64 {
  460. return uint64(num)<<3 | uint64(typ&7)
  461. }
  462. // DecodeZigZag decodes a zig-zag-encoded uint64 as an int64.
  463. // Input: {…, 5, 3, 1, 0, 2, 4, 6, …}
  464. // Output: {…, -3, -2, -1, 0, +1, +2, +3, …}
  465. func DecodeZigZag(x uint64) int64 {
  466. return int64(x>>1) ^ int64(x)<<63>>63
  467. }
  468. // EncodeZigZag encodes an int64 as a zig-zag-encoded uint64.
  469. // Input: {…, -3, -2, -1, 0, +1, +2, +3, …}
  470. // Output: {…, 5, 3, 1, 0, 2, 4, 6, …}
  471. func EncodeZigZag(x int64) uint64 {
  472. return uint64(x<<1) ^ uint64(x>>63)
  473. }
  474. // DecodeBool decodes a uint64 as a bool.
  475. // Input: { 0, 1, 2, …}
  476. // Output: {false, true, true, …}
  477. func DecodeBool(x uint64) bool {
  478. return x != 0
  479. }
  480. // EncodeBool encodes a bool as a uint64.
  481. // Input: {false, true}
  482. // Output: { 0, 1}
  483. func EncodeBool(x bool) uint64 {
  484. if x {
  485. return 1
  486. }
  487. return 0
  488. }