yaml.go 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380
  1. package yaml
  2. import (
  3. "bytes"
  4. "encoding/json"
  5. "fmt"
  6. "io"
  7. "reflect"
  8. "strconv"
  9. "gopkg.in/yaml.v2"
  10. )
  11. // Marshal marshals the object into JSON then converts JSON to YAML and returns the
  12. // YAML.
  13. func Marshal(o interface{}) ([]byte, error) {
  14. j, err := json.Marshal(o)
  15. if err != nil {
  16. return nil, fmt.Errorf("error marshaling into JSON: %v", err)
  17. }
  18. y, err := JSONToYAML(j)
  19. if err != nil {
  20. return nil, fmt.Errorf("error converting JSON to YAML: %v", err)
  21. }
  22. return y, nil
  23. }
  24. // JSONOpt is a decoding option for decoding from JSON format.
  25. type JSONOpt func(*json.Decoder) *json.Decoder
  26. // Unmarshal converts YAML to JSON then uses JSON to unmarshal into an object,
  27. // optionally configuring the behavior of the JSON unmarshal.
  28. func Unmarshal(y []byte, o interface{}, opts ...JSONOpt) error {
  29. return yamlUnmarshal(y, o, false, opts...)
  30. }
  31. // UnmarshalStrict strictly converts YAML to JSON then uses JSON to unmarshal
  32. // into an object, optionally configuring the behavior of the JSON unmarshal.
  33. func UnmarshalStrict(y []byte, o interface{}, opts ...JSONOpt) error {
  34. return yamlUnmarshal(y, o, true, append(opts, DisallowUnknownFields)...)
  35. }
  36. // yamlUnmarshal unmarshals the given YAML byte stream into the given interface,
  37. // optionally performing the unmarshalling strictly
  38. func yamlUnmarshal(y []byte, o interface{}, strict bool, opts ...JSONOpt) error {
  39. vo := reflect.ValueOf(o)
  40. unmarshalFn := yaml.Unmarshal
  41. if strict {
  42. unmarshalFn = yaml.UnmarshalStrict
  43. }
  44. j, err := yamlToJSON(y, &vo, unmarshalFn)
  45. if err != nil {
  46. return fmt.Errorf("error converting YAML to JSON: %v", err)
  47. }
  48. err = jsonUnmarshal(bytes.NewReader(j), o, opts...)
  49. if err != nil {
  50. return fmt.Errorf("error unmarshaling JSON: %v", err)
  51. }
  52. return nil
  53. }
  54. // jsonUnmarshal unmarshals the JSON byte stream from the given reader into the
  55. // object, optionally applying decoder options prior to decoding. We are not
  56. // using json.Unmarshal directly as we want the chance to pass in non-default
  57. // options.
  58. func jsonUnmarshal(r io.Reader, o interface{}, opts ...JSONOpt) error {
  59. d := json.NewDecoder(r)
  60. for _, opt := range opts {
  61. d = opt(d)
  62. }
  63. if err := d.Decode(&o); err != nil {
  64. return fmt.Errorf("while decoding JSON: %v", err)
  65. }
  66. return nil
  67. }
  68. // JSONToYAML Converts JSON to YAML.
  69. func JSONToYAML(j []byte) ([]byte, error) {
  70. // Convert the JSON to an object.
  71. var jsonObj interface{}
  72. // We are using yaml.Unmarshal here (instead of json.Unmarshal) because the
  73. // Go JSON library doesn't try to pick the right number type (int, float,
  74. // etc.) when unmarshalling to interface{}, it just picks float64
  75. // universally. go-yaml does go through the effort of picking the right
  76. // number type, so we can preserve number type throughout this process.
  77. err := yaml.Unmarshal(j, &jsonObj)
  78. if err != nil {
  79. return nil, err
  80. }
  81. // Marshal this object into YAML.
  82. return yaml.Marshal(jsonObj)
  83. }
  84. // YAMLToJSON converts YAML to JSON. Since JSON is a subset of YAML,
  85. // passing JSON through this method should be a no-op.
  86. //
  87. // Things YAML can do that are not supported by JSON:
  88. // * In YAML you can have binary and null keys in your maps. These are invalid
  89. // in JSON. (int and float keys are converted to strings.)
  90. // * Binary data in YAML with the !!binary tag is not supported. If you want to
  91. // use binary data with this library, encode the data as base64 as usual but do
  92. // not use the !!binary tag in your YAML. This will ensure the original base64
  93. // encoded data makes it all the way through to the JSON.
  94. //
  95. // For strict decoding of YAML, use YAMLToJSONStrict.
  96. func YAMLToJSON(y []byte) ([]byte, error) {
  97. return yamlToJSON(y, nil, yaml.Unmarshal)
  98. }
  99. // YAMLToJSONStrict is like YAMLToJSON but enables strict YAML decoding,
  100. // returning an error on any duplicate field names.
  101. func YAMLToJSONStrict(y []byte) ([]byte, error) {
  102. return yamlToJSON(y, nil, yaml.UnmarshalStrict)
  103. }
  104. func yamlToJSON(y []byte, jsonTarget *reflect.Value, yamlUnmarshal func([]byte, interface{}) error) ([]byte, error) {
  105. // Convert the YAML to an object.
  106. var yamlObj interface{}
  107. err := yamlUnmarshal(y, &yamlObj)
  108. if err != nil {
  109. return nil, err
  110. }
  111. // YAML objects are not completely compatible with JSON objects (e.g. you
  112. // can have non-string keys in YAML). So, convert the YAML-compatible object
  113. // to a JSON-compatible object, failing with an error if irrecoverable
  114. // incompatibilties happen along the way.
  115. jsonObj, err := convertToJSONableObject(yamlObj, jsonTarget)
  116. if err != nil {
  117. return nil, err
  118. }
  119. // Convert this object to JSON and return the data.
  120. return json.Marshal(jsonObj)
  121. }
  122. func convertToJSONableObject(yamlObj interface{}, jsonTarget *reflect.Value) (interface{}, error) {
  123. var err error
  124. // Resolve jsonTarget to a concrete value (i.e. not a pointer or an
  125. // interface). We pass decodingNull as false because we're not actually
  126. // decoding into the value, we're just checking if the ultimate target is a
  127. // string.
  128. if jsonTarget != nil {
  129. ju, tu, pv := indirect(*jsonTarget, false)
  130. // We have a JSON or Text Umarshaler at this level, so we can't be trying
  131. // to decode into a string.
  132. if ju != nil || tu != nil {
  133. jsonTarget = nil
  134. } else {
  135. jsonTarget = &pv
  136. }
  137. }
  138. // If yamlObj is a number or a boolean, check if jsonTarget is a string -
  139. // if so, coerce. Else return normal.
  140. // If yamlObj is a map or array, find the field that each key is
  141. // unmarshaling to, and when you recurse pass the reflect.Value for that
  142. // field back into this function.
  143. switch typedYAMLObj := yamlObj.(type) {
  144. case map[interface{}]interface{}:
  145. // JSON does not support arbitrary keys in a map, so we must convert
  146. // these keys to strings.
  147. //
  148. // From my reading of go-yaml v2 (specifically the resolve function),
  149. // keys can only have the types string, int, int64, float64, binary
  150. // (unsupported), or null (unsupported).
  151. strMap := make(map[string]interface{})
  152. for k, v := range typedYAMLObj {
  153. // Resolve the key to a string first.
  154. var keyString string
  155. switch typedKey := k.(type) {
  156. case string:
  157. keyString = typedKey
  158. case int:
  159. keyString = strconv.Itoa(typedKey)
  160. case int64:
  161. // go-yaml will only return an int64 as a key if the system
  162. // architecture is 32-bit and the key's value is between 32-bit
  163. // and 64-bit. Otherwise the key type will simply be int.
  164. keyString = strconv.FormatInt(typedKey, 10)
  165. case float64:
  166. // Stolen from go-yaml to use the same conversion to string as
  167. // the go-yaml library uses to convert float to string when
  168. // Marshaling.
  169. s := strconv.FormatFloat(typedKey, 'g', -1, 32)
  170. switch s {
  171. case "+Inf":
  172. s = ".inf"
  173. case "-Inf":
  174. s = "-.inf"
  175. case "NaN":
  176. s = ".nan"
  177. }
  178. keyString = s
  179. case bool:
  180. if typedKey {
  181. keyString = "true"
  182. } else {
  183. keyString = "false"
  184. }
  185. default:
  186. return nil, fmt.Errorf("Unsupported map key of type: %s, key: %+#v, value: %+#v",
  187. reflect.TypeOf(k), k, v)
  188. }
  189. // jsonTarget should be a struct or a map. If it's a struct, find
  190. // the field it's going to map to and pass its reflect.Value. If
  191. // it's a map, find the element type of the map and pass the
  192. // reflect.Value created from that type. If it's neither, just pass
  193. // nil - JSON conversion will error for us if it's a real issue.
  194. if jsonTarget != nil {
  195. t := *jsonTarget
  196. if t.Kind() == reflect.Struct {
  197. keyBytes := []byte(keyString)
  198. // Find the field that the JSON library would use.
  199. var f *field
  200. fields := cachedTypeFields(t.Type())
  201. for i := range fields {
  202. ff := &fields[i]
  203. if bytes.Equal(ff.nameBytes, keyBytes) {
  204. f = ff
  205. break
  206. }
  207. // Do case-insensitive comparison.
  208. if f == nil && ff.equalFold(ff.nameBytes, keyBytes) {
  209. f = ff
  210. }
  211. }
  212. if f != nil {
  213. // Find the reflect.Value of the most preferential
  214. // struct field.
  215. jtf := t.Field(f.index[0])
  216. strMap[keyString], err = convertToJSONableObject(v, &jtf)
  217. if err != nil {
  218. return nil, err
  219. }
  220. continue
  221. }
  222. } else if t.Kind() == reflect.Map {
  223. // Create a zero value of the map's element type to use as
  224. // the JSON target.
  225. jtv := reflect.Zero(t.Type().Elem())
  226. strMap[keyString], err = convertToJSONableObject(v, &jtv)
  227. if err != nil {
  228. return nil, err
  229. }
  230. continue
  231. }
  232. }
  233. strMap[keyString], err = convertToJSONableObject(v, nil)
  234. if err != nil {
  235. return nil, err
  236. }
  237. }
  238. return strMap, nil
  239. case []interface{}:
  240. // We need to recurse into arrays in case there are any
  241. // map[interface{}]interface{}'s inside and to convert any
  242. // numbers to strings.
  243. // If jsonTarget is a slice (which it really should be), find the
  244. // thing it's going to map to. If it's not a slice, just pass nil
  245. // - JSON conversion will error for us if it's a real issue.
  246. var jsonSliceElemValue *reflect.Value
  247. if jsonTarget != nil {
  248. t := *jsonTarget
  249. if t.Kind() == reflect.Slice {
  250. // By default slices point to nil, but we need a reflect.Value
  251. // pointing to a value of the slice type, so we create one here.
  252. ev := reflect.Indirect(reflect.New(t.Type().Elem()))
  253. jsonSliceElemValue = &ev
  254. }
  255. }
  256. // Make and use a new array.
  257. arr := make([]interface{}, len(typedYAMLObj))
  258. for i, v := range typedYAMLObj {
  259. arr[i], err = convertToJSONableObject(v, jsonSliceElemValue)
  260. if err != nil {
  261. return nil, err
  262. }
  263. }
  264. return arr, nil
  265. default:
  266. // If the target type is a string and the YAML type is a number,
  267. // convert the YAML type to a string.
  268. if jsonTarget != nil && (*jsonTarget).Kind() == reflect.String {
  269. // Based on my reading of go-yaml, it may return int, int64,
  270. // float64, or uint64.
  271. var s string
  272. switch typedVal := typedYAMLObj.(type) {
  273. case int:
  274. s = strconv.FormatInt(int64(typedVal), 10)
  275. case int64:
  276. s = strconv.FormatInt(typedVal, 10)
  277. case float64:
  278. s = strconv.FormatFloat(typedVal, 'g', -1, 32)
  279. case uint64:
  280. s = strconv.FormatUint(typedVal, 10)
  281. case bool:
  282. if typedVal {
  283. s = "true"
  284. } else {
  285. s = "false"
  286. }
  287. }
  288. if len(s) > 0 {
  289. yamlObj = interface{}(s)
  290. }
  291. }
  292. return yamlObj, nil
  293. }
  294. }
  295. // JSONObjectToYAMLObject converts an in-memory JSON object into a YAML in-memory MapSlice,
  296. // without going through a byte representation. A nil or empty map[string]interface{} input is
  297. // converted to an empty map, i.e. yaml.MapSlice(nil).
  298. //
  299. // interface{} slices stay interface{} slices. map[string]interface{} becomes yaml.MapSlice.
  300. //
  301. // int64 and float64 are down casted following the logic of github.com/go-yaml/yaml:
  302. // - float64s are down-casted as far as possible without data-loss to int, int64, uint64.
  303. // - int64s are down-casted to int if possible without data-loss.
  304. //
  305. // Big int/int64/uint64 do not lose precision as in the json-yaml roundtripping case.
  306. //
  307. // string, bool and any other types are unchanged.
  308. func JSONObjectToYAMLObject(j map[string]interface{}) yaml.MapSlice {
  309. if len(j) == 0 {
  310. return nil
  311. }
  312. ret := make(yaml.MapSlice, 0, len(j))
  313. for k, v := range j {
  314. ret = append(ret, yaml.MapItem{Key: k, Value: jsonToYAMLValue(v)})
  315. }
  316. return ret
  317. }
  318. func jsonToYAMLValue(j interface{}) interface{} {
  319. switch j := j.(type) {
  320. case map[string]interface{}:
  321. if j == nil {
  322. return interface{}(nil)
  323. }
  324. return JSONObjectToYAMLObject(j)
  325. case []interface{}:
  326. if j == nil {
  327. return interface{}(nil)
  328. }
  329. ret := make([]interface{}, len(j))
  330. for i := range j {
  331. ret[i] = jsonToYAMLValue(j[i])
  332. }
  333. return ret
  334. case float64:
  335. // replicate the logic in https://github.com/go-yaml/yaml/blob/51d6538a90f86fe93ac480b35f37b2be17fef232/resolve.go#L151
  336. if i64 := int64(j); j == float64(i64) {
  337. if i := int(i64); i64 == int64(i) {
  338. return i
  339. }
  340. return i64
  341. }
  342. if ui64 := uint64(j); j == float64(ui64) {
  343. return ui64
  344. }
  345. return j
  346. case int64:
  347. if i := int(j); j == int64(i) {
  348. return i
  349. }
  350. return j
  351. }
  352. return j
  353. }