parser.go 7.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283
  1. package configuration
  2. import (
  3. "fmt"
  4. "os"
  5. "reflect"
  6. "sort"
  7. "strconv"
  8. "strings"
  9. "github.com/Sirupsen/logrus"
  10. "gopkg.in/yaml.v2"
  11. )
  12. // Version is a major/minor version pair of the form Major.Minor
  13. // Major version upgrades indicate structure or type changes
  14. // Minor version upgrades should be strictly additive
  15. type Version string
  16. // MajorMinorVersion constructs a Version from its Major and Minor components
  17. func MajorMinorVersion(major, minor uint) Version {
  18. return Version(fmt.Sprintf("%d.%d", major, minor))
  19. }
  20. func (version Version) major() (uint, error) {
  21. majorPart := strings.Split(string(version), ".")[0]
  22. major, err := strconv.ParseUint(majorPart, 10, 0)
  23. return uint(major), err
  24. }
  25. // Major returns the major version portion of a Version
  26. func (version Version) Major() uint {
  27. major, _ := version.major()
  28. return major
  29. }
  30. func (version Version) minor() (uint, error) {
  31. minorPart := strings.Split(string(version), ".")[1]
  32. minor, err := strconv.ParseUint(minorPart, 10, 0)
  33. return uint(minor), err
  34. }
  35. // Minor returns the minor version portion of a Version
  36. func (version Version) Minor() uint {
  37. minor, _ := version.minor()
  38. return minor
  39. }
  40. // VersionedParseInfo defines how a specific version of a configuration should
  41. // be parsed into the current version
  42. type VersionedParseInfo struct {
  43. // Version is the version which this parsing information relates to
  44. Version Version
  45. // ParseAs defines the type which a configuration file of this version
  46. // should be parsed into
  47. ParseAs reflect.Type
  48. // ConversionFunc defines a method for converting the parsed configuration
  49. // (of type ParseAs) into the current configuration version
  50. // Note: this method signature is very unclear with the absence of generics
  51. ConversionFunc func(interface{}) (interface{}, error)
  52. }
  53. type envVar struct {
  54. name string
  55. value string
  56. }
  57. type envVars []envVar
  58. func (a envVars) Len() int { return len(a) }
  59. func (a envVars) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
  60. func (a envVars) Less(i, j int) bool { return a[i].name < a[j].name }
  61. // Parser can be used to parse a configuration file and environment of a defined
  62. // version into a unified output structure
  63. type Parser struct {
  64. prefix string
  65. mapping map[Version]VersionedParseInfo
  66. env envVars
  67. }
  68. // NewParser returns a *Parser with the given environment prefix which handles
  69. // versioned configurations which match the given parseInfos
  70. func NewParser(prefix string, parseInfos []VersionedParseInfo) *Parser {
  71. p := Parser{prefix: prefix, mapping: make(map[Version]VersionedParseInfo)}
  72. for _, parseInfo := range parseInfos {
  73. p.mapping[parseInfo.Version] = parseInfo
  74. }
  75. for _, env := range os.Environ() {
  76. envParts := strings.SplitN(env, "=", 2)
  77. p.env = append(p.env, envVar{envParts[0], envParts[1]})
  78. }
  79. // We must sort the environment variables lexically by name so that
  80. // more specific variables are applied before less specific ones
  81. // (i.e. REGISTRY_STORAGE before
  82. // REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY). This sucks, but it's a
  83. // lot simpler and easier to get right than unmarshalling map entries
  84. // into temporaries and merging with the existing entry.
  85. sort.Sort(p.env)
  86. return &p
  87. }
  88. // Parse reads in the given []byte and environment and writes the resulting
  89. // configuration into the input v
  90. //
  91. // Environment variables may be used to override configuration parameters other
  92. // than version, following the scheme below:
  93. // v.Abc may be replaced by the value of PREFIX_ABC,
  94. // v.Abc.Xyz may be replaced by the value of PREFIX_ABC_XYZ, and so forth
  95. func (p *Parser) Parse(in []byte, v interface{}) error {
  96. var versionedStruct struct {
  97. Version Version
  98. }
  99. if err := yaml.Unmarshal(in, &versionedStruct); err != nil {
  100. return err
  101. }
  102. parseInfo, ok := p.mapping[versionedStruct.Version]
  103. if !ok {
  104. return fmt.Errorf("Unsupported version: %q", versionedStruct.Version)
  105. }
  106. parseAs := reflect.New(parseInfo.ParseAs)
  107. err := yaml.Unmarshal(in, parseAs.Interface())
  108. if err != nil {
  109. return err
  110. }
  111. for _, envVar := range p.env {
  112. pathStr := envVar.name
  113. if strings.HasPrefix(pathStr, strings.ToUpper(p.prefix)+"_") {
  114. path := strings.Split(pathStr, "_")
  115. err = p.overwriteFields(parseAs, pathStr, path[1:], envVar.value)
  116. if err != nil {
  117. return err
  118. }
  119. }
  120. }
  121. c, err := parseInfo.ConversionFunc(parseAs.Interface())
  122. if err != nil {
  123. return err
  124. }
  125. reflect.ValueOf(v).Elem().Set(reflect.Indirect(reflect.ValueOf(c)))
  126. return nil
  127. }
  128. // overwriteFields replaces configuration values with alternate values specified
  129. // through the environment. Precondition: an empty path slice must never be
  130. // passed in.
  131. func (p *Parser) overwriteFields(v reflect.Value, fullpath string, path []string, payload string) error {
  132. for v.Kind() == reflect.Ptr {
  133. if v.IsNil() {
  134. panic("encountered nil pointer while handling environment variable " + fullpath)
  135. }
  136. v = reflect.Indirect(v)
  137. }
  138. switch v.Kind() {
  139. case reflect.Struct:
  140. return p.overwriteStruct(v, fullpath, path, payload)
  141. case reflect.Map:
  142. return p.overwriteMap(v, fullpath, path, payload)
  143. case reflect.Interface:
  144. if v.NumMethod() == 0 {
  145. if !v.IsNil() {
  146. return p.overwriteFields(v.Elem(), fullpath, path, payload)
  147. }
  148. // Interface was empty; create an implicit map
  149. var template map[string]interface{}
  150. wrappedV := reflect.MakeMap(reflect.TypeOf(template))
  151. v.Set(wrappedV)
  152. return p.overwriteMap(wrappedV, fullpath, path, payload)
  153. }
  154. }
  155. return nil
  156. }
  157. func (p *Parser) overwriteStruct(v reflect.Value, fullpath string, path []string, payload string) error {
  158. // Generate case-insensitive map of struct fields
  159. byUpperCase := make(map[string]int)
  160. for i := 0; i < v.NumField(); i++ {
  161. sf := v.Type().Field(i)
  162. upper := strings.ToUpper(sf.Name)
  163. if _, present := byUpperCase[upper]; present {
  164. panic(fmt.Sprintf("field name collision in configuration object: %s", sf.Name))
  165. }
  166. byUpperCase[upper] = i
  167. }
  168. fieldIndex, present := byUpperCase[path[0]]
  169. if !present {
  170. logrus.Warnf("Ignoring unrecognized environment variable %s", fullpath)
  171. return nil
  172. }
  173. field := v.Field(fieldIndex)
  174. sf := v.Type().Field(fieldIndex)
  175. if len(path) == 1 {
  176. // Env var specifies this field directly
  177. fieldVal := reflect.New(sf.Type)
  178. err := yaml.Unmarshal([]byte(payload), fieldVal.Interface())
  179. if err != nil {
  180. return err
  181. }
  182. field.Set(reflect.Indirect(fieldVal))
  183. return nil
  184. }
  185. // If the field is nil, must create an object
  186. switch sf.Type.Kind() {
  187. case reflect.Map:
  188. if field.IsNil() {
  189. field.Set(reflect.MakeMap(sf.Type))
  190. }
  191. case reflect.Ptr:
  192. if field.IsNil() {
  193. field.Set(reflect.New(sf.Type))
  194. }
  195. }
  196. err := p.overwriteFields(field, fullpath, path[1:], payload)
  197. if err != nil {
  198. return err
  199. }
  200. return nil
  201. }
  202. func (p *Parser) overwriteMap(m reflect.Value, fullpath string, path []string, payload string) error {
  203. if m.Type().Key().Kind() != reflect.String {
  204. // non-string keys unsupported
  205. logrus.Warnf("Ignoring environment variable %s involving map with non-string keys", fullpath)
  206. return nil
  207. }
  208. if len(path) > 1 {
  209. // If a matching key exists, get its value and continue the
  210. // overwriting process.
  211. for _, k := range m.MapKeys() {
  212. if strings.ToUpper(k.String()) == path[0] {
  213. mapValue := m.MapIndex(k)
  214. // If the existing value is nil, we want to
  215. // recreate it instead of using this value.
  216. if (mapValue.Kind() == reflect.Ptr ||
  217. mapValue.Kind() == reflect.Interface ||
  218. mapValue.Kind() == reflect.Map) &&
  219. mapValue.IsNil() {
  220. break
  221. }
  222. return p.overwriteFields(mapValue, fullpath, path[1:], payload)
  223. }
  224. }
  225. }
  226. // (Re)create this key
  227. var mapValue reflect.Value
  228. if m.Type().Elem().Kind() == reflect.Map {
  229. mapValue = reflect.MakeMap(m.Type().Elem())
  230. } else {
  231. mapValue = reflect.New(m.Type().Elem())
  232. }
  233. if len(path) > 1 {
  234. err := p.overwriteFields(mapValue, fullpath, path[1:], payload)
  235. if err != nil {
  236. return err
  237. }
  238. } else {
  239. err := yaml.Unmarshal([]byte(payload), mapValue.Interface())
  240. if err != nil {
  241. return err
  242. }
  243. }
  244. m.SetMapIndex(reflect.ValueOf(strings.ToLower(path[0])), reflect.Indirect(mapValue))
  245. return nil
  246. }