prop.go 8.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294
  1. // Copyright 2011 Google Inc. All rights reserved.
  2. // Use of this source code is governed by the Apache 2.0
  3. // license that can be found in the LICENSE file.
  4. package datastore
  5. import (
  6. "fmt"
  7. "reflect"
  8. "strings"
  9. "sync"
  10. "unicode"
  11. )
  12. // Entities with more than this many indexed properties will not be saved.
  13. const maxIndexedProperties = 20000
  14. // []byte fields more than 1 megabyte long will not be loaded or saved.
  15. const maxBlobLen = 1 << 20
  16. // Property is a name/value pair plus some metadata. A datastore entity's
  17. // contents are loaded and saved as a sequence of Properties. An entity can
  18. // have multiple Properties with the same name, provided that p.Multiple is
  19. // true on all of that entity's Properties with that name.
  20. type Property struct {
  21. // Name is the property name.
  22. Name string
  23. // Value is the property value. The valid types are:
  24. // - int64
  25. // - bool
  26. // - string
  27. // - float64
  28. // - ByteString
  29. // - *Key
  30. // - time.Time
  31. // - appengine.BlobKey
  32. // - appengine.GeoPoint
  33. // - []byte (up to 1 megabyte in length)
  34. // This set is smaller than the set of valid struct field types that the
  35. // datastore can load and save. A Property Value cannot be a slice (apart
  36. // from []byte); use multiple Properties instead. Also, a Value's type
  37. // must be explicitly on the list above; it is not sufficient for the
  38. // underlying type to be on that list. For example, a Value of "type
  39. // myInt64 int64" is invalid. Smaller-width integers and floats are also
  40. // invalid. Again, this is more restrictive than the set of valid struct
  41. // field types.
  42. //
  43. // A Value will have an opaque type when loading entities from an index,
  44. // such as via a projection query. Load entities into a struct instead
  45. // of a PropertyLoadSaver when using a projection query.
  46. //
  47. // A Value may also be the nil interface value; this is equivalent to
  48. // Python's None but not directly representable by a Go struct. Loading
  49. // a nil-valued property into a struct will set that field to the zero
  50. // value.
  51. Value interface{}
  52. // NoIndex is whether the datastore cannot index this property.
  53. NoIndex bool
  54. // Multiple is whether the entity can have multiple properties with
  55. // the same name. Even if a particular instance only has one property with
  56. // a certain name, Multiple should be true if a struct would best represent
  57. // it as a field of type []T instead of type T.
  58. Multiple bool
  59. }
  60. // ByteString is a short byte slice (up to 1500 bytes) that can be indexed.
  61. type ByteString []byte
  62. // PropertyLoadSaver can be converted from and to a slice of Properties.
  63. type PropertyLoadSaver interface {
  64. Load([]Property) error
  65. Save() ([]Property, error)
  66. }
  67. // PropertyList converts a []Property to implement PropertyLoadSaver.
  68. type PropertyList []Property
  69. var (
  70. typeOfPropertyLoadSaver = reflect.TypeOf((*PropertyLoadSaver)(nil)).Elem()
  71. typeOfPropertyList = reflect.TypeOf(PropertyList(nil))
  72. )
  73. // Load loads all of the provided properties into l.
  74. // It does not first reset *l to an empty slice.
  75. func (l *PropertyList) Load(p []Property) error {
  76. *l = append(*l, p...)
  77. return nil
  78. }
  79. // Save saves all of l's properties as a slice or Properties.
  80. func (l *PropertyList) Save() ([]Property, error) {
  81. return *l, nil
  82. }
  83. // validPropertyName returns whether name consists of one or more valid Go
  84. // identifiers joined by ".".
  85. func validPropertyName(name string) bool {
  86. if name == "" {
  87. return false
  88. }
  89. for _, s := range strings.Split(name, ".") {
  90. if s == "" {
  91. return false
  92. }
  93. first := true
  94. for _, c := range s {
  95. if first {
  96. first = false
  97. if c != '_' && !unicode.IsLetter(c) {
  98. return false
  99. }
  100. } else {
  101. if c != '_' && !unicode.IsLetter(c) && !unicode.IsDigit(c) {
  102. return false
  103. }
  104. }
  105. }
  106. }
  107. return true
  108. }
  109. // structTag is the parsed `datastore:"name,options"` tag of a struct field.
  110. // If a field has no tag, or the tag has an empty name, then the structTag's
  111. // name is just the field name. A "-" name means that the datastore ignores
  112. // that field.
  113. type structTag struct {
  114. name string
  115. noIndex bool
  116. }
  117. // structCodec describes how to convert a struct to and from a sequence of
  118. // properties.
  119. type structCodec struct {
  120. // byIndex gives the structTag for the i'th field.
  121. byIndex []structTag
  122. // byName gives the field codec for the structTag with the given name.
  123. byName map[string]fieldCodec
  124. // hasSlice is whether a struct or any of its nested or embedded structs
  125. // has a slice-typed field (other than []byte).
  126. hasSlice bool
  127. // complete is whether the structCodec is complete. An incomplete
  128. // structCodec may be encountered when walking a recursive struct.
  129. complete bool
  130. }
  131. // fieldCodec is a struct field's index and, if that struct field's type is
  132. // itself a struct, that substruct's structCodec.
  133. type fieldCodec struct {
  134. index int
  135. substructCodec *structCodec
  136. }
  137. // structCodecs collects the structCodecs that have already been calculated.
  138. var (
  139. structCodecsMutex sync.Mutex
  140. structCodecs = make(map[reflect.Type]*structCodec)
  141. )
  142. // getStructCodec returns the structCodec for the given struct type.
  143. func getStructCodec(t reflect.Type) (*structCodec, error) {
  144. structCodecsMutex.Lock()
  145. defer structCodecsMutex.Unlock()
  146. return getStructCodecLocked(t)
  147. }
  148. // getStructCodecLocked implements getStructCodec. The structCodecsMutex must
  149. // be held when calling this function.
  150. func getStructCodecLocked(t reflect.Type) (ret *structCodec, retErr error) {
  151. c, ok := structCodecs[t]
  152. if ok {
  153. return c, nil
  154. }
  155. c = &structCodec{
  156. byIndex: make([]structTag, t.NumField()),
  157. byName: make(map[string]fieldCodec),
  158. }
  159. // Add c to the structCodecs map before we are sure it is good. If t is
  160. // a recursive type, it needs to find the incomplete entry for itself in
  161. // the map.
  162. structCodecs[t] = c
  163. defer func() {
  164. if retErr != nil {
  165. delete(structCodecs, t)
  166. }
  167. }()
  168. for i := range c.byIndex {
  169. f := t.Field(i)
  170. name, opts := f.Tag.Get("datastore"), ""
  171. if i := strings.Index(name, ","); i != -1 {
  172. name, opts = name[:i], name[i+1:]
  173. }
  174. if name == "" {
  175. if !f.Anonymous {
  176. name = f.Name
  177. }
  178. } else if name == "-" {
  179. c.byIndex[i] = structTag{name: name}
  180. continue
  181. } else if !validPropertyName(name) {
  182. return nil, fmt.Errorf("datastore: struct tag has invalid property name: %q", name)
  183. }
  184. substructType, fIsSlice := reflect.Type(nil), false
  185. switch f.Type.Kind() {
  186. case reflect.Struct:
  187. substructType = f.Type
  188. case reflect.Slice:
  189. if f.Type.Elem().Kind() == reflect.Struct {
  190. substructType = f.Type.Elem()
  191. }
  192. fIsSlice = f.Type != typeOfByteSlice
  193. c.hasSlice = c.hasSlice || fIsSlice
  194. }
  195. if substructType != nil && substructType != typeOfTime && substructType != typeOfGeoPoint {
  196. if name != "" {
  197. name = name + "."
  198. }
  199. sub, err := getStructCodecLocked(substructType)
  200. if err != nil {
  201. return nil, err
  202. }
  203. if !sub.complete {
  204. return nil, fmt.Errorf("datastore: recursive struct: field %q", f.Name)
  205. }
  206. if fIsSlice && sub.hasSlice {
  207. return nil, fmt.Errorf(
  208. "datastore: flattening nested structs leads to a slice of slices: field %q", f.Name)
  209. }
  210. c.hasSlice = c.hasSlice || sub.hasSlice
  211. for relName := range sub.byName {
  212. absName := name + relName
  213. if _, ok := c.byName[absName]; ok {
  214. return nil, fmt.Errorf("datastore: struct tag has repeated property name: %q", absName)
  215. }
  216. c.byName[absName] = fieldCodec{index: i, substructCodec: sub}
  217. }
  218. } else {
  219. if _, ok := c.byName[name]; ok {
  220. return nil, fmt.Errorf("datastore: struct tag has repeated property name: %q", name)
  221. }
  222. c.byName[name] = fieldCodec{index: i}
  223. }
  224. c.byIndex[i] = structTag{
  225. name: name,
  226. noIndex: opts == "noindex",
  227. }
  228. }
  229. c.complete = true
  230. return c, nil
  231. }
  232. // structPLS adapts a struct to be a PropertyLoadSaver.
  233. type structPLS struct {
  234. v reflect.Value
  235. codec *structCodec
  236. }
  237. // newStructPLS returns a PropertyLoadSaver for the struct pointer p.
  238. func newStructPLS(p interface{}) (PropertyLoadSaver, error) {
  239. v := reflect.ValueOf(p)
  240. if v.Kind() != reflect.Ptr || v.Elem().Kind() != reflect.Struct {
  241. return nil, ErrInvalidEntityType
  242. }
  243. v = v.Elem()
  244. codec, err := getStructCodec(v.Type())
  245. if err != nil {
  246. return nil, err
  247. }
  248. return structPLS{v, codec}, nil
  249. }
  250. // LoadStruct loads the properties from p to dst.
  251. // dst must be a struct pointer.
  252. func LoadStruct(dst interface{}, p []Property) error {
  253. x, err := newStructPLS(dst)
  254. if err != nil {
  255. return err
  256. }
  257. return x.Load(p)
  258. }
  259. // SaveStruct returns the properties from src as a slice of Properties.
  260. // src must be a struct pointer.
  261. func SaveStruct(src interface{}) ([]Property, error) {
  262. x, err := newStructPLS(src)
  263. if err != nil {
  264. return nil, err
  265. }
  266. return x.Save()
  267. }