load.go 8.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334
  1. // Copyright 2011 Google Inc. All rights reserved.
  2. // Use of this source code is governed by the Apache 2.0
  3. // license that can be found in the LICENSE file.
  4. package datastore
  5. import (
  6. "fmt"
  7. "reflect"
  8. "time"
  9. "google.golang.org/appengine"
  10. pb "google.golang.org/appengine/internal/datastore"
  11. )
  12. var (
  13. typeOfBlobKey = reflect.TypeOf(appengine.BlobKey(""))
  14. typeOfByteSlice = reflect.TypeOf([]byte(nil))
  15. typeOfByteString = reflect.TypeOf(ByteString(nil))
  16. typeOfGeoPoint = reflect.TypeOf(appengine.GeoPoint{})
  17. typeOfTime = reflect.TypeOf(time.Time{})
  18. )
  19. // typeMismatchReason returns a string explaining why the property p could not
  20. // be stored in an entity field of type v.Type().
  21. func typeMismatchReason(p Property, v reflect.Value) string {
  22. entityType := "empty"
  23. switch p.Value.(type) {
  24. case int64:
  25. entityType = "int"
  26. case bool:
  27. entityType = "bool"
  28. case string:
  29. entityType = "string"
  30. case float64:
  31. entityType = "float"
  32. case *Key:
  33. entityType = "*datastore.Key"
  34. case time.Time:
  35. entityType = "time.Time"
  36. case appengine.BlobKey:
  37. entityType = "appengine.BlobKey"
  38. case appengine.GeoPoint:
  39. entityType = "appengine.GeoPoint"
  40. case ByteString:
  41. entityType = "datastore.ByteString"
  42. case []byte:
  43. entityType = "[]byte"
  44. }
  45. return fmt.Sprintf("type mismatch: %s versus %v", entityType, v.Type())
  46. }
  47. type propertyLoader struct {
  48. // m holds the number of times a substruct field like "Foo.Bar.Baz" has
  49. // been seen so far. The map is constructed lazily.
  50. m map[string]int
  51. }
  52. func (l *propertyLoader) load(codec *structCodec, structValue reflect.Value, p Property, requireSlice bool) string {
  53. var v reflect.Value
  54. // Traverse a struct's struct-typed fields.
  55. for name := p.Name; ; {
  56. decoder, ok := codec.byName[name]
  57. if !ok {
  58. return "no such struct field"
  59. }
  60. v = structValue.Field(decoder.index)
  61. if !v.IsValid() {
  62. return "no such struct field"
  63. }
  64. if !v.CanSet() {
  65. return "cannot set struct field"
  66. }
  67. if decoder.substructCodec == nil {
  68. break
  69. }
  70. if v.Kind() == reflect.Slice {
  71. if l.m == nil {
  72. l.m = make(map[string]int)
  73. }
  74. index := l.m[p.Name]
  75. l.m[p.Name] = index + 1
  76. for v.Len() <= index {
  77. v.Set(reflect.Append(v, reflect.New(v.Type().Elem()).Elem()))
  78. }
  79. structValue = v.Index(index)
  80. requireSlice = false
  81. } else {
  82. structValue = v
  83. }
  84. // Strip the "I." from "I.X".
  85. name = name[len(codec.byIndex[decoder.index].name):]
  86. codec = decoder.substructCodec
  87. }
  88. var slice reflect.Value
  89. if v.Kind() == reflect.Slice && v.Type().Elem().Kind() != reflect.Uint8 {
  90. slice = v
  91. v = reflect.New(v.Type().Elem()).Elem()
  92. } else if requireSlice {
  93. return "multiple-valued property requires a slice field type"
  94. }
  95. // Convert indexValues to a Go value with a meaning derived from the
  96. // destination type.
  97. pValue := p.Value
  98. if iv, ok := pValue.(indexValue); ok {
  99. meaning := pb.Property_NO_MEANING
  100. switch v.Type() {
  101. case typeOfBlobKey:
  102. meaning = pb.Property_BLOBKEY
  103. case typeOfByteSlice:
  104. meaning = pb.Property_BLOB
  105. case typeOfByteString:
  106. meaning = pb.Property_BYTESTRING
  107. case typeOfGeoPoint:
  108. meaning = pb.Property_GEORSS_POINT
  109. case typeOfTime:
  110. meaning = pb.Property_GD_WHEN
  111. }
  112. var err error
  113. pValue, err = propValue(iv.value, meaning)
  114. if err != nil {
  115. return err.Error()
  116. }
  117. }
  118. switch v.Kind() {
  119. case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
  120. x, ok := pValue.(int64)
  121. if !ok && pValue != nil {
  122. return typeMismatchReason(p, v)
  123. }
  124. if v.OverflowInt(x) {
  125. return fmt.Sprintf("value %v overflows struct field of type %v", x, v.Type())
  126. }
  127. v.SetInt(x)
  128. case reflect.Bool:
  129. x, ok := pValue.(bool)
  130. if !ok && pValue != nil {
  131. return typeMismatchReason(p, v)
  132. }
  133. v.SetBool(x)
  134. case reflect.String:
  135. switch x := pValue.(type) {
  136. case appengine.BlobKey:
  137. v.SetString(string(x))
  138. case ByteString:
  139. v.SetString(string(x))
  140. case string:
  141. v.SetString(x)
  142. default:
  143. if pValue != nil {
  144. return typeMismatchReason(p, v)
  145. }
  146. }
  147. case reflect.Float32, reflect.Float64:
  148. x, ok := pValue.(float64)
  149. if !ok && pValue != nil {
  150. return typeMismatchReason(p, v)
  151. }
  152. if v.OverflowFloat(x) {
  153. return fmt.Sprintf("value %v overflows struct field of type %v", x, v.Type())
  154. }
  155. v.SetFloat(x)
  156. case reflect.Ptr:
  157. x, ok := pValue.(*Key)
  158. if !ok && pValue != nil {
  159. return typeMismatchReason(p, v)
  160. }
  161. if _, ok := v.Interface().(*Key); !ok {
  162. return typeMismatchReason(p, v)
  163. }
  164. v.Set(reflect.ValueOf(x))
  165. case reflect.Struct:
  166. switch v.Type() {
  167. case typeOfTime:
  168. x, ok := pValue.(time.Time)
  169. if !ok && pValue != nil {
  170. return typeMismatchReason(p, v)
  171. }
  172. v.Set(reflect.ValueOf(x))
  173. case typeOfGeoPoint:
  174. x, ok := pValue.(appengine.GeoPoint)
  175. if !ok && pValue != nil {
  176. return typeMismatchReason(p, v)
  177. }
  178. v.Set(reflect.ValueOf(x))
  179. default:
  180. return typeMismatchReason(p, v)
  181. }
  182. case reflect.Slice:
  183. x, ok := pValue.([]byte)
  184. if !ok {
  185. if y, yok := pValue.(ByteString); yok {
  186. x, ok = []byte(y), true
  187. }
  188. }
  189. if !ok && pValue != nil {
  190. return typeMismatchReason(p, v)
  191. }
  192. if v.Type().Elem().Kind() != reflect.Uint8 {
  193. return typeMismatchReason(p, v)
  194. }
  195. v.SetBytes(x)
  196. default:
  197. return typeMismatchReason(p, v)
  198. }
  199. if slice.IsValid() {
  200. slice.Set(reflect.Append(slice, v))
  201. }
  202. return ""
  203. }
  204. // loadEntity loads an EntityProto into PropertyLoadSaver or struct pointer.
  205. func loadEntity(dst interface{}, src *pb.EntityProto) (err error) {
  206. props, err := protoToProperties(src)
  207. if err != nil {
  208. return err
  209. }
  210. if e, ok := dst.(PropertyLoadSaver); ok {
  211. return e.Load(props)
  212. }
  213. return LoadStruct(dst, props)
  214. }
  215. func (s structPLS) Load(props []Property) error {
  216. var fieldName, reason string
  217. var l propertyLoader
  218. for _, p := range props {
  219. if errStr := l.load(s.codec, s.v, p, p.Multiple); errStr != "" {
  220. // We don't return early, as we try to load as many properties as possible.
  221. // It is valid to load an entity into a struct that cannot fully represent it.
  222. // That case returns an error, but the caller is free to ignore it.
  223. fieldName, reason = p.Name, errStr
  224. }
  225. }
  226. if reason != "" {
  227. return &ErrFieldMismatch{
  228. StructType: s.v.Type(),
  229. FieldName: fieldName,
  230. Reason: reason,
  231. }
  232. }
  233. return nil
  234. }
  235. func protoToProperties(src *pb.EntityProto) ([]Property, error) {
  236. props, rawProps := src.Property, src.RawProperty
  237. out := make([]Property, 0, len(props)+len(rawProps))
  238. for {
  239. var (
  240. x *pb.Property
  241. noIndex bool
  242. )
  243. if len(props) > 0 {
  244. x, props = props[0], props[1:]
  245. } else if len(rawProps) > 0 {
  246. x, rawProps = rawProps[0], rawProps[1:]
  247. noIndex = true
  248. } else {
  249. break
  250. }
  251. var value interface{}
  252. if x.Meaning != nil && *x.Meaning == pb.Property_INDEX_VALUE {
  253. value = indexValue{x.Value}
  254. } else {
  255. var err error
  256. value, err = propValue(x.Value, x.GetMeaning())
  257. if err != nil {
  258. return nil, err
  259. }
  260. }
  261. out = append(out, Property{
  262. Name: x.GetName(),
  263. Value: value,
  264. NoIndex: noIndex,
  265. Multiple: x.GetMultiple(),
  266. })
  267. }
  268. return out, nil
  269. }
  270. // propValue returns a Go value that combines the raw PropertyValue with a
  271. // meaning. For example, an Int64Value with GD_WHEN becomes a time.Time.
  272. func propValue(v *pb.PropertyValue, m pb.Property_Meaning) (interface{}, error) {
  273. switch {
  274. case v.Int64Value != nil:
  275. if m == pb.Property_GD_WHEN {
  276. return fromUnixMicro(*v.Int64Value), nil
  277. } else {
  278. return *v.Int64Value, nil
  279. }
  280. case v.BooleanValue != nil:
  281. return *v.BooleanValue, nil
  282. case v.StringValue != nil:
  283. if m == pb.Property_BLOB {
  284. return []byte(*v.StringValue), nil
  285. } else if m == pb.Property_BLOBKEY {
  286. return appengine.BlobKey(*v.StringValue), nil
  287. } else if m == pb.Property_BYTESTRING {
  288. return ByteString(*v.StringValue), nil
  289. } else {
  290. return *v.StringValue, nil
  291. }
  292. case v.DoubleValue != nil:
  293. return *v.DoubleValue, nil
  294. case v.Referencevalue != nil:
  295. key, err := referenceValueToKey(v.Referencevalue)
  296. if err != nil {
  297. return nil, err
  298. }
  299. return key, nil
  300. case v.Pointvalue != nil:
  301. // NOTE: Strangely, latitude maps to X, longitude to Y.
  302. return appengine.GeoPoint{Lat: v.Pointvalue.GetX(), Lng: v.Pointvalue.GetY()}, nil
  303. }
  304. return nil, nil
  305. }
  306. // indexValue is a Property value that is created when entities are loaded from
  307. // an index, such as from a projection query.
  308. //
  309. // Such Property values do not contain all of the metadata required to be
  310. // faithfully represented as a Go value, and are instead represented as an
  311. // opaque indexValue. Load the properties into a concrete struct type (e.g. by
  312. // passing a struct pointer to Iterator.Next) to reconstruct actual Go values
  313. // of type int, string, time.Time, etc.
  314. type indexValue struct {
  315. value *pb.PropertyValue
  316. }