reader.go 5.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241
  1. // Copyright 2017 Google Inc. All Rights Reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package compiler
  15. import (
  16. "errors"
  17. "fmt"
  18. "io/ioutil"
  19. "log"
  20. "net/http"
  21. "net/url"
  22. "path/filepath"
  23. "strings"
  24. yaml "gopkg.in/yaml.v2"
  25. )
  26. var fileCache map[string][]byte
  27. var infoCache map[string]interface{}
  28. var count int64
  29. var verboseReader = false
  30. var fileCacheEnable = true
  31. var infoCacheEnable = true
  32. func initializeFileCache() {
  33. if fileCache == nil {
  34. fileCache = make(map[string][]byte, 0)
  35. }
  36. }
  37. func initializeInfoCache() {
  38. if infoCache == nil {
  39. infoCache = make(map[string]interface{}, 0)
  40. }
  41. }
  42. func EnableFileCache() {
  43. fileCacheEnable = true
  44. }
  45. func EnableInfoCache() {
  46. infoCacheEnable = true
  47. }
  48. func DisableFileCache() {
  49. fileCacheEnable = false
  50. }
  51. func DisableInfoCache() {
  52. infoCacheEnable = false
  53. }
  54. func RemoveFromFileCache(fileurl string) {
  55. if !fileCacheEnable {
  56. return
  57. }
  58. initializeFileCache()
  59. delete(fileCache, fileurl)
  60. }
  61. func RemoveFromInfoCache(filename string) {
  62. if !infoCacheEnable {
  63. return
  64. }
  65. initializeInfoCache()
  66. delete(infoCache, filename)
  67. }
  68. func GetInfoCache() map[string]interface{} {
  69. if infoCache == nil {
  70. initializeInfoCache()
  71. }
  72. return infoCache
  73. }
  74. func ClearFileCache() {
  75. fileCache = make(map[string][]byte, 0)
  76. }
  77. func ClearInfoCache() {
  78. infoCache = make(map[string]interface{})
  79. }
  80. func ClearCaches() {
  81. ClearFileCache()
  82. ClearInfoCache()
  83. }
  84. // FetchFile gets a specified file from the local filesystem or a remote location.
  85. func FetchFile(fileurl string) ([]byte, error) {
  86. var bytes []byte
  87. initializeFileCache()
  88. if fileCacheEnable {
  89. bytes, ok := fileCache[fileurl]
  90. if ok {
  91. if verboseReader {
  92. log.Printf("Cache hit %s", fileurl)
  93. }
  94. return bytes, nil
  95. }
  96. if verboseReader {
  97. log.Printf("Fetching %s", fileurl)
  98. }
  99. }
  100. response, err := http.Get(fileurl)
  101. if err != nil {
  102. return nil, err
  103. }
  104. defer response.Body.Close()
  105. if response.StatusCode != 200 {
  106. return nil, errors.New(fmt.Sprintf("Error downloading %s: %s", fileurl, response.Status))
  107. }
  108. bytes, err = ioutil.ReadAll(response.Body)
  109. if fileCacheEnable && err == nil {
  110. fileCache[fileurl] = bytes
  111. }
  112. return bytes, err
  113. }
  114. // ReadBytesForFile reads the bytes of a file.
  115. func ReadBytesForFile(filename string) ([]byte, error) {
  116. // is the filename a url?
  117. fileurl, _ := url.Parse(filename)
  118. if fileurl.Scheme != "" {
  119. // yes, fetch it
  120. bytes, err := FetchFile(filename)
  121. if err != nil {
  122. return nil, err
  123. }
  124. return bytes, nil
  125. }
  126. // no, it's a local filename
  127. bytes, err := ioutil.ReadFile(filename)
  128. if err != nil {
  129. return nil, err
  130. }
  131. return bytes, nil
  132. }
  133. // ReadInfoFromBytes unmarshals a file as a yaml.MapSlice.
  134. func ReadInfoFromBytes(filename string, bytes []byte) (interface{}, error) {
  135. initializeInfoCache()
  136. if infoCacheEnable {
  137. cachedInfo, ok := infoCache[filename]
  138. if ok {
  139. if verboseReader {
  140. log.Printf("Cache hit info for file %s", filename)
  141. }
  142. return cachedInfo, nil
  143. }
  144. if verboseReader {
  145. log.Printf("Reading info for file %s", filename)
  146. }
  147. }
  148. var info yaml.MapSlice
  149. err := yaml.Unmarshal(bytes, &info)
  150. if err != nil {
  151. return nil, err
  152. }
  153. if infoCacheEnable && len(filename) > 0 {
  154. infoCache[filename] = info
  155. }
  156. return info, nil
  157. }
  158. // ReadInfoForRef reads a file and return the fragment needed to resolve a $ref.
  159. func ReadInfoForRef(basefile string, ref string) (interface{}, error) {
  160. initializeInfoCache()
  161. if infoCacheEnable {
  162. info, ok := infoCache[ref]
  163. if ok {
  164. if verboseReader {
  165. log.Printf("Cache hit for ref %s#%s", basefile, ref)
  166. }
  167. return info, nil
  168. }
  169. if verboseReader {
  170. log.Printf("Reading info for ref %s#%s", basefile, ref)
  171. }
  172. }
  173. count = count + 1
  174. basedir, _ := filepath.Split(basefile)
  175. parts := strings.Split(ref, "#")
  176. var filename string
  177. if parts[0] != "" {
  178. filename = parts[0]
  179. if _, err := url.ParseRequestURI(parts[0]); err != nil {
  180. // It is not an URL, so the file is local
  181. filename = basedir + parts[0]
  182. }
  183. } else {
  184. filename = basefile
  185. }
  186. bytes, err := ReadBytesForFile(filename)
  187. if err != nil {
  188. return nil, err
  189. }
  190. info, err := ReadInfoFromBytes(filename, bytes)
  191. if err != nil {
  192. log.Printf("File error: %v\n", err)
  193. } else {
  194. if len(parts) > 1 {
  195. path := strings.Split(parts[1], "/")
  196. for i, key := range path {
  197. if i > 0 {
  198. m, ok := info.(yaml.MapSlice)
  199. if ok {
  200. found := false
  201. for _, section := range m {
  202. if section.Key == key {
  203. info = section.Value
  204. found = true
  205. }
  206. }
  207. if !found {
  208. infoCache[ref] = nil
  209. return nil, NewError(nil, fmt.Sprintf("could not resolve %s", ref))
  210. }
  211. }
  212. }
  213. }
  214. }
  215. }
  216. if infoCacheEnable {
  217. infoCache[ref] = info
  218. }
  219. return info, nil
  220. }