lookup.go 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412
  1. // Copyright 2013 The Go Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style
  3. // license that can be found in the LICENSE file.
  4. package language
  5. import (
  6. "bytes"
  7. "fmt"
  8. "sort"
  9. "strconv"
  10. "golang.org/x/text/internal/tag"
  11. )
  12. // findIndex tries to find the given tag in idx and returns a standardized error
  13. // if it could not be found.
  14. func findIndex(idx tag.Index, key []byte, form string) (index int, err error) {
  15. if !tag.FixCase(form, key) {
  16. return 0, ErrSyntax
  17. }
  18. i := idx.Index(key)
  19. if i == -1 {
  20. return 0, NewValueError(key)
  21. }
  22. return i, nil
  23. }
  24. func searchUint(imap []uint16, key uint16) int {
  25. return sort.Search(len(imap), func(i int) bool {
  26. return imap[i] >= key
  27. })
  28. }
  29. type Language uint16
  30. // getLangID returns the langID of s if s is a canonical subtag
  31. // or langUnknown if s is not a canonical subtag.
  32. func getLangID(s []byte) (Language, error) {
  33. if len(s) == 2 {
  34. return getLangISO2(s)
  35. }
  36. return getLangISO3(s)
  37. }
  38. // TODO language normalization as well as the AliasMaps could be moved to the
  39. // higher level package, but it is a bit tricky to separate the generation.
  40. func (id Language) Canonicalize() (Language, AliasType) {
  41. return normLang(id)
  42. }
  43. // mapLang returns the mapped langID of id according to mapping m.
  44. func normLang(id Language) (Language, AliasType) {
  45. k := sort.Search(len(AliasMap), func(i int) bool {
  46. return AliasMap[i].From >= uint16(id)
  47. })
  48. if k < len(AliasMap) && AliasMap[k].From == uint16(id) {
  49. return Language(AliasMap[k].To), AliasTypes[k]
  50. }
  51. return id, AliasTypeUnknown
  52. }
  53. // getLangISO2 returns the langID for the given 2-letter ISO language code
  54. // or unknownLang if this does not exist.
  55. func getLangISO2(s []byte) (Language, error) {
  56. if !tag.FixCase("zz", s) {
  57. return 0, ErrSyntax
  58. }
  59. if i := lang.Index(s); i != -1 && lang.Elem(i)[3] != 0 {
  60. return Language(i), nil
  61. }
  62. return 0, NewValueError(s)
  63. }
  64. const base = 'z' - 'a' + 1
  65. func strToInt(s []byte) uint {
  66. v := uint(0)
  67. for i := 0; i < len(s); i++ {
  68. v *= base
  69. v += uint(s[i] - 'a')
  70. }
  71. return v
  72. }
  73. // converts the given integer to the original ASCII string passed to strToInt.
  74. // len(s) must match the number of characters obtained.
  75. func intToStr(v uint, s []byte) {
  76. for i := len(s) - 1; i >= 0; i-- {
  77. s[i] = byte(v%base) + 'a'
  78. v /= base
  79. }
  80. }
  81. // getLangISO3 returns the langID for the given 3-letter ISO language code
  82. // or unknownLang if this does not exist.
  83. func getLangISO3(s []byte) (Language, error) {
  84. if tag.FixCase("und", s) {
  85. // first try to match canonical 3-letter entries
  86. for i := lang.Index(s[:2]); i != -1; i = lang.Next(s[:2], i) {
  87. if e := lang.Elem(i); e[3] == 0 && e[2] == s[2] {
  88. // We treat "und" as special and always translate it to "unspecified".
  89. // Note that ZZ and Zzzz are private use and are not treated as
  90. // unspecified by default.
  91. id := Language(i)
  92. if id == nonCanonicalUnd {
  93. return 0, nil
  94. }
  95. return id, nil
  96. }
  97. }
  98. if i := altLangISO3.Index(s); i != -1 {
  99. return Language(altLangIndex[altLangISO3.Elem(i)[3]]), nil
  100. }
  101. n := strToInt(s)
  102. if langNoIndex[n/8]&(1<<(n%8)) != 0 {
  103. return Language(n) + langNoIndexOffset, nil
  104. }
  105. // Check for non-canonical uses of ISO3.
  106. for i := lang.Index(s[:1]); i != -1; i = lang.Next(s[:1], i) {
  107. if e := lang.Elem(i); e[2] == s[1] && e[3] == s[2] {
  108. return Language(i), nil
  109. }
  110. }
  111. return 0, NewValueError(s)
  112. }
  113. return 0, ErrSyntax
  114. }
  115. // StringToBuf writes the string to b and returns the number of bytes
  116. // written. cap(b) must be >= 3.
  117. func (id Language) StringToBuf(b []byte) int {
  118. if id >= langNoIndexOffset {
  119. intToStr(uint(id)-langNoIndexOffset, b[:3])
  120. return 3
  121. } else if id == 0 {
  122. return copy(b, "und")
  123. }
  124. l := lang[id<<2:]
  125. if l[3] == 0 {
  126. return copy(b, l[:3])
  127. }
  128. return copy(b, l[:2])
  129. }
  130. // String returns the BCP 47 representation of the langID.
  131. // Use b as variable name, instead of id, to ensure the variable
  132. // used is consistent with that of Base in which this type is embedded.
  133. func (b Language) String() string {
  134. if b == 0 {
  135. return "und"
  136. } else if b >= langNoIndexOffset {
  137. b -= langNoIndexOffset
  138. buf := [3]byte{}
  139. intToStr(uint(b), buf[:])
  140. return string(buf[:])
  141. }
  142. l := lang.Elem(int(b))
  143. if l[3] == 0 {
  144. return l[:3]
  145. }
  146. return l[:2]
  147. }
  148. // ISO3 returns the ISO 639-3 language code.
  149. func (b Language) ISO3() string {
  150. if b == 0 || b >= langNoIndexOffset {
  151. return b.String()
  152. }
  153. l := lang.Elem(int(b))
  154. if l[3] == 0 {
  155. return l[:3]
  156. } else if l[2] == 0 {
  157. return altLangISO3.Elem(int(l[3]))[:3]
  158. }
  159. // This allocation will only happen for 3-letter ISO codes
  160. // that are non-canonical BCP 47 language identifiers.
  161. return l[0:1] + l[2:4]
  162. }
  163. // IsPrivateUse reports whether this language code is reserved for private use.
  164. func (b Language) IsPrivateUse() bool {
  165. return langPrivateStart <= b && b <= langPrivateEnd
  166. }
  167. // SuppressScript returns the script marked as SuppressScript in the IANA
  168. // language tag repository, or 0 if there is no such script.
  169. func (b Language) SuppressScript() Script {
  170. if b < langNoIndexOffset {
  171. return Script(suppressScript[b])
  172. }
  173. return 0
  174. }
  175. type Region uint16
  176. // getRegionID returns the region id for s if s is a valid 2-letter region code
  177. // or unknownRegion.
  178. func getRegionID(s []byte) (Region, error) {
  179. if len(s) == 3 {
  180. if isAlpha(s[0]) {
  181. return getRegionISO3(s)
  182. }
  183. if i, err := strconv.ParseUint(string(s), 10, 10); err == nil {
  184. return getRegionM49(int(i))
  185. }
  186. }
  187. return getRegionISO2(s)
  188. }
  189. // getRegionISO2 returns the regionID for the given 2-letter ISO country code
  190. // or unknownRegion if this does not exist.
  191. func getRegionISO2(s []byte) (Region, error) {
  192. i, err := findIndex(regionISO, s, "ZZ")
  193. if err != nil {
  194. return 0, err
  195. }
  196. return Region(i) + isoRegionOffset, nil
  197. }
  198. // getRegionISO3 returns the regionID for the given 3-letter ISO country code
  199. // or unknownRegion if this does not exist.
  200. func getRegionISO3(s []byte) (Region, error) {
  201. if tag.FixCase("ZZZ", s) {
  202. for i := regionISO.Index(s[:1]); i != -1; i = regionISO.Next(s[:1], i) {
  203. if e := regionISO.Elem(i); e[2] == s[1] && e[3] == s[2] {
  204. return Region(i) + isoRegionOffset, nil
  205. }
  206. }
  207. for i := 0; i < len(altRegionISO3); i += 3 {
  208. if tag.Compare(altRegionISO3[i:i+3], s) == 0 {
  209. return Region(altRegionIDs[i/3]), nil
  210. }
  211. }
  212. return 0, NewValueError(s)
  213. }
  214. return 0, ErrSyntax
  215. }
  216. func getRegionM49(n int) (Region, error) {
  217. if 0 < n && n <= 999 {
  218. const (
  219. searchBits = 7
  220. regionBits = 9
  221. regionMask = 1<<regionBits - 1
  222. )
  223. idx := n >> searchBits
  224. buf := fromM49[m49Index[idx]:m49Index[idx+1]]
  225. val := uint16(n) << regionBits // we rely on bits shifting out
  226. i := sort.Search(len(buf), func(i int) bool {
  227. return buf[i] >= val
  228. })
  229. if r := fromM49[int(m49Index[idx])+i]; r&^regionMask == val {
  230. return Region(r & regionMask), nil
  231. }
  232. }
  233. var e ValueError
  234. fmt.Fprint(bytes.NewBuffer([]byte(e.v[:])), n)
  235. return 0, e
  236. }
  237. // normRegion returns a region if r is deprecated or 0 otherwise.
  238. // TODO: consider supporting BYS (-> BLR), CSK (-> 200 or CZ), PHI (-> PHL) and AFI (-> DJ).
  239. // TODO: consider mapping split up regions to new most populous one (like CLDR).
  240. func normRegion(r Region) Region {
  241. m := regionOldMap
  242. k := sort.Search(len(m), func(i int) bool {
  243. return m[i].From >= uint16(r)
  244. })
  245. if k < len(m) && m[k].From == uint16(r) {
  246. return Region(m[k].To)
  247. }
  248. return 0
  249. }
  250. const (
  251. iso3166UserAssigned = 1 << iota
  252. ccTLD
  253. bcp47Region
  254. )
  255. func (r Region) typ() byte {
  256. return regionTypes[r]
  257. }
  258. // String returns the BCP 47 representation for the region.
  259. // It returns "ZZ" for an unspecified region.
  260. func (r Region) String() string {
  261. if r < isoRegionOffset {
  262. if r == 0 {
  263. return "ZZ"
  264. }
  265. return fmt.Sprintf("%03d", r.M49())
  266. }
  267. r -= isoRegionOffset
  268. return regionISO.Elem(int(r))[:2]
  269. }
  270. // ISO3 returns the 3-letter ISO code of r.
  271. // Note that not all regions have a 3-letter ISO code.
  272. // In such cases this method returns "ZZZ".
  273. func (r Region) ISO3() string {
  274. if r < isoRegionOffset {
  275. return "ZZZ"
  276. }
  277. r -= isoRegionOffset
  278. reg := regionISO.Elem(int(r))
  279. switch reg[2] {
  280. case 0:
  281. return altRegionISO3[reg[3]:][:3]
  282. case ' ':
  283. return "ZZZ"
  284. }
  285. return reg[0:1] + reg[2:4]
  286. }
  287. // M49 returns the UN M.49 encoding of r, or 0 if this encoding
  288. // is not defined for r.
  289. func (r Region) M49() int {
  290. return int(m49[r])
  291. }
  292. // IsPrivateUse reports whether r has the ISO 3166 User-assigned status. This
  293. // may include private-use tags that are assigned by CLDR and used in this
  294. // implementation. So IsPrivateUse and IsCountry can be simultaneously true.
  295. func (r Region) IsPrivateUse() bool {
  296. return r.typ()&iso3166UserAssigned != 0
  297. }
  298. type Script uint8
  299. // getScriptID returns the script id for string s. It assumes that s
  300. // is of the format [A-Z][a-z]{3}.
  301. func getScriptID(idx tag.Index, s []byte) (Script, error) {
  302. i, err := findIndex(idx, s, "Zzzz")
  303. return Script(i), err
  304. }
  305. // String returns the script code in title case.
  306. // It returns "Zzzz" for an unspecified script.
  307. func (s Script) String() string {
  308. if s == 0 {
  309. return "Zzzz"
  310. }
  311. return script.Elem(int(s))
  312. }
  313. // IsPrivateUse reports whether this script code is reserved for private use.
  314. func (s Script) IsPrivateUse() bool {
  315. return _Qaaa <= s && s <= _Qabx
  316. }
  317. const (
  318. maxAltTaglen = len("en-US-POSIX")
  319. maxLen = maxAltTaglen
  320. )
  321. var (
  322. // grandfatheredMap holds a mapping from legacy and grandfathered tags to
  323. // their base language or index to more elaborate tag.
  324. grandfatheredMap = map[[maxLen]byte]int16{
  325. [maxLen]byte{'a', 'r', 't', '-', 'l', 'o', 'j', 'b', 'a', 'n'}: _jbo, // art-lojban
  326. [maxLen]byte{'i', '-', 'a', 'm', 'i'}: _ami, // i-ami
  327. [maxLen]byte{'i', '-', 'b', 'n', 'n'}: _bnn, // i-bnn
  328. [maxLen]byte{'i', '-', 'h', 'a', 'k'}: _hak, // i-hak
  329. [maxLen]byte{'i', '-', 'k', 'l', 'i', 'n', 'g', 'o', 'n'}: _tlh, // i-klingon
  330. [maxLen]byte{'i', '-', 'l', 'u', 'x'}: _lb, // i-lux
  331. [maxLen]byte{'i', '-', 'n', 'a', 'v', 'a', 'j', 'o'}: _nv, // i-navajo
  332. [maxLen]byte{'i', '-', 'p', 'w', 'n'}: _pwn, // i-pwn
  333. [maxLen]byte{'i', '-', 't', 'a', 'o'}: _tao, // i-tao
  334. [maxLen]byte{'i', '-', 't', 'a', 'y'}: _tay, // i-tay
  335. [maxLen]byte{'i', '-', 't', 's', 'u'}: _tsu, // i-tsu
  336. [maxLen]byte{'n', 'o', '-', 'b', 'o', 'k'}: _nb, // no-bok
  337. [maxLen]byte{'n', 'o', '-', 'n', 'y', 'n'}: _nn, // no-nyn
  338. [maxLen]byte{'s', 'g', 'n', '-', 'b', 'e', '-', 'f', 'r'}: _sfb, // sgn-BE-FR
  339. [maxLen]byte{'s', 'g', 'n', '-', 'b', 'e', '-', 'n', 'l'}: _vgt, // sgn-BE-NL
  340. [maxLen]byte{'s', 'g', 'n', '-', 'c', 'h', '-', 'd', 'e'}: _sgg, // sgn-CH-DE
  341. [maxLen]byte{'z', 'h', '-', 'g', 'u', 'o', 'y', 'u'}: _cmn, // zh-guoyu
  342. [maxLen]byte{'z', 'h', '-', 'h', 'a', 'k', 'k', 'a'}: _hak, // zh-hakka
  343. [maxLen]byte{'z', 'h', '-', 'm', 'i', 'n', '-', 'n', 'a', 'n'}: _nan, // zh-min-nan
  344. [maxLen]byte{'z', 'h', '-', 'x', 'i', 'a', 'n', 'g'}: _hsn, // zh-xiang
  345. // Grandfathered tags with no modern replacement will be converted as
  346. // follows:
  347. [maxLen]byte{'c', 'e', 'l', '-', 'g', 'a', 'u', 'l', 'i', 's', 'h'}: -1, // cel-gaulish
  348. [maxLen]byte{'e', 'n', '-', 'g', 'b', '-', 'o', 'e', 'd'}: -2, // en-GB-oed
  349. [maxLen]byte{'i', '-', 'd', 'e', 'f', 'a', 'u', 'l', 't'}: -3, // i-default
  350. [maxLen]byte{'i', '-', 'e', 'n', 'o', 'c', 'h', 'i', 'a', 'n'}: -4, // i-enochian
  351. [maxLen]byte{'i', '-', 'm', 'i', 'n', 'g', 'o'}: -5, // i-mingo
  352. [maxLen]byte{'z', 'h', '-', 'm', 'i', 'n'}: -6, // zh-min
  353. // CLDR-specific tag.
  354. [maxLen]byte{'r', 'o', 'o', 't'}: 0, // root
  355. [maxLen]byte{'e', 'n', '-', 'u', 's', '-', 'p', 'o', 's', 'i', 'x'}: -7, // en_US_POSIX"
  356. }
  357. altTagIndex = [...]uint8{0, 17, 31, 45, 61, 74, 86, 102}
  358. altTags = "xtg-x-cel-gaulishen-GB-oxendicten-x-i-defaultund-x-i-enochiansee-x-i-mingonan-x-zh-minen-US-u-va-posix"
  359. )
  360. func grandfathered(s [maxAltTaglen]byte) (t Tag, ok bool) {
  361. if v, ok := grandfatheredMap[s]; ok {
  362. if v < 0 {
  363. return Make(altTags[altTagIndex[-v-1]:altTagIndex[-v]]), true
  364. }
  365. t.LangID = Language(v)
  366. return t, true
  367. }
  368. return t, false
  369. }