parser.go 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603
  1. package jmespath
  2. import (
  3. "encoding/json"
  4. "fmt"
  5. "strconv"
  6. "strings"
  7. )
  8. type astNodeType int
  9. //go:generate stringer -type astNodeType
  10. const (
  11. ASTEmpty astNodeType = iota
  12. ASTComparator
  13. ASTCurrentNode
  14. ASTExpRef
  15. ASTFunctionExpression
  16. ASTField
  17. ASTFilterProjection
  18. ASTFlatten
  19. ASTIdentity
  20. ASTIndex
  21. ASTIndexExpression
  22. ASTKeyValPair
  23. ASTLiteral
  24. ASTMultiSelectHash
  25. ASTMultiSelectList
  26. ASTOrExpression
  27. ASTAndExpression
  28. ASTNotExpression
  29. ASTPipe
  30. ASTProjection
  31. ASTSubexpression
  32. ASTSlice
  33. ASTValueProjection
  34. )
  35. // ASTNode represents the abstract syntax tree of a JMESPath expression.
  36. type ASTNode struct {
  37. nodeType astNodeType
  38. value interface{}
  39. children []ASTNode
  40. }
  41. func (node ASTNode) String() string {
  42. return node.PrettyPrint(0)
  43. }
  44. // PrettyPrint will pretty print the parsed AST.
  45. // The AST is an implementation detail and this pretty print
  46. // function is provided as a convenience method to help with
  47. // debugging. You should not rely on its output as the internal
  48. // structure of the AST may change at any time.
  49. func (node ASTNode) PrettyPrint(indent int) string {
  50. spaces := strings.Repeat(" ", indent)
  51. output := fmt.Sprintf("%s%s {\n", spaces, node.nodeType)
  52. nextIndent := indent + 2
  53. if node.value != nil {
  54. if converted, ok := node.value.(fmt.Stringer); ok {
  55. // Account for things like comparator nodes
  56. // that are enums with a String() method.
  57. output += fmt.Sprintf("%svalue: %s\n", strings.Repeat(" ", nextIndent), converted.String())
  58. } else {
  59. output += fmt.Sprintf("%svalue: %#v\n", strings.Repeat(" ", nextIndent), node.value)
  60. }
  61. }
  62. lastIndex := len(node.children)
  63. if lastIndex > 0 {
  64. output += fmt.Sprintf("%schildren: {\n", strings.Repeat(" ", nextIndent))
  65. childIndent := nextIndent + 2
  66. for _, elem := range node.children {
  67. output += elem.PrettyPrint(childIndent)
  68. }
  69. }
  70. output += fmt.Sprintf("%s}\n", spaces)
  71. return output
  72. }
  73. var bindingPowers = map[tokType]int{
  74. tEOF: 0,
  75. tUnquotedIdentifier: 0,
  76. tQuotedIdentifier: 0,
  77. tRbracket: 0,
  78. tRparen: 0,
  79. tComma: 0,
  80. tRbrace: 0,
  81. tNumber: 0,
  82. tCurrent: 0,
  83. tExpref: 0,
  84. tColon: 0,
  85. tPipe: 1,
  86. tOr: 2,
  87. tAnd: 3,
  88. tEQ: 5,
  89. tLT: 5,
  90. tLTE: 5,
  91. tGT: 5,
  92. tGTE: 5,
  93. tNE: 5,
  94. tFlatten: 9,
  95. tStar: 20,
  96. tFilter: 21,
  97. tDot: 40,
  98. tNot: 45,
  99. tLbrace: 50,
  100. tLbracket: 55,
  101. tLparen: 60,
  102. }
  103. // Parser holds state about the current expression being parsed.
  104. type Parser struct {
  105. expression string
  106. tokens []token
  107. index int
  108. }
  109. // NewParser creates a new JMESPath parser.
  110. func NewParser() *Parser {
  111. p := Parser{}
  112. return &p
  113. }
  114. // Parse will compile a JMESPath expression.
  115. func (p *Parser) Parse(expression string) (ASTNode, error) {
  116. lexer := NewLexer()
  117. p.expression = expression
  118. p.index = 0
  119. tokens, err := lexer.tokenize(expression)
  120. if err != nil {
  121. return ASTNode{}, err
  122. }
  123. p.tokens = tokens
  124. parsed, err := p.parseExpression(0)
  125. if err != nil {
  126. return ASTNode{}, err
  127. }
  128. if p.current() != tEOF {
  129. return ASTNode{}, p.syntaxError(fmt.Sprintf(
  130. "Unexpected token at the end of the expresssion: %s", p.current()))
  131. }
  132. return parsed, nil
  133. }
  134. func (p *Parser) parseExpression(bindingPower int) (ASTNode, error) {
  135. var err error
  136. leftToken := p.lookaheadToken(0)
  137. p.advance()
  138. leftNode, err := p.nud(leftToken)
  139. if err != nil {
  140. return ASTNode{}, err
  141. }
  142. currentToken := p.current()
  143. for bindingPower < bindingPowers[currentToken] {
  144. p.advance()
  145. leftNode, err = p.led(currentToken, leftNode)
  146. if err != nil {
  147. return ASTNode{}, err
  148. }
  149. currentToken = p.current()
  150. }
  151. return leftNode, nil
  152. }
  153. func (p *Parser) parseIndexExpression() (ASTNode, error) {
  154. if p.lookahead(0) == tColon || p.lookahead(1) == tColon {
  155. return p.parseSliceExpression()
  156. }
  157. indexStr := p.lookaheadToken(0).value
  158. parsedInt, err := strconv.Atoi(indexStr)
  159. if err != nil {
  160. return ASTNode{}, err
  161. }
  162. indexNode := ASTNode{nodeType: ASTIndex, value: parsedInt}
  163. p.advance()
  164. if err := p.match(tRbracket); err != nil {
  165. return ASTNode{}, err
  166. }
  167. return indexNode, nil
  168. }
  169. func (p *Parser) parseSliceExpression() (ASTNode, error) {
  170. parts := []*int{nil, nil, nil}
  171. index := 0
  172. current := p.current()
  173. for current != tRbracket && index < 3 {
  174. if current == tColon {
  175. index++
  176. p.advance()
  177. } else if current == tNumber {
  178. parsedInt, err := strconv.Atoi(p.lookaheadToken(0).value)
  179. if err != nil {
  180. return ASTNode{}, err
  181. }
  182. parts[index] = &parsedInt
  183. p.advance()
  184. } else {
  185. return ASTNode{}, p.syntaxError(
  186. "Expected tColon or tNumber" + ", received: " + p.current().String())
  187. }
  188. current = p.current()
  189. }
  190. if err := p.match(tRbracket); err != nil {
  191. return ASTNode{}, err
  192. }
  193. return ASTNode{
  194. nodeType: ASTSlice,
  195. value: parts,
  196. }, nil
  197. }
  198. func (p *Parser) match(tokenType tokType) error {
  199. if p.current() == tokenType {
  200. p.advance()
  201. return nil
  202. }
  203. return p.syntaxError("Expected " + tokenType.String() + ", received: " + p.current().String())
  204. }
  205. func (p *Parser) led(tokenType tokType, node ASTNode) (ASTNode, error) {
  206. switch tokenType {
  207. case tDot:
  208. if p.current() != tStar {
  209. right, err := p.parseDotRHS(bindingPowers[tDot])
  210. return ASTNode{
  211. nodeType: ASTSubexpression,
  212. children: []ASTNode{node, right},
  213. }, err
  214. }
  215. p.advance()
  216. right, err := p.parseProjectionRHS(bindingPowers[tDot])
  217. return ASTNode{
  218. nodeType: ASTValueProjection,
  219. children: []ASTNode{node, right},
  220. }, err
  221. case tPipe:
  222. right, err := p.parseExpression(bindingPowers[tPipe])
  223. return ASTNode{nodeType: ASTPipe, children: []ASTNode{node, right}}, err
  224. case tOr:
  225. right, err := p.parseExpression(bindingPowers[tOr])
  226. return ASTNode{nodeType: ASTOrExpression, children: []ASTNode{node, right}}, err
  227. case tAnd:
  228. right, err := p.parseExpression(bindingPowers[tAnd])
  229. return ASTNode{nodeType: ASTAndExpression, children: []ASTNode{node, right}}, err
  230. case tLparen:
  231. name := node.value
  232. var args []ASTNode
  233. for p.current() != tRparen {
  234. expression, err := p.parseExpression(0)
  235. if err != nil {
  236. return ASTNode{}, err
  237. }
  238. if p.current() == tComma {
  239. if err := p.match(tComma); err != nil {
  240. return ASTNode{}, err
  241. }
  242. }
  243. args = append(args, expression)
  244. }
  245. if err := p.match(tRparen); err != nil {
  246. return ASTNode{}, err
  247. }
  248. return ASTNode{
  249. nodeType: ASTFunctionExpression,
  250. value: name,
  251. children: args,
  252. }, nil
  253. case tFilter:
  254. return p.parseFilter(node)
  255. case tFlatten:
  256. left := ASTNode{nodeType: ASTFlatten, children: []ASTNode{node}}
  257. right, err := p.parseProjectionRHS(bindingPowers[tFlatten])
  258. return ASTNode{
  259. nodeType: ASTProjection,
  260. children: []ASTNode{left, right},
  261. }, err
  262. case tEQ, tNE, tGT, tGTE, tLT, tLTE:
  263. right, err := p.parseExpression(bindingPowers[tokenType])
  264. if err != nil {
  265. return ASTNode{}, err
  266. }
  267. return ASTNode{
  268. nodeType: ASTComparator,
  269. value: tokenType,
  270. children: []ASTNode{node, right},
  271. }, nil
  272. case tLbracket:
  273. tokenType := p.current()
  274. var right ASTNode
  275. var err error
  276. if tokenType == tNumber || tokenType == tColon {
  277. right, err = p.parseIndexExpression()
  278. if err != nil {
  279. return ASTNode{}, err
  280. }
  281. return p.projectIfSlice(node, right)
  282. }
  283. // Otherwise this is a projection.
  284. if err := p.match(tStar); err != nil {
  285. return ASTNode{}, err
  286. }
  287. if err := p.match(tRbracket); err != nil {
  288. return ASTNode{}, err
  289. }
  290. right, err = p.parseProjectionRHS(bindingPowers[tStar])
  291. if err != nil {
  292. return ASTNode{}, err
  293. }
  294. return ASTNode{
  295. nodeType: ASTProjection,
  296. children: []ASTNode{node, right},
  297. }, nil
  298. }
  299. return ASTNode{}, p.syntaxError("Unexpected token: " + tokenType.String())
  300. }
  301. func (p *Parser) nud(token token) (ASTNode, error) {
  302. switch token.tokenType {
  303. case tJSONLiteral:
  304. var parsed interface{}
  305. err := json.Unmarshal([]byte(token.value), &parsed)
  306. if err != nil {
  307. return ASTNode{}, err
  308. }
  309. return ASTNode{nodeType: ASTLiteral, value: parsed}, nil
  310. case tStringLiteral:
  311. return ASTNode{nodeType: ASTLiteral, value: token.value}, nil
  312. case tUnquotedIdentifier:
  313. return ASTNode{
  314. nodeType: ASTField,
  315. value: token.value,
  316. }, nil
  317. case tQuotedIdentifier:
  318. node := ASTNode{nodeType: ASTField, value: token.value}
  319. if p.current() == tLparen {
  320. return ASTNode{}, p.syntaxErrorToken("Can't have quoted identifier as function name.", token)
  321. }
  322. return node, nil
  323. case tStar:
  324. left := ASTNode{nodeType: ASTIdentity}
  325. var right ASTNode
  326. var err error
  327. if p.current() == tRbracket {
  328. right = ASTNode{nodeType: ASTIdentity}
  329. } else {
  330. right, err = p.parseProjectionRHS(bindingPowers[tStar])
  331. }
  332. return ASTNode{nodeType: ASTValueProjection, children: []ASTNode{left, right}}, err
  333. case tFilter:
  334. return p.parseFilter(ASTNode{nodeType: ASTIdentity})
  335. case tLbrace:
  336. return p.parseMultiSelectHash()
  337. case tFlatten:
  338. left := ASTNode{
  339. nodeType: ASTFlatten,
  340. children: []ASTNode{{nodeType: ASTIdentity}},
  341. }
  342. right, err := p.parseProjectionRHS(bindingPowers[tFlatten])
  343. if err != nil {
  344. return ASTNode{}, err
  345. }
  346. return ASTNode{nodeType: ASTProjection, children: []ASTNode{left, right}}, nil
  347. case tLbracket:
  348. tokenType := p.current()
  349. //var right ASTNode
  350. if tokenType == tNumber || tokenType == tColon {
  351. right, err := p.parseIndexExpression()
  352. if err != nil {
  353. return ASTNode{}, nil
  354. }
  355. return p.projectIfSlice(ASTNode{nodeType: ASTIdentity}, right)
  356. } else if tokenType == tStar && p.lookahead(1) == tRbracket {
  357. p.advance()
  358. p.advance()
  359. right, err := p.parseProjectionRHS(bindingPowers[tStar])
  360. if err != nil {
  361. return ASTNode{}, err
  362. }
  363. return ASTNode{
  364. nodeType: ASTProjection,
  365. children: []ASTNode{{nodeType: ASTIdentity}, right},
  366. }, nil
  367. } else {
  368. return p.parseMultiSelectList()
  369. }
  370. case tCurrent:
  371. return ASTNode{nodeType: ASTCurrentNode}, nil
  372. case tExpref:
  373. expression, err := p.parseExpression(bindingPowers[tExpref])
  374. if err != nil {
  375. return ASTNode{}, err
  376. }
  377. return ASTNode{nodeType: ASTExpRef, children: []ASTNode{expression}}, nil
  378. case tNot:
  379. expression, err := p.parseExpression(bindingPowers[tNot])
  380. if err != nil {
  381. return ASTNode{}, err
  382. }
  383. return ASTNode{nodeType: ASTNotExpression, children: []ASTNode{expression}}, nil
  384. case tLparen:
  385. expression, err := p.parseExpression(0)
  386. if err != nil {
  387. return ASTNode{}, err
  388. }
  389. if err := p.match(tRparen); err != nil {
  390. return ASTNode{}, err
  391. }
  392. return expression, nil
  393. case tEOF:
  394. return ASTNode{}, p.syntaxErrorToken("Incomplete expression", token)
  395. }
  396. return ASTNode{}, p.syntaxErrorToken("Invalid token: "+token.tokenType.String(), token)
  397. }
  398. func (p *Parser) parseMultiSelectList() (ASTNode, error) {
  399. var expressions []ASTNode
  400. for {
  401. expression, err := p.parseExpression(0)
  402. if err != nil {
  403. return ASTNode{}, err
  404. }
  405. expressions = append(expressions, expression)
  406. if p.current() == tRbracket {
  407. break
  408. }
  409. err = p.match(tComma)
  410. if err != nil {
  411. return ASTNode{}, err
  412. }
  413. }
  414. err := p.match(tRbracket)
  415. if err != nil {
  416. return ASTNode{}, err
  417. }
  418. return ASTNode{
  419. nodeType: ASTMultiSelectList,
  420. children: expressions,
  421. }, nil
  422. }
  423. func (p *Parser) parseMultiSelectHash() (ASTNode, error) {
  424. var children []ASTNode
  425. for {
  426. keyToken := p.lookaheadToken(0)
  427. if err := p.match(tUnquotedIdentifier); err != nil {
  428. if err := p.match(tQuotedIdentifier); err != nil {
  429. return ASTNode{}, p.syntaxError("Expected tQuotedIdentifier or tUnquotedIdentifier")
  430. }
  431. }
  432. keyName := keyToken.value
  433. err := p.match(tColon)
  434. if err != nil {
  435. return ASTNode{}, err
  436. }
  437. value, err := p.parseExpression(0)
  438. if err != nil {
  439. return ASTNode{}, err
  440. }
  441. node := ASTNode{
  442. nodeType: ASTKeyValPair,
  443. value: keyName,
  444. children: []ASTNode{value},
  445. }
  446. children = append(children, node)
  447. if p.current() == tComma {
  448. err := p.match(tComma)
  449. if err != nil {
  450. return ASTNode{}, nil
  451. }
  452. } else if p.current() == tRbrace {
  453. err := p.match(tRbrace)
  454. if err != nil {
  455. return ASTNode{}, nil
  456. }
  457. break
  458. }
  459. }
  460. return ASTNode{
  461. nodeType: ASTMultiSelectHash,
  462. children: children,
  463. }, nil
  464. }
  465. func (p *Parser) projectIfSlice(left ASTNode, right ASTNode) (ASTNode, error) {
  466. indexExpr := ASTNode{
  467. nodeType: ASTIndexExpression,
  468. children: []ASTNode{left, right},
  469. }
  470. if right.nodeType == ASTSlice {
  471. right, err := p.parseProjectionRHS(bindingPowers[tStar])
  472. return ASTNode{
  473. nodeType: ASTProjection,
  474. children: []ASTNode{indexExpr, right},
  475. }, err
  476. }
  477. return indexExpr, nil
  478. }
  479. func (p *Parser) parseFilter(node ASTNode) (ASTNode, error) {
  480. var right, condition ASTNode
  481. var err error
  482. condition, err = p.parseExpression(0)
  483. if err != nil {
  484. return ASTNode{}, err
  485. }
  486. if err := p.match(tRbracket); err != nil {
  487. return ASTNode{}, err
  488. }
  489. if p.current() == tFlatten {
  490. right = ASTNode{nodeType: ASTIdentity}
  491. } else {
  492. right, err = p.parseProjectionRHS(bindingPowers[tFilter])
  493. if err != nil {
  494. return ASTNode{}, err
  495. }
  496. }
  497. return ASTNode{
  498. nodeType: ASTFilterProjection,
  499. children: []ASTNode{node, right, condition},
  500. }, nil
  501. }
  502. func (p *Parser) parseDotRHS(bindingPower int) (ASTNode, error) {
  503. lookahead := p.current()
  504. if tokensOneOf([]tokType{tQuotedIdentifier, tUnquotedIdentifier, tStar}, lookahead) {
  505. return p.parseExpression(bindingPower)
  506. } else if lookahead == tLbracket {
  507. if err := p.match(tLbracket); err != nil {
  508. return ASTNode{}, err
  509. }
  510. return p.parseMultiSelectList()
  511. } else if lookahead == tLbrace {
  512. if err := p.match(tLbrace); err != nil {
  513. return ASTNode{}, err
  514. }
  515. return p.parseMultiSelectHash()
  516. }
  517. return ASTNode{}, p.syntaxError("Expected identifier, lbracket, or lbrace")
  518. }
  519. func (p *Parser) parseProjectionRHS(bindingPower int) (ASTNode, error) {
  520. current := p.current()
  521. if bindingPowers[current] < 10 {
  522. return ASTNode{nodeType: ASTIdentity}, nil
  523. } else if current == tLbracket {
  524. return p.parseExpression(bindingPower)
  525. } else if current == tFilter {
  526. return p.parseExpression(bindingPower)
  527. } else if current == tDot {
  528. err := p.match(tDot)
  529. if err != nil {
  530. return ASTNode{}, err
  531. }
  532. return p.parseDotRHS(bindingPower)
  533. } else {
  534. return ASTNode{}, p.syntaxError("Error")
  535. }
  536. }
  537. func (p *Parser) lookahead(number int) tokType {
  538. return p.lookaheadToken(number).tokenType
  539. }
  540. func (p *Parser) current() tokType {
  541. return p.lookahead(0)
  542. }
  543. func (p *Parser) lookaheadToken(number int) token {
  544. return p.tokens[p.index+number]
  545. }
  546. func (p *Parser) advance() {
  547. p.index++
  548. }
  549. func tokensOneOf(elements []tokType, token tokType) bool {
  550. for _, elem := range elements {
  551. if elem == token {
  552. return true
  553. }
  554. }
  555. return false
  556. }
  557. func (p *Parser) syntaxError(msg string) SyntaxError {
  558. return SyntaxError{
  559. msg: msg,
  560. Expression: p.expression,
  561. Offset: p.lookaheadToken(0).position,
  562. }
  563. }
  564. // Create a SyntaxError based on the provided token.
  565. // This differs from syntaxError() which creates a SyntaxError
  566. // based on the current lookahead token.
  567. func (p *Parser) syntaxErrorToken(msg string, t token) SyntaxError {
  568. return SyntaxError{
  569. msg: msg,
  570. Expression: p.expression,
  571. Offset: t.position,
  572. }
  573. }