本站源代码
Du kan inte välja fler än 25 ämnen Ämnen måste starta med en bokstav eller siffra, kan innehålla bindestreck ('-') och vara max 35 tecken långa.

443 lines
11KB

  1. // TOML Parser.
  2. package toml
  3. import (
  4. "errors"
  5. "fmt"
  6. "math"
  7. "reflect"
  8. "regexp"
  9. "strconv"
  10. "strings"
  11. "time"
  12. )
  13. type tomlParser struct {
  14. flowIdx int
  15. flow []token
  16. tree *Tree
  17. currentTable []string
  18. seenTableKeys []string
  19. }
  20. type tomlParserStateFn func() tomlParserStateFn
  21. // Formats and panics an error message based on a token
  22. func (p *tomlParser) raiseError(tok *token, msg string, args ...interface{}) {
  23. panic(tok.Position.String() + ": " + fmt.Sprintf(msg, args...))
  24. }
  25. func (p *tomlParser) run() {
  26. for state := p.parseStart; state != nil; {
  27. state = state()
  28. }
  29. }
  30. func (p *tomlParser) peek() *token {
  31. if p.flowIdx >= len(p.flow) {
  32. return nil
  33. }
  34. return &p.flow[p.flowIdx]
  35. }
  36. func (p *tomlParser) assume(typ tokenType) {
  37. tok := p.getToken()
  38. if tok == nil {
  39. p.raiseError(tok, "was expecting token %s, but token stream is empty", tok)
  40. }
  41. if tok.typ != typ {
  42. p.raiseError(tok, "was expecting token %s, but got %s instead", typ, tok)
  43. }
  44. }
  45. func (p *tomlParser) getToken() *token {
  46. tok := p.peek()
  47. if tok == nil {
  48. return nil
  49. }
  50. p.flowIdx++
  51. return tok
  52. }
  53. func (p *tomlParser) parseStart() tomlParserStateFn {
  54. tok := p.peek()
  55. // end of stream, parsing is finished
  56. if tok == nil {
  57. return nil
  58. }
  59. switch tok.typ {
  60. case tokenDoubleLeftBracket:
  61. return p.parseGroupArray
  62. case tokenLeftBracket:
  63. return p.parseGroup
  64. case tokenKey:
  65. return p.parseAssign
  66. case tokenEOF:
  67. return nil
  68. case tokenError:
  69. p.raiseError(tok, "parsing error: %s", tok.String())
  70. default:
  71. p.raiseError(tok, "unexpected token %s", tok.typ)
  72. }
  73. return nil
  74. }
  75. func (p *tomlParser) parseGroupArray() tomlParserStateFn {
  76. startToken := p.getToken() // discard the [[
  77. key := p.getToken()
  78. if key.typ != tokenKeyGroupArray {
  79. p.raiseError(key, "unexpected token %s, was expecting a table array key", key)
  80. }
  81. // get or create table array element at the indicated part in the path
  82. keys, err := parseKey(key.val)
  83. if err != nil {
  84. p.raiseError(key, "invalid table array key: %s", err)
  85. }
  86. p.tree.createSubTree(keys[:len(keys)-1], startToken.Position) // create parent entries
  87. destTree := p.tree.GetPath(keys)
  88. var array []*Tree
  89. if destTree == nil {
  90. array = make([]*Tree, 0)
  91. } else if target, ok := destTree.([]*Tree); ok && target != nil {
  92. array = destTree.([]*Tree)
  93. } else {
  94. p.raiseError(key, "key %s is already assigned and not of type table array", key)
  95. }
  96. p.currentTable = keys
  97. // add a new tree to the end of the table array
  98. newTree := newTree()
  99. newTree.position = startToken.Position
  100. array = append(array, newTree)
  101. p.tree.SetPath(p.currentTable, array)
  102. // remove all keys that were children of this table array
  103. prefix := key.val + "."
  104. found := false
  105. for ii := 0; ii < len(p.seenTableKeys); {
  106. tableKey := p.seenTableKeys[ii]
  107. if strings.HasPrefix(tableKey, prefix) {
  108. p.seenTableKeys = append(p.seenTableKeys[:ii], p.seenTableKeys[ii+1:]...)
  109. } else {
  110. found = (tableKey == key.val)
  111. ii++
  112. }
  113. }
  114. // keep this key name from use by other kinds of assignments
  115. if !found {
  116. p.seenTableKeys = append(p.seenTableKeys, key.val)
  117. }
  118. // move to next parser state
  119. p.assume(tokenDoubleRightBracket)
  120. return p.parseStart
  121. }
  122. func (p *tomlParser) parseGroup() tomlParserStateFn {
  123. startToken := p.getToken() // discard the [
  124. key := p.getToken()
  125. if key.typ != tokenKeyGroup {
  126. p.raiseError(key, "unexpected token %s, was expecting a table key", key)
  127. }
  128. for _, item := range p.seenTableKeys {
  129. if item == key.val {
  130. p.raiseError(key, "duplicated tables")
  131. }
  132. }
  133. p.seenTableKeys = append(p.seenTableKeys, key.val)
  134. keys, err := parseKey(key.val)
  135. if err != nil {
  136. p.raiseError(key, "invalid table array key: %s", err)
  137. }
  138. if err := p.tree.createSubTree(keys, startToken.Position); err != nil {
  139. p.raiseError(key, "%s", err)
  140. }
  141. p.assume(tokenRightBracket)
  142. p.currentTable = keys
  143. return p.parseStart
  144. }
  145. func (p *tomlParser) parseAssign() tomlParserStateFn {
  146. key := p.getToken()
  147. p.assume(tokenEqual)
  148. parsedKey, err := parseKey(key.val)
  149. if err != nil {
  150. p.raiseError(key, "invalid key: %s", err.Error())
  151. }
  152. value := p.parseRvalue()
  153. var tableKey []string
  154. if len(p.currentTable) > 0 {
  155. tableKey = p.currentTable
  156. } else {
  157. tableKey = []string{}
  158. }
  159. prefixKey := parsedKey[0 : len(parsedKey)-1]
  160. tableKey = append(tableKey, prefixKey...)
  161. // find the table to assign, looking out for arrays of tables
  162. var targetNode *Tree
  163. switch node := p.tree.GetPath(tableKey).(type) {
  164. case []*Tree:
  165. targetNode = node[len(node)-1]
  166. case *Tree:
  167. targetNode = node
  168. case nil:
  169. // create intermediate
  170. if err := p.tree.createSubTree(tableKey, key.Position); err != nil {
  171. p.raiseError(key, "could not create intermediate group: %s", err)
  172. }
  173. targetNode = p.tree.GetPath(tableKey).(*Tree)
  174. default:
  175. p.raiseError(key, "Unknown table type for path: %s",
  176. strings.Join(tableKey, "."))
  177. }
  178. // assign value to the found table
  179. keyVal := parsedKey[len(parsedKey)-1]
  180. localKey := []string{keyVal}
  181. finalKey := append(tableKey, keyVal)
  182. if targetNode.GetPath(localKey) != nil {
  183. p.raiseError(key, "The following key was defined twice: %s",
  184. strings.Join(finalKey, "."))
  185. }
  186. var toInsert interface{}
  187. switch value.(type) {
  188. case *Tree, []*Tree:
  189. toInsert = value
  190. default:
  191. toInsert = &tomlValue{value: value, position: key.Position}
  192. }
  193. targetNode.values[keyVal] = toInsert
  194. return p.parseStart
  195. }
  196. var numberUnderscoreInvalidRegexp *regexp.Regexp
  197. var hexNumberUnderscoreInvalidRegexp *regexp.Regexp
  198. func numberContainsInvalidUnderscore(value string) error {
  199. if numberUnderscoreInvalidRegexp.MatchString(value) {
  200. return errors.New("invalid use of _ in number")
  201. }
  202. return nil
  203. }
  204. func hexNumberContainsInvalidUnderscore(value string) error {
  205. if hexNumberUnderscoreInvalidRegexp.MatchString(value) {
  206. return errors.New("invalid use of _ in hex number")
  207. }
  208. return nil
  209. }
  210. func cleanupNumberToken(value string) string {
  211. cleanedVal := strings.Replace(value, "_", "", -1)
  212. return cleanedVal
  213. }
  214. func (p *tomlParser) parseRvalue() interface{} {
  215. tok := p.getToken()
  216. if tok == nil || tok.typ == tokenEOF {
  217. p.raiseError(tok, "expecting a value")
  218. }
  219. switch tok.typ {
  220. case tokenString:
  221. return tok.val
  222. case tokenTrue:
  223. return true
  224. case tokenFalse:
  225. return false
  226. case tokenInf:
  227. if tok.val[0] == '-' {
  228. return math.Inf(-1)
  229. }
  230. return math.Inf(1)
  231. case tokenNan:
  232. return math.NaN()
  233. case tokenInteger:
  234. cleanedVal := cleanupNumberToken(tok.val)
  235. var err error
  236. var val int64
  237. if len(cleanedVal) >= 3 && cleanedVal[0] == '0' {
  238. switch cleanedVal[1] {
  239. case 'x':
  240. err = hexNumberContainsInvalidUnderscore(tok.val)
  241. if err != nil {
  242. p.raiseError(tok, "%s", err)
  243. }
  244. val, err = strconv.ParseInt(cleanedVal[2:], 16, 64)
  245. case 'o':
  246. err = numberContainsInvalidUnderscore(tok.val)
  247. if err != nil {
  248. p.raiseError(tok, "%s", err)
  249. }
  250. val, err = strconv.ParseInt(cleanedVal[2:], 8, 64)
  251. case 'b':
  252. err = numberContainsInvalidUnderscore(tok.val)
  253. if err != nil {
  254. p.raiseError(tok, "%s", err)
  255. }
  256. val, err = strconv.ParseInt(cleanedVal[2:], 2, 64)
  257. default:
  258. panic("invalid base") // the lexer should catch this first
  259. }
  260. } else {
  261. err = numberContainsInvalidUnderscore(tok.val)
  262. if err != nil {
  263. p.raiseError(tok, "%s", err)
  264. }
  265. val, err = strconv.ParseInt(cleanedVal, 10, 64)
  266. }
  267. if err != nil {
  268. p.raiseError(tok, "%s", err)
  269. }
  270. return val
  271. case tokenFloat:
  272. err := numberContainsInvalidUnderscore(tok.val)
  273. if err != nil {
  274. p.raiseError(tok, "%s", err)
  275. }
  276. cleanedVal := cleanupNumberToken(tok.val)
  277. val, err := strconv.ParseFloat(cleanedVal, 64)
  278. if err != nil {
  279. p.raiseError(tok, "%s", err)
  280. }
  281. return val
  282. case tokenDate:
  283. val, err := time.ParseInLocation(time.RFC3339Nano, tok.val, time.UTC)
  284. if err != nil {
  285. p.raiseError(tok, "%s", err)
  286. }
  287. return val
  288. case tokenLeftBracket:
  289. return p.parseArray()
  290. case tokenLeftCurlyBrace:
  291. return p.parseInlineTable()
  292. case tokenEqual:
  293. p.raiseError(tok, "cannot have multiple equals for the same key")
  294. case tokenError:
  295. p.raiseError(tok, "%s", tok)
  296. }
  297. p.raiseError(tok, "never reached")
  298. return nil
  299. }
  300. func tokenIsComma(t *token) bool {
  301. return t != nil && t.typ == tokenComma
  302. }
  303. func (p *tomlParser) parseInlineTable() *Tree {
  304. tree := newTree()
  305. var previous *token
  306. Loop:
  307. for {
  308. follow := p.peek()
  309. if follow == nil || follow.typ == tokenEOF {
  310. p.raiseError(follow, "unterminated inline table")
  311. }
  312. switch follow.typ {
  313. case tokenRightCurlyBrace:
  314. p.getToken()
  315. break Loop
  316. case tokenKey, tokenInteger, tokenString:
  317. if !tokenIsComma(previous) && previous != nil {
  318. p.raiseError(follow, "comma expected between fields in inline table")
  319. }
  320. key := p.getToken()
  321. p.assume(tokenEqual)
  322. value := p.parseRvalue()
  323. tree.Set(key.val, value)
  324. case tokenComma:
  325. if previous == nil {
  326. p.raiseError(follow, "inline table cannot start with a comma")
  327. }
  328. if tokenIsComma(previous) {
  329. p.raiseError(follow, "need field between two commas in inline table")
  330. }
  331. p.getToken()
  332. default:
  333. p.raiseError(follow, "unexpected token type in inline table: %s", follow.String())
  334. }
  335. previous = follow
  336. }
  337. if tokenIsComma(previous) {
  338. p.raiseError(previous, "trailing comma at the end of inline table")
  339. }
  340. return tree
  341. }
  342. func (p *tomlParser) parseArray() interface{} {
  343. var array []interface{}
  344. arrayType := reflect.TypeOf(nil)
  345. for {
  346. follow := p.peek()
  347. if follow == nil || follow.typ == tokenEOF {
  348. p.raiseError(follow, "unterminated array")
  349. }
  350. if follow.typ == tokenRightBracket {
  351. p.getToken()
  352. break
  353. }
  354. val := p.parseRvalue()
  355. if arrayType == nil {
  356. arrayType = reflect.TypeOf(val)
  357. }
  358. if reflect.TypeOf(val) != arrayType {
  359. p.raiseError(follow, "mixed types in array")
  360. }
  361. array = append(array, val)
  362. follow = p.peek()
  363. if follow == nil || follow.typ == tokenEOF {
  364. p.raiseError(follow, "unterminated array")
  365. }
  366. if follow.typ != tokenRightBracket && follow.typ != tokenComma {
  367. p.raiseError(follow, "missing comma")
  368. }
  369. if follow.typ == tokenComma {
  370. p.getToken()
  371. }
  372. }
  373. // An array of Trees is actually an array of inline
  374. // tables, which is a shorthand for a table array. If the
  375. // array was not converted from []interface{} to []*Tree,
  376. // the two notations would not be equivalent.
  377. if arrayType == reflect.TypeOf(newTree()) {
  378. tomlArray := make([]*Tree, len(array))
  379. for i, v := range array {
  380. tomlArray[i] = v.(*Tree)
  381. }
  382. return tomlArray
  383. }
  384. return array
  385. }
  386. func parseToml(flow []token) *Tree {
  387. result := newTree()
  388. result.position = Position{1, 1}
  389. parser := &tomlParser{
  390. flowIdx: 0,
  391. flow: flow,
  392. tree: result,
  393. currentTable: make([]string, 0),
  394. seenTableKeys: make([]string, 0),
  395. }
  396. parser.run()
  397. return result
  398. }
  399. func init() {
  400. numberUnderscoreInvalidRegexp = regexp.MustCompile(`([^\d]_|_[^\d])|_$|^_`)
  401. hexNumberUnderscoreInvalidRegexp = regexp.MustCompile(`(^0x_)|([^\da-f]_|_[^\da-f])|_$|^_`)
  402. }
上海开阖软件有限公司 沪ICP备12045867号-1