parser.go 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477
  1. // Copyright 2015 Unknwon
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License"): you may
  4. // not use this file except in compliance with the License. You may obtain
  5. // a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  11. // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  12. // License for the specific language governing permissions and limitations
  13. // under the License.
  14. package ini
  15. import (
  16. "bufio"
  17. "bytes"
  18. "fmt"
  19. "io"
  20. "regexp"
  21. "strconv"
  22. "strings"
  23. "unicode"
  24. )
  25. var pythonMultiline = regexp.MustCompile("^(\\s+)([^\n]+)")
  26. type tokenType int
  27. const (
  28. _TOKEN_INVALID tokenType = iota
  29. _TOKEN_COMMENT
  30. _TOKEN_SECTION
  31. _TOKEN_KEY
  32. )
  33. type parser struct {
  34. buf *bufio.Reader
  35. isEOF bool
  36. count int
  37. comment *bytes.Buffer
  38. }
  39. func newParser(r io.Reader) *parser {
  40. return &parser{
  41. buf: bufio.NewReader(r),
  42. count: 1,
  43. comment: &bytes.Buffer{},
  44. }
  45. }
  46. // BOM handles header of UTF-8, UTF-16 LE and UTF-16 BE's BOM format.
  47. // http://en.wikipedia.org/wiki/Byte_order_mark#Representations_of_byte_order_marks_by_encoding
  48. func (p *parser) BOM() error {
  49. mask, err := p.buf.Peek(2)
  50. if err != nil && err != io.EOF {
  51. return err
  52. } else if len(mask) < 2 {
  53. return nil
  54. }
  55. switch {
  56. case mask[0] == 254 && mask[1] == 255:
  57. fallthrough
  58. case mask[0] == 255 && mask[1] == 254:
  59. p.buf.Read(mask)
  60. case mask[0] == 239 && mask[1] == 187:
  61. mask, err := p.buf.Peek(3)
  62. if err != nil && err != io.EOF {
  63. return err
  64. } else if len(mask) < 3 {
  65. return nil
  66. }
  67. if mask[2] == 191 {
  68. p.buf.Read(mask)
  69. }
  70. }
  71. return nil
  72. }
  73. func (p *parser) readUntil(delim byte) ([]byte, error) {
  74. data, err := p.buf.ReadBytes(delim)
  75. if err != nil {
  76. if err == io.EOF {
  77. p.isEOF = true
  78. } else {
  79. return nil, err
  80. }
  81. }
  82. return data, nil
  83. }
  84. func cleanComment(in []byte) ([]byte, bool) {
  85. i := bytes.IndexAny(in, "#;")
  86. if i == -1 {
  87. return nil, false
  88. }
  89. return in[i:], true
  90. }
  91. func readKeyName(in []byte) (string, int, error) {
  92. line := string(in)
  93. // Check if key name surrounded by quotes.
  94. var keyQuote string
  95. if line[0] == '"' {
  96. if len(line) > 6 && string(line[0:3]) == `"""` {
  97. keyQuote = `"""`
  98. } else {
  99. keyQuote = `"`
  100. }
  101. } else if line[0] == '`' {
  102. keyQuote = "`"
  103. }
  104. // Get out key name
  105. endIdx := -1
  106. if len(keyQuote) > 0 {
  107. startIdx := len(keyQuote)
  108. // FIXME: fail case -> """"""name"""=value
  109. pos := strings.Index(line[startIdx:], keyQuote)
  110. if pos == -1 {
  111. return "", -1, fmt.Errorf("missing closing key quote: %s", line)
  112. }
  113. pos += startIdx
  114. // Find key-value delimiter
  115. i := strings.IndexAny(line[pos+startIdx:], "=:")
  116. if i < 0 {
  117. return "", -1, ErrDelimiterNotFound{line}
  118. }
  119. endIdx = pos + i
  120. return strings.TrimSpace(line[startIdx:pos]), endIdx + startIdx + 1, nil
  121. }
  122. endIdx = strings.IndexAny(line, "=:")
  123. if endIdx < 0 {
  124. return "", -1, ErrDelimiterNotFound{line}
  125. }
  126. return strings.TrimSpace(line[0:endIdx]), endIdx + 1, nil
  127. }
  128. func (p *parser) readMultilines(line, val, valQuote string) (string, error) {
  129. for {
  130. data, err := p.readUntil('\n')
  131. if err != nil {
  132. return "", err
  133. }
  134. next := string(data)
  135. pos := strings.LastIndex(next, valQuote)
  136. if pos > -1 {
  137. val += next[:pos]
  138. comment, has := cleanComment([]byte(next[pos:]))
  139. if has {
  140. p.comment.Write(bytes.TrimSpace(comment))
  141. }
  142. break
  143. }
  144. val += next
  145. if p.isEOF {
  146. return "", fmt.Errorf("missing closing key quote from '%s' to '%s'", line, next)
  147. }
  148. }
  149. return val, nil
  150. }
  151. func (p *parser) readContinuationLines(val string) (string, error) {
  152. for {
  153. data, err := p.readUntil('\n')
  154. if err != nil {
  155. return "", err
  156. }
  157. next := strings.TrimSpace(string(data))
  158. if len(next) == 0 {
  159. break
  160. }
  161. val += next
  162. if val[len(val)-1] != '\\' {
  163. break
  164. }
  165. val = val[:len(val)-1]
  166. }
  167. return val, nil
  168. }
  169. // hasSurroundedQuote check if and only if the first and last characters
  170. // are quotes \" or \'.
  171. // It returns false if any other parts also contain same kind of quotes.
  172. func hasSurroundedQuote(in string, quote byte) bool {
  173. return len(in) >= 2 && in[0] == quote && in[len(in)-1] == quote &&
  174. strings.IndexByte(in[1:], quote) == len(in)-2
  175. }
  176. func (p *parser) readValue(in []byte,
  177. parserBufferSize int,
  178. ignoreContinuation, ignoreInlineComment, unescapeValueDoubleQuotes, unescapeValueCommentSymbols, allowPythonMultilines bool) (string, error) {
  179. line := strings.TrimLeftFunc(string(in), unicode.IsSpace)
  180. if len(line) == 0 {
  181. return "", nil
  182. }
  183. var valQuote string
  184. if len(line) > 3 && string(line[0:3]) == `"""` {
  185. valQuote = `"""`
  186. } else if line[0] == '`' {
  187. valQuote = "`"
  188. } else if unescapeValueDoubleQuotes && line[0] == '"' {
  189. valQuote = `"`
  190. }
  191. if len(valQuote) > 0 {
  192. startIdx := len(valQuote)
  193. pos := strings.LastIndex(line[startIdx:], valQuote)
  194. // Check for multi-line value
  195. if pos == -1 {
  196. return p.readMultilines(line, line[startIdx:], valQuote)
  197. }
  198. if unescapeValueDoubleQuotes && valQuote == `"` {
  199. return strings.Replace(line[startIdx:pos+startIdx], `\"`, `"`, -1), nil
  200. }
  201. return line[startIdx : pos+startIdx], nil
  202. }
  203. lastChar := line[len(line)-1]
  204. // Won't be able to reach here if value only contains whitespace
  205. line = strings.TrimSpace(line)
  206. trimmedLastChar := line[len(line)-1]
  207. // Check continuation lines when desired
  208. if !ignoreContinuation && trimmedLastChar == '\\' {
  209. return p.readContinuationLines(line[:len(line)-1])
  210. }
  211. // Check if ignore inline comment
  212. if !ignoreInlineComment {
  213. i := strings.IndexAny(line, "#;")
  214. if i > -1 {
  215. p.comment.WriteString(line[i:])
  216. line = strings.TrimSpace(line[:i])
  217. }
  218. }
  219. // Trim single and double quotes
  220. if hasSurroundedQuote(line, '\'') ||
  221. hasSurroundedQuote(line, '"') {
  222. line = line[1 : len(line)-1]
  223. } else if len(valQuote) == 0 && unescapeValueCommentSymbols {
  224. if strings.Contains(line, `\;`) {
  225. line = strings.Replace(line, `\;`, ";", -1)
  226. }
  227. if strings.Contains(line, `\#`) {
  228. line = strings.Replace(line, `\#`, "#", -1)
  229. }
  230. } else if allowPythonMultilines && lastChar == '\n' {
  231. parserBufferPeekResult, _ := p.buf.Peek(parserBufferSize)
  232. peekBuffer := bytes.NewBuffer(parserBufferPeekResult)
  233. identSize := -1
  234. val := line
  235. for {
  236. peekData, peekErr := peekBuffer.ReadBytes('\n')
  237. if peekErr != nil {
  238. if peekErr == io.EOF {
  239. return val, nil
  240. }
  241. return "", peekErr
  242. }
  243. peekMatches := pythonMultiline.FindStringSubmatch(string(peekData))
  244. if len(peekMatches) != 3 {
  245. return val, nil
  246. }
  247. currentIdentSize := len(peekMatches[1])
  248. // NOTE: Return if not a python-ini multi-line value.
  249. if currentIdentSize < 0 {
  250. return val, nil
  251. }
  252. identSize = currentIdentSize
  253. // NOTE: Just advance the parser reader (buffer) in-sync with the peek buffer.
  254. _, err := p.readUntil('\n')
  255. if err != nil {
  256. return "", err
  257. }
  258. val += fmt.Sprintf("\n%s", peekMatches[2])
  259. }
  260. // NOTE: If it was a Python multi-line value,
  261. // return the appended value.
  262. if identSize > 0 {
  263. return val, nil
  264. }
  265. }
  266. return line, nil
  267. }
  268. // parse parses data through an io.Reader.
  269. func (f *File) parse(reader io.Reader) (err error) {
  270. p := newParser(reader)
  271. if err = p.BOM(); err != nil {
  272. return fmt.Errorf("BOM: %v", err)
  273. }
  274. // Ignore error because default section name is never empty string.
  275. name := DEFAULT_SECTION
  276. if f.options.Insensitive {
  277. name = strings.ToLower(DEFAULT_SECTION)
  278. }
  279. section, _ := f.NewSection(name)
  280. // This "last" is not strictly equivalent to "previous one" if current key is not the first nested key
  281. var isLastValueEmpty bool
  282. var lastRegularKey *Key
  283. var line []byte
  284. var inUnparseableSection bool
  285. // NOTE: Iterate and increase `currentPeekSize` until
  286. // the size of the parser buffer is found.
  287. // TODO: When Golang 1.10 is the lowest version supported,
  288. // replace with `parserBufferSize := p.buf.Size()`.
  289. parserBufferSize := 0
  290. // NOTE: Peek 1kb at a time.
  291. currentPeekSize := 1024
  292. if f.options.AllowPythonMultilineValues {
  293. for {
  294. peekBytes, _ := p.buf.Peek(currentPeekSize)
  295. peekBytesLength := len(peekBytes)
  296. if parserBufferSize >= peekBytesLength {
  297. break
  298. }
  299. currentPeekSize *= 2
  300. parserBufferSize = peekBytesLength
  301. }
  302. }
  303. for !p.isEOF {
  304. line, err = p.readUntil('\n')
  305. if err != nil {
  306. return err
  307. }
  308. if f.options.AllowNestedValues &&
  309. isLastValueEmpty && len(line) > 0 {
  310. if line[0] == ' ' || line[0] == '\t' {
  311. lastRegularKey.addNestedValue(string(bytes.TrimSpace(line)))
  312. continue
  313. }
  314. }
  315. line = bytes.TrimLeftFunc(line, unicode.IsSpace)
  316. if len(line) == 0 {
  317. continue
  318. }
  319. // Comments
  320. if line[0] == '#' || line[0] == ';' {
  321. // Note: we do not care ending line break,
  322. // it is needed for adding second line,
  323. // so just clean it once at the end when set to value.
  324. p.comment.Write(line)
  325. continue
  326. }
  327. // Section
  328. if line[0] == '[' {
  329. // Read to the next ']' (TODO: support quoted strings)
  330. // TODO(unknwon): use LastIndexByte when stop supporting Go1.4
  331. closeIdx := bytes.LastIndex(line, []byte("]"))
  332. if closeIdx == -1 {
  333. return fmt.Errorf("unclosed section: %s", line)
  334. }
  335. name := string(line[1:closeIdx])
  336. section, err = f.NewSection(name)
  337. if err != nil {
  338. return err
  339. }
  340. comment, has := cleanComment(line[closeIdx+1:])
  341. if has {
  342. p.comment.Write(comment)
  343. }
  344. section.Comment = strings.TrimSpace(p.comment.String())
  345. // Reset aotu-counter and comments
  346. p.comment.Reset()
  347. p.count = 1
  348. inUnparseableSection = false
  349. for i := range f.options.UnparseableSections {
  350. if f.options.UnparseableSections[i] == name ||
  351. (f.options.Insensitive && strings.ToLower(f.options.UnparseableSections[i]) == strings.ToLower(name)) {
  352. inUnparseableSection = true
  353. continue
  354. }
  355. }
  356. continue
  357. }
  358. if inUnparseableSection {
  359. section.isRawSection = true
  360. section.rawBody += string(line)
  361. continue
  362. }
  363. kname, offset, err := readKeyName(line)
  364. if err != nil {
  365. // Treat as boolean key when desired, and whole line is key name.
  366. if IsErrDelimiterNotFound(err) && f.options.AllowBooleanKeys {
  367. kname, err := p.readValue(line,
  368. parserBufferSize,
  369. f.options.IgnoreContinuation,
  370. f.options.IgnoreInlineComment,
  371. f.options.UnescapeValueDoubleQuotes,
  372. f.options.UnescapeValueCommentSymbols,
  373. f.options.AllowPythonMultilineValues)
  374. if err != nil {
  375. return err
  376. }
  377. key, err := section.NewBooleanKey(kname)
  378. if err != nil {
  379. return err
  380. }
  381. key.Comment = strings.TrimSpace(p.comment.String())
  382. p.comment.Reset()
  383. continue
  384. }
  385. return err
  386. }
  387. // Auto increment.
  388. isAutoIncr := false
  389. if kname == "-" {
  390. isAutoIncr = true
  391. kname = "#" + strconv.Itoa(p.count)
  392. p.count++
  393. }
  394. value, err := p.readValue(line[offset:],
  395. parserBufferSize,
  396. f.options.IgnoreContinuation,
  397. f.options.IgnoreInlineComment,
  398. f.options.UnescapeValueDoubleQuotes,
  399. f.options.UnescapeValueCommentSymbols,
  400. f.options.AllowPythonMultilineValues)
  401. if err != nil {
  402. return err
  403. }
  404. isLastValueEmpty = len(value) == 0
  405. key, err := section.NewKey(kname, value)
  406. if err != nil {
  407. return err
  408. }
  409. key.isAutoIncrement = isAutoIncr
  410. key.Comment = strings.TrimSpace(p.comment.String())
  411. p.comment.Reset()
  412. lastRegularKey = key
  413. }
  414. return nil
  415. }