decoder.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356
  1. """Implementation of JSONDecoder
  2. """
  3. import re
  4. from json import scanner
  5. try:
  6. from _json import scanstring as c_scanstring
  7. except ImportError:
  8. c_scanstring = None
  9. __all__ = ['JSONDecoder', 'JSONDecodeError']
  10. FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL
  11. NaN = float('nan')
  12. PosInf = float('inf')
  13. NegInf = float('-inf')
  14. class JSONDecodeError(ValueError):
  15. """Subclass of ValueError with the following additional properties:
  16. msg: The unformatted error message
  17. doc: The JSON document being parsed
  18. pos: The start index of doc where parsing failed
  19. lineno: The line corresponding to pos
  20. colno: The column corresponding to pos
  21. """
  22. # Note that this exception is used from _json
  23. def __init__(self, msg, doc, pos):
  24. lineno = doc.count('\n', 0, pos) + 1
  25. colno = pos - doc.rfind('\n', 0, pos)
  26. errmsg = '%s: line %d column %d (char %d)' % (msg, lineno, colno, pos)
  27. ValueError.__init__(self, errmsg)
  28. self.msg = msg
  29. self.doc = doc
  30. self.pos = pos
  31. self.lineno = lineno
  32. self.colno = colno
  33. def __reduce__(self):
  34. return self.__class__, (self.msg, self.doc, self.pos)
  35. _CONSTANTS = {
  36. '-Infinity': NegInf,
  37. 'Infinity': PosInf,
  38. 'NaN': NaN,
  39. }
  40. STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS)
  41. BACKSLASH = {
  42. '"': '"', '\\': '\\', '/': '/',
  43. 'b': '\b', 'f': '\f', 'n': '\n', 'r': '\r', 't': '\t',
  44. }
  45. def _decode_uXXXX(s, pos):
  46. esc = s[pos + 1:pos + 5]
  47. if len(esc) == 4 and esc[1] not in 'xX':
  48. try:
  49. return int(esc, 16)
  50. except ValueError:
  51. pass
  52. msg = "Invalid \\uXXXX escape"
  53. raise JSONDecodeError(msg, s, pos)
  54. def py_scanstring(s, end, strict=True,
  55. _b=BACKSLASH, _m=STRINGCHUNK.match):
  56. """Scan the string s for a JSON string. End is the index of the
  57. character in s after the quote that started the JSON string.
  58. Unescapes all valid JSON string escape sequences and raises ValueError
  59. on attempt to decode an invalid string. If strict is False then literal
  60. control characters are allowed in the string.
  61. Returns a tuple of the decoded string and the index of the character in s
  62. after the end quote."""
  63. chunks = []
  64. _append = chunks.append
  65. begin = end - 1
  66. while 1:
  67. chunk = _m(s, end)
  68. if chunk is None:
  69. raise JSONDecodeError("Unterminated string starting at", s, begin)
  70. end = chunk.end()
  71. content, terminator = chunk.groups()
  72. # Content is contains zero or more unescaped string characters
  73. if content:
  74. _append(content)
  75. # Terminator is the end of string, a literal control character,
  76. # or a backslash denoting that an escape sequence follows
  77. if terminator == '"':
  78. break
  79. elif terminator != '\\':
  80. if strict:
  81. #msg = "Invalid control character %r at" % (terminator,)
  82. msg = "Invalid control character {0!r} at".format(terminator)
  83. raise JSONDecodeError(msg, s, end)
  84. else:
  85. _append(terminator)
  86. continue
  87. try:
  88. esc = s[end]
  89. except IndexError:
  90. raise JSONDecodeError("Unterminated string starting at",
  91. s, begin) from None
  92. # If not a unicode escape sequence, must be in the lookup table
  93. if esc != 'u':
  94. try:
  95. char = _b[esc]
  96. except KeyError:
  97. msg = "Invalid \\escape: {0!r}".format(esc)
  98. raise JSONDecodeError(msg, s, end)
  99. end += 1
  100. else:
  101. uni = _decode_uXXXX(s, end)
  102. end += 5
  103. if 0xd800 <= uni <= 0xdbff and s[end:end + 2] == '\\u':
  104. uni2 = _decode_uXXXX(s, end + 1)
  105. if 0xdc00 <= uni2 <= 0xdfff:
  106. uni = 0x10000 + (((uni - 0xd800) << 10) | (uni2 - 0xdc00))
  107. end += 6
  108. char = chr(uni)
  109. _append(char)
  110. return ''.join(chunks), end
  111. # Use speedup if available
  112. scanstring = c_scanstring or py_scanstring
  113. WHITESPACE = re.compile(r'[ \t\n\r]*', FLAGS)
  114. WHITESPACE_STR = ' \t\n\r'
  115. def JSONObject(s_and_end, strict, scan_once, object_hook, object_pairs_hook,
  116. memo=None, _w=WHITESPACE.match, _ws=WHITESPACE_STR):
  117. s, end = s_and_end
  118. pairs = []
  119. pairs_append = pairs.append
  120. # Backwards compatibility
  121. if memo is None:
  122. memo = {}
  123. memo_get = memo.setdefault
  124. # Use a slice to prevent IndexError from being raised, the following
  125. # check will raise a more specific ValueError if the string is empty
  126. nextchar = s[end:end + 1]
  127. # Normally we expect nextchar == '"'
  128. if nextchar != '"':
  129. if nextchar in _ws:
  130. end = _w(s, end).end()
  131. nextchar = s[end:end + 1]
  132. # Trivial empty object
  133. if nextchar == '}':
  134. if object_pairs_hook is not None:
  135. result = object_pairs_hook(pairs)
  136. return result, end + 1
  137. pairs = {}
  138. if object_hook is not None:
  139. pairs = object_hook(pairs)
  140. return pairs, end + 1
  141. elif nextchar != '"':
  142. raise JSONDecodeError(
  143. "Expecting property name enclosed in double quotes", s, end)
  144. end += 1
  145. while True:
  146. key, end = scanstring(s, end, strict)
  147. key = memo_get(key, key)
  148. # To skip some function call overhead we optimize the fast paths where
  149. # the JSON key separator is ": " or just ":".
  150. if s[end:end + 1] != ':':
  151. end = _w(s, end).end()
  152. if s[end:end + 1] != ':':
  153. raise JSONDecodeError("Expecting ':' delimiter", s, end)
  154. end += 1
  155. try:
  156. if s[end] in _ws:
  157. end += 1
  158. if s[end] in _ws:
  159. end = _w(s, end + 1).end()
  160. except IndexError:
  161. pass
  162. try:
  163. value, end = scan_once(s, end)
  164. except StopIteration as err:
  165. raise JSONDecodeError("Expecting value", s, err.value) from None
  166. pairs_append((key, value))
  167. try:
  168. nextchar = s[end]
  169. if nextchar in _ws:
  170. end = _w(s, end + 1).end()
  171. nextchar = s[end]
  172. except IndexError:
  173. nextchar = ''
  174. end += 1
  175. if nextchar == '}':
  176. break
  177. elif nextchar != ',':
  178. raise JSONDecodeError("Expecting ',' delimiter", s, end - 1)
  179. end = _w(s, end).end()
  180. nextchar = s[end:end + 1]
  181. end += 1
  182. if nextchar != '"':
  183. raise JSONDecodeError(
  184. "Expecting property name enclosed in double quotes", s, end - 1)
  185. if object_pairs_hook is not None:
  186. result = object_pairs_hook(pairs)
  187. return result, end
  188. pairs = dict(pairs)
  189. if object_hook is not None:
  190. pairs = object_hook(pairs)
  191. return pairs, end
  192. def JSONArray(s_and_end, scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR):
  193. s, end = s_and_end
  194. values = []
  195. nextchar = s[end:end + 1]
  196. if nextchar in _ws:
  197. end = _w(s, end + 1).end()
  198. nextchar = s[end:end + 1]
  199. # Look-ahead for trivial empty array
  200. if nextchar == ']':
  201. return values, end + 1
  202. _append = values.append
  203. while True:
  204. try:
  205. value, end = scan_once(s, end)
  206. except StopIteration as err:
  207. raise JSONDecodeError("Expecting value", s, err.value) from None
  208. _append(value)
  209. nextchar = s[end:end + 1]
  210. if nextchar in _ws:
  211. end = _w(s, end + 1).end()
  212. nextchar = s[end:end + 1]
  213. end += 1
  214. if nextchar == ']':
  215. break
  216. elif nextchar != ',':
  217. raise JSONDecodeError("Expecting ',' delimiter", s, end - 1)
  218. try:
  219. if s[end] in _ws:
  220. end += 1
  221. if s[end] in _ws:
  222. end = _w(s, end + 1).end()
  223. except IndexError:
  224. pass
  225. return values, end
  226. class JSONDecoder(object):
  227. """Simple JSON <http://json.org> decoder
  228. Performs the following translations in decoding by default:
  229. +---------------+-------------------+
  230. | JSON | Python |
  231. +===============+===================+
  232. | object | dict |
  233. +---------------+-------------------+
  234. | array | list |
  235. +---------------+-------------------+
  236. | string | str |
  237. +---------------+-------------------+
  238. | number (int) | int |
  239. +---------------+-------------------+
  240. | number (real) | float |
  241. +---------------+-------------------+
  242. | true | True |
  243. +---------------+-------------------+
  244. | false | False |
  245. +---------------+-------------------+
  246. | null | None |
  247. +---------------+-------------------+
  248. It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as
  249. their corresponding ``float`` values, which is outside the JSON spec.
  250. """
  251. def __init__(self, *, object_hook=None, parse_float=None,
  252. parse_int=None, parse_constant=None, strict=True,
  253. object_pairs_hook=None):
  254. """``object_hook``, if specified, will be called with the result
  255. of every JSON object decoded and its return value will be used in
  256. place of the given ``dict``. This can be used to provide custom
  257. deserializations (e.g. to support JSON-RPC class hinting).
  258. ``object_pairs_hook``, if specified will be called with the result of
  259. every JSON object decoded with an ordered list of pairs. The return
  260. value of ``object_pairs_hook`` will be used instead of the ``dict``.
  261. This feature can be used to implement custom decoders.
  262. If ``object_hook`` is also defined, the ``object_pairs_hook`` takes
  263. priority.
  264. ``parse_float``, if specified, will be called with the string
  265. of every JSON float to be decoded. By default this is equivalent to
  266. float(num_str). This can be used to use another datatype or parser
  267. for JSON floats (e.g. decimal.Decimal).
  268. ``parse_int``, if specified, will be called with the string
  269. of every JSON int to be decoded. By default this is equivalent to
  270. int(num_str). This can be used to use another datatype or parser
  271. for JSON integers (e.g. float).
  272. ``parse_constant``, if specified, will be called with one of the
  273. following strings: -Infinity, Infinity, NaN.
  274. This can be used to raise an exception if invalid JSON numbers
  275. are encountered.
  276. If ``strict`` is false (true is the default), then control
  277. characters will be allowed inside strings. Control characters in
  278. this context are those with character codes in the 0-31 range,
  279. including ``'\\t'`` (tab), ``'\\n'``, ``'\\r'`` and ``'\\0'``.
  280. """
  281. self.object_hook = object_hook
  282. self.parse_float = parse_float or float
  283. self.parse_int = parse_int or int
  284. self.parse_constant = parse_constant or _CONSTANTS.__getitem__
  285. self.strict = strict
  286. self.object_pairs_hook = object_pairs_hook
  287. self.parse_object = JSONObject
  288. self.parse_array = JSONArray
  289. self.parse_string = scanstring
  290. self.memo = {}
  291. self.scan_once = scanner.make_scanner(self)
  292. def decode(self, s, _w=WHITESPACE.match):
  293. """Return the Python representation of ``s`` (a ``str`` instance
  294. containing a JSON document).
  295. """
  296. obj, end = self.raw_decode(s, idx=_w(s, 0).end())
  297. end = _w(s, end).end()
  298. if end != len(s):
  299. raise JSONDecodeError("Extra data", s, end)
  300. return obj
  301. def raw_decode(self, s, idx=0):
  302. """Decode a JSON document from ``s`` (a ``str`` beginning with
  303. a JSON document) and return a 2-tuple of the Python
  304. representation and the index in ``s`` where the document ended.
  305. This can be used to decode a JSON document from a string that may
  306. have extraneous data at the end.
  307. """
  308. try:
  309. obj, end = self.scan_once(s, idx)
  310. except StopIteration as err:
  311. raise JSONDecodeError("Expecting value", s, err.value) from None
  312. return obj, end