ElementPath.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404
  1. #
  2. # ElementTree
  3. # $Id: ElementPath.py 3375 2008-02-13 08:05:08Z fredrik $
  4. #
  5. # limited xpath support for element trees
  6. #
  7. # history:
  8. # 2003-05-23 fl created
  9. # 2003-05-28 fl added support for // etc
  10. # 2003-08-27 fl fixed parsing of periods in element names
  11. # 2007-09-10 fl new selection engine
  12. # 2007-09-12 fl fixed parent selector
  13. # 2007-09-13 fl added iterfind; changed findall to return a list
  14. # 2007-11-30 fl added namespaces support
  15. # 2009-10-30 fl added child element value filter
  16. #
  17. # Copyright (c) 2003-2009 by Fredrik Lundh. All rights reserved.
  18. #
  19. # fredrik@pythonware.com
  20. # http://www.pythonware.com
  21. #
  22. # --------------------------------------------------------------------
  23. # The ElementTree toolkit is
  24. #
  25. # Copyright (c) 1999-2009 by Fredrik Lundh
  26. #
  27. # By obtaining, using, and/or copying this software and/or its
  28. # associated documentation, you agree that you have read, understood,
  29. # and will comply with the following terms and conditions:
  30. #
  31. # Permission to use, copy, modify, and distribute this software and
  32. # its associated documentation for any purpose and without fee is
  33. # hereby granted, provided that the above copyright notice appears in
  34. # all copies, and that both that copyright notice and this permission
  35. # notice appear in supporting documentation, and that the name of
  36. # Secret Labs AB or the author not be used in advertising or publicity
  37. # pertaining to distribution of the software without specific, written
  38. # prior permission.
  39. #
  40. # SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD
  41. # TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANT-
  42. # ABILITY AND FITNESS. IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR
  43. # BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY
  44. # DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
  45. # WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
  46. # ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
  47. # OF THIS SOFTWARE.
  48. # --------------------------------------------------------------------
  49. # Licensed to PSF under a Contributor Agreement.
  50. # See https://www.python.org/psf/license for licensing details.
  51. ##
  52. # Implementation module for XPath support. There's usually no reason
  53. # to import this module directly; the <b>ElementTree</b> does this for
  54. # you, if needed.
  55. ##
  56. import re
  57. xpath_tokenizer_re = re.compile(
  58. r"("
  59. r"'[^']*'|\"[^\"]*\"|"
  60. r"::|"
  61. r"//?|"
  62. r"\.\.|"
  63. r"\(\)|"
  64. r"[/.*:\[\]\(\)@=])|"
  65. r"((?:\{[^}]+\})?[^/\[\]\(\)@=\s]+)|"
  66. r"\s+"
  67. )
  68. def xpath_tokenizer(pattern, namespaces=None):
  69. default_namespace = namespaces.get('') if namespaces else None
  70. parsing_attribute = False
  71. for token in xpath_tokenizer_re.findall(pattern):
  72. ttype, tag = token
  73. if tag and tag[0] != "{":
  74. if ":" in tag:
  75. prefix, uri = tag.split(":", 1)
  76. try:
  77. if not namespaces:
  78. raise KeyError
  79. yield ttype, "{%s}%s" % (namespaces[prefix], uri)
  80. except KeyError:
  81. raise SyntaxError("prefix %r not found in prefix map" % prefix) from None
  82. elif default_namespace and not parsing_attribute:
  83. yield ttype, "{%s}%s" % (default_namespace, tag)
  84. else:
  85. yield token
  86. parsing_attribute = False
  87. else:
  88. yield token
  89. parsing_attribute = ttype == '@'
  90. def get_parent_map(context):
  91. parent_map = context.parent_map
  92. if parent_map is None:
  93. context.parent_map = parent_map = {}
  94. for p in context.root.iter():
  95. for e in p:
  96. parent_map[e] = p
  97. return parent_map
  98. def _is_wildcard_tag(tag):
  99. return tag[:3] == '{*}' or tag[-2:] == '}*'
  100. def _prepare_tag(tag):
  101. _isinstance, _str = isinstance, str
  102. if tag == '{*}*':
  103. # Same as '*', but no comments or processing instructions.
  104. # It can be a surprise that '*' includes those, but there is no
  105. # justification for '{*}*' doing the same.
  106. def select(context, result):
  107. for elem in result:
  108. if _isinstance(elem.tag, _str):
  109. yield elem
  110. elif tag == '{}*':
  111. # Any tag that is not in a namespace.
  112. def select(context, result):
  113. for elem in result:
  114. el_tag = elem.tag
  115. if _isinstance(el_tag, _str) and el_tag[0] != '{':
  116. yield elem
  117. elif tag[:3] == '{*}':
  118. # The tag in any (or no) namespace.
  119. suffix = tag[2:] # '}name'
  120. no_ns = slice(-len(suffix), None)
  121. tag = tag[3:]
  122. def select(context, result):
  123. for elem in result:
  124. el_tag = elem.tag
  125. if el_tag == tag or _isinstance(el_tag, _str) and el_tag[no_ns] == suffix:
  126. yield elem
  127. elif tag[-2:] == '}*':
  128. # Any tag in the given namespace.
  129. ns = tag[:-1]
  130. ns_only = slice(None, len(ns))
  131. def select(context, result):
  132. for elem in result:
  133. el_tag = elem.tag
  134. if _isinstance(el_tag, _str) and el_tag[ns_only] == ns:
  135. yield elem
  136. else:
  137. raise RuntimeError(f"internal parser error, got {tag}")
  138. return select
  139. def prepare_child(next, token):
  140. tag = token[1]
  141. if _is_wildcard_tag(tag):
  142. select_tag = _prepare_tag(tag)
  143. def select(context, result):
  144. def select_child(result):
  145. for elem in result:
  146. yield from elem
  147. return select_tag(context, select_child(result))
  148. else:
  149. if tag[:2] == '{}':
  150. tag = tag[2:] # '{}tag' == 'tag'
  151. def select(context, result):
  152. for elem in result:
  153. for e in elem:
  154. if e.tag == tag:
  155. yield e
  156. return select
  157. def prepare_star(next, token):
  158. def select(context, result):
  159. for elem in result:
  160. yield from elem
  161. return select
  162. def prepare_self(next, token):
  163. def select(context, result):
  164. yield from result
  165. return select
  166. def prepare_descendant(next, token):
  167. try:
  168. token = next()
  169. except StopIteration:
  170. return
  171. if token[0] == "*":
  172. tag = "*"
  173. elif not token[0]:
  174. tag = token[1]
  175. else:
  176. raise SyntaxError("invalid descendant")
  177. if _is_wildcard_tag(tag):
  178. select_tag = _prepare_tag(tag)
  179. def select(context, result):
  180. def select_child(result):
  181. for elem in result:
  182. for e in elem.iter():
  183. if e is not elem:
  184. yield e
  185. return select_tag(context, select_child(result))
  186. else:
  187. if tag[:2] == '{}':
  188. tag = tag[2:] # '{}tag' == 'tag'
  189. def select(context, result):
  190. for elem in result:
  191. for e in elem.iter(tag):
  192. if e is not elem:
  193. yield e
  194. return select
  195. def prepare_parent(next, token):
  196. def select(context, result):
  197. # FIXME: raise error if .. is applied at toplevel?
  198. parent_map = get_parent_map(context)
  199. result_map = {}
  200. for elem in result:
  201. if elem in parent_map:
  202. parent = parent_map[elem]
  203. if parent not in result_map:
  204. result_map[parent] = None
  205. yield parent
  206. return select
  207. def prepare_predicate(next, token):
  208. # FIXME: replace with real parser!!! refs:
  209. # http://javascript.crockford.com/tdop/tdop.html
  210. signature = []
  211. predicate = []
  212. while 1:
  213. try:
  214. token = next()
  215. except StopIteration:
  216. return
  217. if token[0] == "]":
  218. break
  219. if token == ('', ''):
  220. # ignore whitespace
  221. continue
  222. if token[0] and token[0][:1] in "'\"":
  223. token = "'", token[0][1:-1]
  224. signature.append(token[0] or "-")
  225. predicate.append(token[1])
  226. signature = "".join(signature)
  227. # use signature to determine predicate type
  228. if signature == "@-":
  229. # [@attribute] predicate
  230. key = predicate[1]
  231. def select(context, result):
  232. for elem in result:
  233. if elem.get(key) is not None:
  234. yield elem
  235. return select
  236. if signature == "@-='":
  237. # [@attribute='value']
  238. key = predicate[1]
  239. value = predicate[-1]
  240. def select(context, result):
  241. for elem in result:
  242. if elem.get(key) == value:
  243. yield elem
  244. return select
  245. if signature == "-" and not re.match(r"\-?\d+$", predicate[0]):
  246. # [tag]
  247. tag = predicate[0]
  248. def select(context, result):
  249. for elem in result:
  250. if elem.find(tag) is not None:
  251. yield elem
  252. return select
  253. if signature == ".='" or (signature == "-='" and not re.match(r"\-?\d+$", predicate[0])):
  254. # [.='value'] or [tag='value']
  255. tag = predicate[0]
  256. value = predicate[-1]
  257. if tag:
  258. def select(context, result):
  259. for elem in result:
  260. for e in elem.findall(tag):
  261. if "".join(e.itertext()) == value:
  262. yield elem
  263. break
  264. else:
  265. def select(context, result):
  266. for elem in result:
  267. if "".join(elem.itertext()) == value:
  268. yield elem
  269. return select
  270. if signature == "-" or signature == "-()" or signature == "-()-":
  271. # [index] or [last()] or [last()-index]
  272. if signature == "-":
  273. # [index]
  274. index = int(predicate[0]) - 1
  275. if index < 0:
  276. raise SyntaxError("XPath position >= 1 expected")
  277. else:
  278. if predicate[0] != "last":
  279. raise SyntaxError("unsupported function")
  280. if signature == "-()-":
  281. try:
  282. index = int(predicate[2]) - 1
  283. except ValueError:
  284. raise SyntaxError("unsupported expression")
  285. if index > -2:
  286. raise SyntaxError("XPath offset from last() must be negative")
  287. else:
  288. index = -1
  289. def select(context, result):
  290. parent_map = get_parent_map(context)
  291. for elem in result:
  292. try:
  293. parent = parent_map[elem]
  294. # FIXME: what if the selector is "*" ?
  295. elems = list(parent.findall(elem.tag))
  296. if elems[index] is elem:
  297. yield elem
  298. except (IndexError, KeyError):
  299. pass
  300. return select
  301. raise SyntaxError("invalid predicate")
  302. ops = {
  303. "": prepare_child,
  304. "*": prepare_star,
  305. ".": prepare_self,
  306. "..": prepare_parent,
  307. "//": prepare_descendant,
  308. "[": prepare_predicate,
  309. }
  310. _cache = {}
  311. class _SelectorContext:
  312. parent_map = None
  313. def __init__(self, root):
  314. self.root = root
  315. # --------------------------------------------------------------------
  316. ##
  317. # Generate all matching objects.
  318. def iterfind(elem, path, namespaces=None):
  319. # compile selector pattern
  320. if path[-1:] == "/":
  321. path = path + "*" # implicit all (FIXME: keep this?)
  322. cache_key = (path,)
  323. if namespaces:
  324. cache_key += tuple(sorted(namespaces.items()))
  325. try:
  326. selector = _cache[cache_key]
  327. except KeyError:
  328. if len(_cache) > 100:
  329. _cache.clear()
  330. if path[:1] == "/":
  331. raise SyntaxError("cannot use absolute path on element")
  332. next = iter(xpath_tokenizer(path, namespaces)).__next__
  333. try:
  334. token = next()
  335. except StopIteration:
  336. return
  337. selector = []
  338. while 1:
  339. try:
  340. selector.append(ops[token[0]](next, token))
  341. except StopIteration:
  342. raise SyntaxError("invalid path") from None
  343. try:
  344. token = next()
  345. if token[0] == "/":
  346. token = next()
  347. except StopIteration:
  348. break
  349. _cache[cache_key] = selector
  350. # execute selector pattern
  351. result = [elem]
  352. context = _SelectorContext(elem)
  353. for select in selector:
  354. result = select(context, result)
  355. return result
  356. ##
  357. # Find first matching object.
  358. def find(elem, path, namespaces=None):
  359. return next(iterfind(elem, path, namespaces), None)
  360. ##
  361. # Find all matching objects.
  362. def findall(elem, path, namespaces=None):
  363. return list(iterfind(elem, path, namespaces))
  364. ##
  365. # Find text for first matching object.
  366. def findtext(elem, path, default=None, namespaces=None):
  367. try:
  368. elem = next(iterfind(elem, path, namespaces))
  369. return elem.text or ""
  370. except StopIteration:
  371. return default