_url.py 35 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144
  1. import functools
  2. import sys
  3. import warnings
  4. from collections.abc import Mapping, Sequence
  5. from ipaddress import ip_address
  6. from urllib.parse import SplitResult, parse_qsl, urljoin, urlsplit, urlunsplit, quote
  7. from multidict import MultiDict, MultiDictProxy
  8. import idna
  9. import math
  10. from ._quoting import _Quoter, _Unquoter
  11. DEFAULT_PORTS = {"http": 80, "https": 443, "ws": 80, "wss": 443}
  12. sentinel = object()
  13. def rewrite_module(obj: object) -> object:
  14. obj.__module__ = "yarl"
  15. return obj
  16. class cached_property:
  17. """Use as a class method decorator. It operates almost exactly like
  18. the Python `@property` decorator, but it puts the result of the
  19. method it decorates into the instance dict after the first call,
  20. effectively replacing the function it decorates with an instance
  21. variable. It is, in Python parlance, a data descriptor.
  22. """
  23. def __init__(self, wrapped):
  24. self.wrapped = wrapped
  25. try:
  26. self.__doc__ = wrapped.__doc__
  27. except AttributeError: # pragma: no cover
  28. self.__doc__ = ""
  29. self.name = wrapped.__name__
  30. def __get__(self, inst, owner, _sentinel=sentinel):
  31. if inst is None:
  32. return self
  33. val = inst._cache.get(self.name, _sentinel)
  34. if val is not _sentinel:
  35. return val
  36. val = self.wrapped(inst)
  37. inst._cache[self.name] = val
  38. return val
  39. def __set__(self, inst, value):
  40. raise AttributeError("cached property is read-only")
  41. @rewrite_module
  42. class URL:
  43. # Don't derive from str
  44. # follow pathlib.Path design
  45. # probably URL will not suffer from pathlib problems:
  46. # it's intended for libraries like aiohttp,
  47. # not to be passed into standard library functions like os.open etc.
  48. # URL grammar (RFC 3986)
  49. # pct-encoded = "%" HEXDIG HEXDIG
  50. # reserved = gen-delims / sub-delims
  51. # gen-delims = ":" / "/" / "?" / "#" / "[" / "]" / "@"
  52. # sub-delims = "!" / "$" / "&" / "'" / "(" / ")"
  53. # / "*" / "+" / "," / ";" / "="
  54. # unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~"
  55. # URI = scheme ":" hier-part [ "?" query ] [ "#" fragment ]
  56. # hier-part = "//" authority path-abempty
  57. # / path-absolute
  58. # / path-rootless
  59. # / path-empty
  60. # scheme = ALPHA *( ALPHA / DIGIT / "+" / "-" / "." )
  61. # authority = [ userinfo "@" ] host [ ":" port ]
  62. # userinfo = *( unreserved / pct-encoded / sub-delims / ":" )
  63. # host = IP-literal / IPv4address / reg-name
  64. # IP-literal = "[" ( IPv6address / IPvFuture ) "]"
  65. # IPvFuture = "v" 1*HEXDIG "." 1*( unreserved / sub-delims / ":" )
  66. # IPv6address = 6( h16 ":" ) ls32
  67. # / "::" 5( h16 ":" ) ls32
  68. # / [ h16 ] "::" 4( h16 ":" ) ls32
  69. # / [ *1( h16 ":" ) h16 ] "::" 3( h16 ":" ) ls32
  70. # / [ *2( h16 ":" ) h16 ] "::" 2( h16 ":" ) ls32
  71. # / [ *3( h16 ":" ) h16 ] "::" h16 ":" ls32
  72. # / [ *4( h16 ":" ) h16 ] "::" ls32
  73. # / [ *5( h16 ":" ) h16 ] "::" h16
  74. # / [ *6( h16 ":" ) h16 ] "::"
  75. # ls32 = ( h16 ":" h16 ) / IPv4address
  76. # ; least-significant 32 bits of address
  77. # h16 = 1*4HEXDIG
  78. # ; 16 bits of address represented in hexadecimal
  79. # IPv4address = dec-octet "." dec-octet "." dec-octet "." dec-octet
  80. # dec-octet = DIGIT ; 0-9
  81. # / %x31-39 DIGIT ; 10-99
  82. # / "1" 2DIGIT ; 100-199
  83. # / "2" %x30-34 DIGIT ; 200-249
  84. # / "25" %x30-35 ; 250-255
  85. # reg-name = *( unreserved / pct-encoded / sub-delims )
  86. # port = *DIGIT
  87. # path = path-abempty ; begins with "/" or is empty
  88. # / path-absolute ; begins with "/" but not "//"
  89. # / path-noscheme ; begins with a non-colon segment
  90. # / path-rootless ; begins with a segment
  91. # / path-empty ; zero characters
  92. # path-abempty = *( "/" segment )
  93. # path-absolute = "/" [ segment-nz *( "/" segment ) ]
  94. # path-noscheme = segment-nz-nc *( "/" segment )
  95. # path-rootless = segment-nz *( "/" segment )
  96. # path-empty = 0<pchar>
  97. # segment = *pchar
  98. # segment-nz = 1*pchar
  99. # segment-nz-nc = 1*( unreserved / pct-encoded / sub-delims / "@" )
  100. # ; non-zero-length segment without any colon ":"
  101. # pchar = unreserved / pct-encoded / sub-delims / ":" / "@"
  102. # query = *( pchar / "/" / "?" )
  103. # fragment = *( pchar / "/" / "?" )
  104. # URI-reference = URI / relative-ref
  105. # relative-ref = relative-part [ "?" query ] [ "#" fragment ]
  106. # relative-part = "//" authority path-abempty
  107. # / path-absolute
  108. # / path-noscheme
  109. # / path-empty
  110. # absolute-URI = scheme ":" hier-part [ "?" query ]
  111. __slots__ = ("_cache", "_val")
  112. _QUOTER = _Quoter(requote=False)
  113. _REQUOTER = _Quoter()
  114. _PATH_QUOTER = _Quoter(safe="@:", protected="/+", requote=False)
  115. _PATH_REQUOTER = _Quoter(safe="@:", protected="/+")
  116. _QUERY_QUOTER = _Quoter(safe="?/:@", protected="=+&;", qs=True, requote=False)
  117. _QUERY_REQUOTER = _Quoter(safe="?/:@", protected="=+&;", qs=True)
  118. _QUERY_PART_QUOTER = _Quoter(safe="?/:@", qs=True, requote=False)
  119. _FRAGMENT_QUOTER = _Quoter(safe="?/:@", requote=False)
  120. _FRAGMENT_REQUOTER = _Quoter(safe="?/:@")
  121. _UNQUOTER = _Unquoter()
  122. _PATH_UNQUOTER = _Unquoter(unsafe="+")
  123. _QS_UNQUOTER = _Unquoter(qs=True)
  124. def __new__(cls, val="", *, encoded=False, strict=None):
  125. if strict is not None: # pragma: no cover
  126. warnings.warn("strict parameter is ignored")
  127. if type(val) is cls:
  128. return val
  129. if type(val) is str:
  130. val = urlsplit(val)
  131. elif type(val) is SplitResult:
  132. if not encoded:
  133. raise ValueError("Cannot apply decoding to SplitResult")
  134. elif isinstance(val, str):
  135. val = urlsplit(str(val))
  136. else:
  137. raise TypeError("Constructor parameter should be str")
  138. if not encoded:
  139. if not val[1]: # netloc
  140. netloc = ""
  141. host = ""
  142. else:
  143. host = val.hostname
  144. if host is None:
  145. raise ValueError("Invalid URL: host is required for absolute urls")
  146. try:
  147. port = val.port
  148. except ValueError as e:
  149. raise ValueError(
  150. "Invalid URL: port can't be converted to integer"
  151. ) from e
  152. netloc = cls._make_netloc(
  153. val.username, val.password, host, port, encode=True, requote=True
  154. )
  155. path = cls._PATH_REQUOTER(val[2])
  156. if netloc:
  157. path = cls._normalize_path(path)
  158. cls._validate_authority_uri_abs_path(host=host, path=path)
  159. query = cls._QUERY_REQUOTER(val[3])
  160. fragment = cls._FRAGMENT_REQUOTER(val[4])
  161. val = SplitResult(val[0], netloc, path, query, fragment)
  162. self = object.__new__(cls)
  163. self._val = val
  164. self._cache = {}
  165. return self
  166. @classmethod
  167. def build(
  168. cls,
  169. *,
  170. scheme="",
  171. authority="",
  172. user=None,
  173. password=None,
  174. host="",
  175. port=None,
  176. path="",
  177. query=None,
  178. query_string="",
  179. fragment="",
  180. encoded=False
  181. ):
  182. """Creates and returns a new URL"""
  183. if authority and (user or password or host or port):
  184. raise ValueError(
  185. 'Can\'t mix "authority" with "user", "password", "host" or "port".'
  186. )
  187. if port and not host:
  188. raise ValueError('Can\'t build URL with "port" but without "host".')
  189. if query and query_string:
  190. raise ValueError('Only one of "query" or "query_string" should be passed')
  191. if (
  192. scheme is None
  193. or authority is None
  194. or path is None
  195. or query_string is None
  196. or fragment is None
  197. ):
  198. raise TypeError(
  199. 'NoneType is illegal for "scheme", "authority", "path", '
  200. '"query_string", and "fragment" args, use empty string instead.'
  201. )
  202. if authority:
  203. if encoded:
  204. netloc = authority
  205. else:
  206. tmp = SplitResult("", authority, "", "", "")
  207. netloc = cls._make_netloc(
  208. tmp.username, tmp.password, tmp.hostname, tmp.port, encode=True
  209. )
  210. elif not user and not password and not host and not port:
  211. netloc = ""
  212. else:
  213. netloc = cls._make_netloc(
  214. user, password, host, port, encode=not encoded, encode_host=not encoded
  215. )
  216. if not encoded:
  217. path = cls._PATH_QUOTER(path)
  218. if netloc:
  219. path = cls._normalize_path(path)
  220. cls._validate_authority_uri_abs_path(host=host, path=path)
  221. query_string = cls._QUERY_QUOTER(query_string)
  222. fragment = cls._FRAGMENT_QUOTER(fragment)
  223. url = cls(
  224. SplitResult(scheme, netloc, path, query_string, fragment), encoded=True
  225. )
  226. if query:
  227. return url.with_query(query)
  228. else:
  229. return url
  230. def __init_subclass__(cls):
  231. raise TypeError("Inheritance a class {!r} from URL is forbidden".format(cls))
  232. def __str__(self):
  233. val = self._val
  234. if not val.path and self.is_absolute() and (val.query or val.fragment):
  235. val = val._replace(path="/")
  236. return urlunsplit(val)
  237. def __repr__(self):
  238. return "{}('{}')".format(self.__class__.__name__, str(self))
  239. def __eq__(self, other):
  240. if not type(other) is URL:
  241. return NotImplemented
  242. val1 = self._val
  243. if not val1.path and self.is_absolute():
  244. val1 = val1._replace(path="/")
  245. val2 = other._val
  246. if not val2.path and other.is_absolute():
  247. val2 = val2._replace(path="/")
  248. return val1 == val2
  249. def __hash__(self):
  250. ret = self._cache.get("hash")
  251. if ret is None:
  252. val = self._val
  253. if not val.path and self.is_absolute():
  254. val = val._replace(path="/")
  255. ret = self._cache["hash"] = hash(val)
  256. return ret
  257. def __le__(self, other):
  258. if not type(other) is URL:
  259. return NotImplemented
  260. return self._val <= other._val
  261. def __lt__(self, other):
  262. if not type(other) is URL:
  263. return NotImplemented
  264. return self._val < other._val
  265. def __ge__(self, other):
  266. if not type(other) is URL:
  267. return NotImplemented
  268. return self._val >= other._val
  269. def __gt__(self, other):
  270. if not type(other) is URL:
  271. return NotImplemented
  272. return self._val > other._val
  273. def __truediv__(self, name):
  274. name = self._PATH_QUOTER(name)
  275. if name.startswith("/"):
  276. raise ValueError(
  277. "Appending path {!r} starting from slash is forbidden".format(name)
  278. )
  279. path = self._val.path
  280. if path == "/":
  281. new_path = "/" + name
  282. elif not path and not self.is_absolute():
  283. new_path = name
  284. else:
  285. parts = path.rstrip("/").split("/")
  286. parts.append(name)
  287. new_path = "/".join(parts)
  288. if self.is_absolute():
  289. new_path = self._normalize_path(new_path)
  290. return URL(
  291. self._val._replace(path=new_path, query="", fragment=""), encoded=True
  292. )
  293. def __mod__(self, query):
  294. return self.update_query(query)
  295. def __bool__(self) -> bool:
  296. return bool(
  297. self._val.netloc or self._val.path or self._val.query or self._val.fragment
  298. )
  299. def __getstate__(self):
  300. return (self._val,)
  301. def __setstate__(self, state):
  302. if state[0] is None and isinstance(state[1], dict):
  303. # default style pickle
  304. self._val = state[1]["_val"]
  305. else:
  306. self._val, *unused = state
  307. self._cache = {}
  308. def is_absolute(self):
  309. """A check for absolute URLs.
  310. Return True for absolute ones (having scheme or starting
  311. with //), False otherwise.
  312. """
  313. return self.raw_host is not None
  314. def is_default_port(self):
  315. """A check for default port.
  316. Return True if port is default for specified scheme,
  317. e.g. 'http://python.org' or 'http://python.org:80', False
  318. otherwise.
  319. """
  320. if self.port is None:
  321. return False
  322. default = DEFAULT_PORTS.get(self.scheme)
  323. if default is None:
  324. return False
  325. return self.port == default
  326. def origin(self):
  327. """Return an URL with scheme, host and port parts only.
  328. user, password, path, query and fragment are removed.
  329. """
  330. # TODO: add a keyword-only option for keeping user/pass maybe?
  331. if not self.is_absolute():
  332. raise ValueError("URL should be absolute")
  333. if not self._val.scheme:
  334. raise ValueError("URL should have scheme")
  335. v = self._val
  336. netloc = self._make_netloc(None, None, v.hostname, v.port)
  337. val = v._replace(netloc=netloc, path="", query="", fragment="")
  338. return URL(val, encoded=True)
  339. def relative(self):
  340. """Return a relative part of the URL.
  341. scheme, user, password, host and port are removed.
  342. """
  343. if not self.is_absolute():
  344. raise ValueError("URL should be absolute")
  345. val = self._val._replace(scheme="", netloc="")
  346. return URL(val, encoded=True)
  347. @property
  348. def scheme(self):
  349. """Scheme for absolute URLs.
  350. Empty string for relative URLs or URLs starting with //
  351. """
  352. return self._val.scheme
  353. @property
  354. def raw_authority(self):
  355. """Encoded authority part of URL.
  356. Empty string for relative URLs.
  357. """
  358. return self._val.netloc
  359. @cached_property
  360. def authority(self):
  361. """Decoded authority part of URL.
  362. Empty string for relative URLs.
  363. """
  364. return self._make_netloc(
  365. self.user, self.password, self.host, self.port, encode_host=False
  366. )
  367. @property
  368. def raw_user(self):
  369. """Encoded user part of URL.
  370. None if user is missing.
  371. """
  372. # not .username
  373. ret = self._val.username
  374. if not ret:
  375. return None
  376. return ret
  377. @cached_property
  378. def user(self):
  379. """Decoded user part of URL.
  380. None if user is missing.
  381. """
  382. return self._UNQUOTER(self.raw_user)
  383. @property
  384. def raw_password(self):
  385. """Encoded password part of URL.
  386. None if password is missing.
  387. """
  388. return self._val.password
  389. @cached_property
  390. def password(self):
  391. """Decoded password part of URL.
  392. None if password is missing.
  393. """
  394. return self._UNQUOTER(self.raw_password)
  395. @property
  396. def raw_host(self):
  397. """Encoded host part of URL.
  398. None for relative URLs.
  399. """
  400. # Use host instead of hostname for sake of shortness
  401. # May add .hostname prop later
  402. return self._val.hostname
  403. @cached_property
  404. def host(self):
  405. """Decoded host part of URL.
  406. None for relative URLs.
  407. """
  408. raw = self.raw_host
  409. if raw is None:
  410. return None
  411. if "%" in raw:
  412. # Hack for scoped IPv6 addresses like
  413. # fe80::2%Проверка
  414. # presence of '%' sign means only IPv6 address, so idna is useless.
  415. return raw
  416. return _idna_decode(raw)
  417. @property
  418. def port(self):
  419. """Port part of URL, with scheme-based fallback.
  420. None for relative URLs or URLs without explicit port and
  421. scheme without default port substitution.
  422. """
  423. return self._val.port or DEFAULT_PORTS.get(self._val.scheme)
  424. @property
  425. def explicit_port(self):
  426. """Port part of URL, without scheme-based fallback.
  427. None for relative URLs or URLs without explicit port.
  428. """
  429. return self._val.port
  430. @property
  431. def raw_path(self):
  432. """Encoded path of URL.
  433. / for absolute URLs without path part.
  434. """
  435. ret = self._val.path
  436. if not ret and self.is_absolute():
  437. ret = "/"
  438. return ret
  439. @cached_property
  440. def path(self):
  441. """Decoded path of URL.
  442. / for absolute URLs without path part.
  443. """
  444. return self._PATH_UNQUOTER(self.raw_path)
  445. @cached_property
  446. def query(self):
  447. """A MultiDictProxy representing parsed query parameters in decoded
  448. representation.
  449. Empty value if URL has no query part.
  450. """
  451. ret = MultiDict(parse_qsl(self.raw_query_string, keep_blank_values=True))
  452. return MultiDictProxy(ret)
  453. @property
  454. def raw_query_string(self):
  455. """Encoded query part of URL.
  456. Empty string if query is missing.
  457. """
  458. return self._val.query
  459. @cached_property
  460. def query_string(self):
  461. """Decoded query part of URL.
  462. Empty string if query is missing.
  463. """
  464. return self._QS_UNQUOTER(self.raw_query_string)
  465. @cached_property
  466. def path_qs(self):
  467. """Decoded path of URL with query."""
  468. if not self.query_string:
  469. return self.path
  470. return "{}?{}".format(self.path, self.query_string)
  471. @cached_property
  472. def raw_path_qs(self):
  473. """Encoded path of URL with query."""
  474. if not self.raw_query_string:
  475. return self.raw_path
  476. return "{}?{}".format(self.raw_path, self.raw_query_string)
  477. @property
  478. def raw_fragment(self):
  479. """Encoded fragment part of URL.
  480. Empty string if fragment is missing.
  481. """
  482. return self._val.fragment
  483. @cached_property
  484. def fragment(self):
  485. """Decoded fragment part of URL.
  486. Empty string if fragment is missing.
  487. """
  488. return self._UNQUOTER(self.raw_fragment)
  489. @cached_property
  490. def raw_parts(self):
  491. """A tuple containing encoded *path* parts.
  492. ('/',) for absolute URLs if *path* is missing.
  493. """
  494. path = self._val.path
  495. if self.is_absolute():
  496. if not path:
  497. parts = ["/"]
  498. else:
  499. parts = ["/"] + path[1:].split("/")
  500. else:
  501. if path.startswith("/"):
  502. parts = ["/"] + path[1:].split("/")
  503. else:
  504. parts = path.split("/")
  505. return tuple(parts)
  506. @cached_property
  507. def parts(self):
  508. """A tuple containing decoded *path* parts.
  509. ('/',) for absolute URLs if *path* is missing.
  510. """
  511. return tuple(self._UNQUOTER(part) for part in self.raw_parts)
  512. @cached_property
  513. def parent(self):
  514. """A new URL with last part of path removed and cleaned up query and
  515. fragment.
  516. """
  517. path = self.raw_path
  518. if not path or path == "/":
  519. if self.raw_fragment or self.raw_query_string:
  520. return URL(self._val._replace(query="", fragment=""), encoded=True)
  521. return self
  522. parts = path.split("/")
  523. val = self._val._replace(path="/".join(parts[:-1]), query="", fragment="")
  524. return URL(val, encoded=True)
  525. @cached_property
  526. def raw_name(self):
  527. """The last part of raw_parts."""
  528. parts = self.raw_parts
  529. if self.is_absolute():
  530. parts = parts[1:]
  531. if not parts:
  532. return ""
  533. else:
  534. return parts[-1]
  535. else:
  536. return parts[-1]
  537. @cached_property
  538. def name(self):
  539. """The last part of parts."""
  540. return self._UNQUOTER(self.raw_name)
  541. @staticmethod
  542. def _validate_authority_uri_abs_path(host, path):
  543. """Ensure that path in URL with authority starts with a leading slash.
  544. Raise ValueError if not.
  545. """
  546. if len(host) > 0 and len(path) > 0 and not path.startswith("/"):
  547. raise ValueError(
  548. "Path in a URL with authority should start with a slash ('/') if set"
  549. )
  550. @classmethod
  551. def _normalize_path(cls, path):
  552. # Drop '.' and '..' from path
  553. segments = path.split("/")
  554. resolved_path = []
  555. for seg in segments:
  556. if seg == "..":
  557. try:
  558. resolved_path.pop()
  559. except IndexError:
  560. # ignore any .. segments that would otherwise cause an
  561. # IndexError when popped from resolved_path if
  562. # resolving for rfc3986
  563. pass
  564. elif seg == ".":
  565. continue
  566. else:
  567. resolved_path.append(seg)
  568. if segments[-1] in (".", ".."):
  569. # do some post-processing here.
  570. # if the last segment was a relative dir,
  571. # then we need to append the trailing '/'
  572. resolved_path.append("")
  573. return "/".join(resolved_path)
  574. if sys.version_info >= (3, 7):
  575. @classmethod
  576. def _encode_host(cls, host, human=False):
  577. try:
  578. ip, sep, zone = host.partition("%")
  579. ip = ip_address(ip)
  580. except ValueError:
  581. host = host.lower()
  582. # IDNA encoding is slow,
  583. # skip it for ASCII-only strings
  584. # Don't move the check into _idna_encode() helper
  585. # to reduce the cache size
  586. if human or host.isascii():
  587. return host
  588. host = _idna_encode(host)
  589. else:
  590. host = ip.compressed
  591. if sep:
  592. host += "%" + zone
  593. if ip.version == 6:
  594. host = "[" + host + "]"
  595. return host
  596. else:
  597. # work around for missing str.isascii() in Python <= 3.6
  598. @classmethod
  599. def _encode_host(cls, host, human=False):
  600. try:
  601. ip, sep, zone = host.partition("%")
  602. ip = ip_address(ip)
  603. except ValueError:
  604. host = host.lower()
  605. if human:
  606. return host
  607. for char in host:
  608. if char > "\x7f":
  609. break
  610. else:
  611. return host
  612. host = _idna_encode(host)
  613. else:
  614. host = ip.compressed
  615. if sep:
  616. host += "%" + zone
  617. if ip.version == 6:
  618. host = "[" + host + "]"
  619. return host
  620. @classmethod
  621. def _make_netloc(
  622. cls, user, password, host, port, encode=False, encode_host=True, requote=False
  623. ):
  624. quoter = cls._REQUOTER if requote else cls._QUOTER
  625. if encode_host:
  626. ret = cls._encode_host(host)
  627. else:
  628. ret = host
  629. if port:
  630. ret = ret + ":" + str(port)
  631. if password is not None:
  632. if not user:
  633. user = ""
  634. else:
  635. if encode:
  636. user = quoter(user)
  637. if encode:
  638. password = quoter(password)
  639. user = user + ":" + password
  640. elif user and encode:
  641. user = quoter(user)
  642. if user:
  643. ret = user + "@" + ret
  644. return ret
  645. def with_scheme(self, scheme):
  646. """Return a new URL with scheme replaced."""
  647. # N.B. doesn't cleanup query/fragment
  648. if not isinstance(scheme, str):
  649. raise TypeError("Invalid scheme type")
  650. if not self.is_absolute():
  651. raise ValueError("scheme replacement is not allowed for relative URLs")
  652. return URL(self._val._replace(scheme=scheme.lower()), encoded=True)
  653. def with_user(self, user):
  654. """Return a new URL with user replaced.
  655. Autoencode user if needed.
  656. Clear user/password if user is None.
  657. """
  658. # N.B. doesn't cleanup query/fragment
  659. val = self._val
  660. if user is None:
  661. password = None
  662. elif isinstance(user, str):
  663. user = self._QUOTER(user)
  664. password = val.password
  665. else:
  666. raise TypeError("Invalid user type")
  667. if not self.is_absolute():
  668. raise ValueError("user replacement is not allowed for relative URLs")
  669. return URL(
  670. self._val._replace(
  671. netloc=self._make_netloc(user, password, val.hostname, val.port)
  672. ),
  673. encoded=True,
  674. )
  675. def with_password(self, password):
  676. """Return a new URL with password replaced.
  677. Autoencode password if needed.
  678. Clear password if argument is None.
  679. """
  680. # N.B. doesn't cleanup query/fragment
  681. if password is None:
  682. pass
  683. elif isinstance(password, str):
  684. password = self._QUOTER(password)
  685. else:
  686. raise TypeError("Invalid password type")
  687. if not self.is_absolute():
  688. raise ValueError("password replacement is not allowed for relative URLs")
  689. val = self._val
  690. return URL(
  691. self._val._replace(
  692. netloc=self._make_netloc(val.username, password, val.hostname, val.port)
  693. ),
  694. encoded=True,
  695. )
  696. def with_host(self, host):
  697. """Return a new URL with host replaced.
  698. Autoencode host if needed.
  699. Changing host for relative URLs is not allowed, use .join()
  700. instead.
  701. """
  702. # N.B. doesn't cleanup query/fragment
  703. if not isinstance(host, str):
  704. raise TypeError("Invalid host type")
  705. if not self.is_absolute():
  706. raise ValueError("host replacement is not allowed for relative URLs")
  707. if not host:
  708. raise ValueError("host removing is not allowed")
  709. val = self._val
  710. return URL(
  711. self._val._replace(
  712. netloc=self._make_netloc(val.username, val.password, host, val.port)
  713. ),
  714. encoded=True,
  715. )
  716. def with_port(self, port):
  717. """Return a new URL with port replaced.
  718. Clear port to default if None is passed.
  719. """
  720. # N.B. doesn't cleanup query/fragment
  721. if port is not None and not isinstance(port, int):
  722. raise TypeError("port should be int or None, got {}".format(type(port)))
  723. if not self.is_absolute():
  724. raise ValueError("port replacement is not allowed for relative URLs")
  725. val = self._val
  726. return URL(
  727. self._val._replace(
  728. netloc=self._make_netloc(
  729. val.username, val.password, val.hostname, port, encode=True
  730. )
  731. ),
  732. encoded=True,
  733. )
  734. def with_path(self, path, *, encoded=False):
  735. """Return a new URL with path replaced."""
  736. if not encoded:
  737. path = self._PATH_QUOTER(path)
  738. if self.is_absolute():
  739. path = self._normalize_path(path)
  740. if len(path) > 0 and path[0] != "/":
  741. path = "/" + path
  742. return URL(self._val._replace(path=path, query="", fragment=""), encoded=True)
  743. @classmethod
  744. def _query_seq_pairs(cls, quoter, pairs):
  745. for key, val in pairs:
  746. if isinstance(val, (list, tuple)):
  747. for v in val:
  748. yield quoter(key) + "=" + quoter(cls._query_var(v))
  749. else:
  750. yield quoter(key) + "=" + quoter(cls._query_var(val))
  751. @staticmethod
  752. def _query_var(v):
  753. cls = type(v)
  754. if issubclass(cls, str):
  755. return v
  756. if issubclass(cls, float):
  757. if math.isinf(v):
  758. raise ValueError("float('inf') is not supported")
  759. if math.isnan(v):
  760. raise ValueError("float('nan') is not supported")
  761. return str(float(v))
  762. if issubclass(cls, int) and cls is not bool:
  763. return str(int(v))
  764. raise TypeError(
  765. "Invalid variable type: value "
  766. "should be str, int or float, got {!r} "
  767. "of type {}".format(v, cls)
  768. )
  769. def _get_str_query(self, *args, **kwargs):
  770. if kwargs:
  771. if len(args) > 0:
  772. raise ValueError(
  773. "Either kwargs or single query parameter must be present"
  774. )
  775. query = kwargs
  776. elif len(args) == 1:
  777. query = args[0]
  778. else:
  779. raise ValueError("Either kwargs or single query parameter must be present")
  780. if query is None:
  781. query = ""
  782. elif isinstance(query, Mapping):
  783. quoter = self._QUERY_PART_QUOTER
  784. query = "&".join(self._query_seq_pairs(quoter, query.items()))
  785. elif isinstance(query, str):
  786. query = self._QUERY_QUOTER(query)
  787. elif isinstance(query, (bytes, bytearray, memoryview)):
  788. raise TypeError(
  789. "Invalid query type: bytes, bytearray and memoryview are forbidden"
  790. )
  791. elif isinstance(query, Sequence):
  792. quoter = self._QUERY_PART_QUOTER
  793. # We don't expect sequence values if we're given a list of pairs
  794. # already; only mappings like builtin `dict` which can't have the
  795. # same key pointing to multiple values are allowed to use
  796. # `_query_seq_pairs`.
  797. query = "&".join(
  798. quoter(k) + "=" + quoter(self._query_var(v)) for k, v in query
  799. )
  800. else:
  801. raise TypeError(
  802. "Invalid query type: only str, mapping or "
  803. "sequence of (key, value) pairs is allowed"
  804. )
  805. return query
  806. def with_query(self, *args, **kwargs):
  807. """Return a new URL with query part replaced.
  808. Accepts any Mapping (e.g. dict, multidict.MultiDict instances)
  809. or str, autoencode the argument if needed.
  810. A sequence of (key, value) pairs is supported as well.
  811. It also can take an arbitrary number of keyword arguments.
  812. Clear query if None is passed.
  813. """
  814. # N.B. doesn't cleanup query/fragment
  815. new_query = self._get_str_query(*args, **kwargs)
  816. return URL(
  817. self._val._replace(path=self._val.path, query=new_query), encoded=True
  818. )
  819. def update_query(self, *args, **kwargs):
  820. """Return a new URL with query part updated."""
  821. s = self._get_str_query(*args, **kwargs)
  822. new_query = MultiDict(parse_qsl(s, keep_blank_values=True))
  823. query = MultiDict(self.query)
  824. query.update(new_query)
  825. return URL(self._val._replace(query=self._get_str_query(query)), encoded=True)
  826. def with_fragment(self, fragment):
  827. """Return a new URL with fragment replaced.
  828. Autoencode fragment if needed.
  829. Clear fragment to default if None is passed.
  830. """
  831. # N.B. doesn't cleanup query/fragment
  832. if fragment is None:
  833. raw_fragment = ""
  834. elif not isinstance(fragment, str):
  835. raise TypeError("Invalid fragment type")
  836. else:
  837. raw_fragment = self._FRAGMENT_QUOTER(fragment)
  838. if self.raw_fragment == raw_fragment:
  839. return self
  840. return URL(self._val._replace(fragment=raw_fragment), encoded=True)
  841. def with_name(self, name):
  842. """Return a new URL with name (last part of path) replaced.
  843. Query and fragment parts are cleaned up.
  844. Name is encoded if needed.
  845. """
  846. # N.B. DOES cleanup query/fragment
  847. if not isinstance(name, str):
  848. raise TypeError("Invalid name type")
  849. if "/" in name:
  850. raise ValueError("Slash in name is not allowed")
  851. name = self._PATH_QUOTER(name)
  852. if name in (".", ".."):
  853. raise ValueError(". and .. values are forbidden")
  854. parts = list(self.raw_parts)
  855. if self.is_absolute():
  856. if len(parts) == 1:
  857. parts.append(name)
  858. else:
  859. parts[-1] = name
  860. parts[0] = "" # replace leading '/'
  861. else:
  862. parts[-1] = name
  863. if parts[0] == "/":
  864. parts[0] = "" # replace leading '/'
  865. return URL(
  866. self._val._replace(path="/".join(parts), query="", fragment=""),
  867. encoded=True,
  868. )
  869. def join(self, url):
  870. """Join URLs
  871. Construct a full (“absolute”) URL by combining a “base URL”
  872. (self) with another URL (url).
  873. Informally, this uses components of the base URL, in
  874. particular the addressing scheme, the network location and
  875. (part of) the path, to provide missing components in the
  876. relative URL.
  877. """
  878. # See docs for urllib.parse.urljoin
  879. if not isinstance(url, URL):
  880. raise TypeError("url should be URL")
  881. return URL(urljoin(str(self), str(url)), encoded=True)
  882. def human_repr(self):
  883. """Return decoded human readable string for URL representation."""
  884. user = _human_quote(self.user, "#/:?@")
  885. password = _human_quote(self.password, "#/:?@")
  886. host = self.host
  887. if host:
  888. host = self._encode_host(self.host, human=True)
  889. path = _human_quote(self.path, "#?")
  890. query_string = "&".join(
  891. "{}={}".format(_human_quote(k, "#&+;="), _human_quote(v, "#&+;="))
  892. for k, v in self.query.items()
  893. )
  894. fragment = _human_quote(self.fragment, "")
  895. return urlunsplit(
  896. SplitResult(
  897. self.scheme,
  898. self._make_netloc(
  899. user,
  900. password,
  901. host,
  902. self._val.port,
  903. encode_host=False,
  904. ),
  905. path,
  906. query_string,
  907. fragment,
  908. )
  909. )
  910. def _human_quote(s, unsafe):
  911. if not s:
  912. return s
  913. for c in "%" + unsafe:
  914. if c in s:
  915. s = s.replace(c, "%{:02X}".format(ord(c)))
  916. if s.isprintable():
  917. return s
  918. return "".join(c if c.isprintable() else quote(c) for c in s)
  919. _MAXCACHE = 256
  920. @functools.lru_cache(_MAXCACHE)
  921. def _idna_decode(raw):
  922. try:
  923. return idna.decode(raw.encode("ascii"))
  924. except UnicodeError: # e.g. '::1'
  925. return raw.encode("ascii").decode("idna")
  926. @functools.lru_cache(_MAXCACHE)
  927. def _idna_encode(host):
  928. try:
  929. return idna.encode(host, uts46=True).decode("ascii")
  930. except UnicodeError:
  931. return host.encode("idna").decode("ascii")
  932. @rewrite_module
  933. def cache_clear():
  934. _idna_decode.cache_clear()
  935. _idna_encode.cache_clear()
  936. @rewrite_module
  937. def cache_info():
  938. return {
  939. "idna_encode": _idna_encode.cache_info(),
  940. "idna_decode": _idna_decode.cache_info(),
  941. }
  942. @rewrite_module
  943. def cache_configure(*, idna_encode_size=_MAXCACHE, idna_decode_size=_MAXCACHE):
  944. global _idna_decode, _idna_encode
  945. _idna_encode = functools.lru_cache(idna_encode_size)(_idna_encode.__wrapped__)
  946. _idna_decode = functools.lru_cache(idna_decode_size)(_idna_decode.__wrapped__)