utils.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368
  1. # Copyright (C) 2001-2010 Python Software Foundation
  2. # Author: Barry Warsaw
  3. # Contact: email-sig@python.org
  4. """Miscellaneous utilities."""
  5. __all__ = [
  6. 'collapse_rfc2231_value',
  7. 'decode_params',
  8. 'decode_rfc2231',
  9. 'encode_rfc2231',
  10. 'formataddr',
  11. 'formatdate',
  12. 'format_datetime',
  13. 'getaddresses',
  14. 'make_msgid',
  15. 'mktime_tz',
  16. 'parseaddr',
  17. 'parsedate',
  18. 'parsedate_tz',
  19. 'parsedate_to_datetime',
  20. 'unquote',
  21. ]
  22. import os
  23. import re
  24. import time
  25. import random
  26. import socket
  27. import datetime
  28. import urllib.parse
  29. from email._parseaddr import quote
  30. from email._parseaddr import AddressList as _AddressList
  31. from email._parseaddr import mktime_tz
  32. from email._parseaddr import parsedate, parsedate_tz, _parsedate_tz
  33. # Intrapackage imports
  34. from email.charset import Charset
  35. COMMASPACE = ', '
  36. EMPTYSTRING = ''
  37. UEMPTYSTRING = ''
  38. CRLF = '\r\n'
  39. TICK = "'"
  40. specialsre = re.compile(r'[][\\()<>@,:;".]')
  41. escapesre = re.compile(r'[\\"]')
  42. def _has_surrogates(s):
  43. """Return True if s contains surrogate-escaped binary data."""
  44. # This check is based on the fact that unless there are surrogates, utf8
  45. # (Python's default encoding) can encode any string. This is the fastest
  46. # way to check for surrogates, see issue 11454 for timings.
  47. try:
  48. s.encode()
  49. return False
  50. except UnicodeEncodeError:
  51. return True
  52. # How to deal with a string containing bytes before handing it to the
  53. # application through the 'normal' interface.
  54. def _sanitize(string):
  55. # Turn any escaped bytes into unicode 'unknown' char. If the escaped
  56. # bytes happen to be utf-8 they will instead get decoded, even if they
  57. # were invalid in the charset the source was supposed to be in. This
  58. # seems like it is not a bad thing; a defect was still registered.
  59. original_bytes = string.encode('utf-8', 'surrogateescape')
  60. return original_bytes.decode('utf-8', 'replace')
  61. # Helpers
  62. def formataddr(pair, charset='utf-8'):
  63. """The inverse of parseaddr(), this takes a 2-tuple of the form
  64. (realname, email_address) and returns the string value suitable
  65. for an RFC 2822 From, To or Cc header.
  66. If the first element of pair is false, then the second element is
  67. returned unmodified.
  68. The optional charset is the character set that is used to encode
  69. realname in case realname is not ASCII safe. Can be an instance of str or
  70. a Charset-like object which has a header_encode method. Default is
  71. 'utf-8'.
  72. """
  73. name, address = pair
  74. # The address MUST (per RFC) be ascii, so raise a UnicodeError if it isn't.
  75. address.encode('ascii')
  76. if name:
  77. try:
  78. name.encode('ascii')
  79. except UnicodeEncodeError:
  80. if isinstance(charset, str):
  81. charset = Charset(charset)
  82. encoded_name = charset.header_encode(name)
  83. return "%s <%s>" % (encoded_name, address)
  84. else:
  85. quotes = ''
  86. if specialsre.search(name):
  87. quotes = '"'
  88. name = escapesre.sub(r'\\\g<0>', name)
  89. return '%s%s%s <%s>' % (quotes, name, quotes, address)
  90. return address
  91. def getaddresses(fieldvalues):
  92. """Return a list of (REALNAME, EMAIL) for each fieldvalue."""
  93. all = COMMASPACE.join(str(v) for v in fieldvalues)
  94. a = _AddressList(all)
  95. return a.addresslist
  96. def _format_timetuple_and_zone(timetuple, zone):
  97. return '%s, %02d %s %04d %02d:%02d:%02d %s' % (
  98. ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'][timetuple[6]],
  99. timetuple[2],
  100. ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun',
  101. 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'][timetuple[1] - 1],
  102. timetuple[0], timetuple[3], timetuple[4], timetuple[5],
  103. zone)
  104. def formatdate(timeval=None, localtime=False, usegmt=False):
  105. """Returns a date string as specified by RFC 2822, e.g.:
  106. Fri, 09 Nov 2001 01:08:47 -0000
  107. Optional timeval if given is a floating point time value as accepted by
  108. gmtime() and localtime(), otherwise the current time is used.
  109. Optional localtime is a flag that when True, interprets timeval, and
  110. returns a date relative to the local timezone instead of UTC, properly
  111. taking daylight savings time into account.
  112. Optional argument usegmt means that the timezone is written out as
  113. an ascii string, not numeric one (so "GMT" instead of "+0000"). This
  114. is needed for HTTP, and is only used when localtime==False.
  115. """
  116. # Note: we cannot use strftime() because that honors the locale and RFC
  117. # 2822 requires that day and month names be the English abbreviations.
  118. if timeval is None:
  119. timeval = time.time()
  120. if localtime or usegmt:
  121. dt = datetime.datetime.fromtimestamp(timeval, datetime.timezone.utc)
  122. else:
  123. dt = datetime.datetime.utcfromtimestamp(timeval)
  124. if localtime:
  125. dt = dt.astimezone()
  126. usegmt = False
  127. return format_datetime(dt, usegmt)
  128. def format_datetime(dt, usegmt=False):
  129. """Turn a datetime into a date string as specified in RFC 2822.
  130. If usegmt is True, dt must be an aware datetime with an offset of zero. In
  131. this case 'GMT' will be rendered instead of the normal +0000 required by
  132. RFC2822. This is to support HTTP headers involving date stamps.
  133. """
  134. now = dt.timetuple()
  135. if usegmt:
  136. if dt.tzinfo is None or dt.tzinfo != datetime.timezone.utc:
  137. raise ValueError("usegmt option requires a UTC datetime")
  138. zone = 'GMT'
  139. elif dt.tzinfo is None:
  140. zone = '-0000'
  141. else:
  142. zone = dt.strftime("%z")
  143. return _format_timetuple_and_zone(now, zone)
  144. def make_msgid(idstring=None, domain=None):
  145. """Returns a string suitable for RFC 2822 compliant Message-ID, e.g:
  146. <142480216486.20800.16526388040877946887@nightshade.la.mastaler.com>
  147. Optional idstring if given is a string used to strengthen the
  148. uniqueness of the message id. Optional domain if given provides the
  149. portion of the message id after the '@'. It defaults to the locally
  150. defined hostname.
  151. """
  152. timeval = int(time.time()*100)
  153. pid = os.getpid()
  154. randint = random.getrandbits(64)
  155. if idstring is None:
  156. idstring = ''
  157. else:
  158. idstring = '.' + idstring
  159. if domain is None:
  160. domain = socket.getfqdn()
  161. msgid = '<%d.%d.%d%s@%s>' % (timeval, pid, randint, idstring, domain)
  162. return msgid
  163. def parsedate_to_datetime(data):
  164. *dtuple, tz = _parsedate_tz(data)
  165. if tz is None:
  166. return datetime.datetime(*dtuple[:6])
  167. return datetime.datetime(*dtuple[:6],
  168. tzinfo=datetime.timezone(datetime.timedelta(seconds=tz)))
  169. def parseaddr(addr):
  170. """
  171. Parse addr into its constituent realname and email address parts.
  172. Return a tuple of realname and email address, unless the parse fails, in
  173. which case return a 2-tuple of ('', '').
  174. """
  175. addrs = _AddressList(addr).addresslist
  176. if not addrs:
  177. return '', ''
  178. return addrs[0]
  179. # rfc822.unquote() doesn't properly de-backslash-ify in Python pre-2.3.
  180. def unquote(str):
  181. """Remove quotes from a string."""
  182. if len(str) > 1:
  183. if str.startswith('"') and str.endswith('"'):
  184. return str[1:-1].replace('\\\\', '\\').replace('\\"', '"')
  185. if str.startswith('<') and str.endswith('>'):
  186. return str[1:-1]
  187. return str
  188. # RFC2231-related functions - parameter encoding and decoding
  189. def decode_rfc2231(s):
  190. """Decode string according to RFC 2231"""
  191. parts = s.split(TICK, 2)
  192. if len(parts) <= 2:
  193. return None, None, s
  194. return parts
  195. def encode_rfc2231(s, charset=None, language=None):
  196. """Encode string according to RFC 2231.
  197. If neither charset nor language is given, then s is returned as-is. If
  198. charset is given but not language, the string is encoded using the empty
  199. string for language.
  200. """
  201. s = urllib.parse.quote(s, safe='', encoding=charset or 'ascii')
  202. if charset is None and language is None:
  203. return s
  204. if language is None:
  205. language = ''
  206. return "%s'%s'%s" % (charset, language, s)
  207. rfc2231_continuation = re.compile(r'^(?P<name>\w+)\*((?P<num>[0-9]+)\*?)?$',
  208. re.ASCII)
  209. def decode_params(params):
  210. """Decode parameters list according to RFC 2231.
  211. params is a sequence of 2-tuples containing (param name, string value).
  212. """
  213. new_params = [params[0]]
  214. # Map parameter's name to a list of continuations. The values are a
  215. # 3-tuple of the continuation number, the string value, and a flag
  216. # specifying whether a particular segment is %-encoded.
  217. rfc2231_params = {}
  218. for name, value in params[1:]:
  219. encoded = name.endswith('*')
  220. value = unquote(value)
  221. mo = rfc2231_continuation.match(name)
  222. if mo:
  223. name, num = mo.group('name', 'num')
  224. if num is not None:
  225. num = int(num)
  226. rfc2231_params.setdefault(name, []).append((num, value, encoded))
  227. else:
  228. new_params.append((name, '"%s"' % quote(value)))
  229. if rfc2231_params:
  230. for name, continuations in rfc2231_params.items():
  231. value = []
  232. extended = False
  233. # Sort by number
  234. continuations.sort()
  235. # And now append all values in numerical order, converting
  236. # %-encodings for the encoded segments. If any of the
  237. # continuation names ends in a *, then the entire string, after
  238. # decoding segments and concatenating, must have the charset and
  239. # language specifiers at the beginning of the string.
  240. for num, s, encoded in continuations:
  241. if encoded:
  242. # Decode as "latin-1", so the characters in s directly
  243. # represent the percent-encoded octet values.
  244. # collapse_rfc2231_value treats this as an octet sequence.
  245. s = urllib.parse.unquote(s, encoding="latin-1")
  246. extended = True
  247. value.append(s)
  248. value = quote(EMPTYSTRING.join(value))
  249. if extended:
  250. charset, language, value = decode_rfc2231(value)
  251. new_params.append((name, (charset, language, '"%s"' % value)))
  252. else:
  253. new_params.append((name, '"%s"' % value))
  254. return new_params
  255. def collapse_rfc2231_value(value, errors='replace',
  256. fallback_charset='us-ascii'):
  257. if not isinstance(value, tuple) or len(value) != 3:
  258. return unquote(value)
  259. # While value comes to us as a unicode string, we need it to be a bytes
  260. # object. We do not want bytes() normal utf-8 decoder, we want a straight
  261. # interpretation of the string as character bytes.
  262. charset, language, text = value
  263. if charset is None:
  264. # Issue 17369: if charset/lang is None, decode_rfc2231 couldn't parse
  265. # the value, so use the fallback_charset.
  266. charset = fallback_charset
  267. rawbytes = bytes(text, 'raw-unicode-escape')
  268. try:
  269. return str(rawbytes, charset, errors)
  270. except LookupError:
  271. # charset is not a known codec.
  272. return unquote(text)
  273. #
  274. # datetime doesn't provide a localtime function yet, so provide one. Code
  275. # adapted from the patch in issue 9527. This may not be perfect, but it is
  276. # better than not having it.
  277. #
  278. def localtime(dt=None, isdst=-1):
  279. """Return local time as an aware datetime object.
  280. If called without arguments, return current time. Otherwise *dt*
  281. argument should be a datetime instance, and it is converted to the
  282. local time zone according to the system time zone database. If *dt* is
  283. naive (that is, dt.tzinfo is None), it is assumed to be in local time.
  284. In this case, a positive or zero value for *isdst* causes localtime to
  285. presume initially that summer time (for example, Daylight Saving Time)
  286. is or is not (respectively) in effect for the specified time. A
  287. negative value for *isdst* causes the localtime() function to attempt
  288. to divine whether summer time is in effect for the specified time.
  289. """
  290. if dt is None:
  291. return datetime.datetime.now(datetime.timezone.utc).astimezone()
  292. if dt.tzinfo is not None:
  293. return dt.astimezone()
  294. # We have a naive datetime. Convert to a (localtime) timetuple and pass to
  295. # system mktime together with the isdst hint. System mktime will return
  296. # seconds since epoch.
  297. tm = dt.timetuple()[:-1] + (isdst,)
  298. seconds = time.mktime(tm)
  299. localtm = time.localtime(seconds)
  300. try:
  301. delta = datetime.timedelta(seconds=localtm.tm_gmtoff)
  302. tz = datetime.timezone(delta, localtm.tm_zone)
  303. except AttributeError:
  304. # Compute UTC offset and compare with the value implied by tm_isdst.
  305. # If the values match, use the zone name implied by tm_isdst.
  306. delta = dt - datetime.datetime(*time.gmtime(seconds)[:6])
  307. dst = time.daylight and localtm.tm_isdst > 0
  308. gmtoff = -(time.altzone if dst else time.timezone)
  309. if delta == datetime.timedelta(seconds=gmtoff):
  310. tz = datetime.timezone(delta, time.tzname[dst])
  311. else:
  312. tz = datetime.timezone(delta)
  313. return dt.replace(tzinfo=tz)