encoding.py 8.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262
  1. import codecs
  2. import datetime
  3. import locale
  4. from decimal import Decimal
  5. from types import NoneType
  6. from urllib.parse import quote
  7. from django.utils.functional import Promise
  8. class DjangoUnicodeDecodeError(UnicodeDecodeError):
  9. def __str__(self):
  10. return "%s. You passed in %r (%s)" % (
  11. super().__str__(),
  12. self.object,
  13. type(self.object),
  14. )
  15. def smart_str(s, encoding="utf-8", strings_only=False, errors="strict"):
  16. """
  17. Return a string representing 's'. Treat bytestrings using the 'encoding'
  18. codec.
  19. If strings_only is True, don't convert (some) non-string-like objects.
  20. """
  21. if isinstance(s, Promise):
  22. # The input is the result of a gettext_lazy() call.
  23. return s
  24. return force_str(s, encoding, strings_only, errors)
  25. _PROTECTED_TYPES = (
  26. NoneType,
  27. int,
  28. float,
  29. Decimal,
  30. datetime.datetime,
  31. datetime.date,
  32. datetime.time,
  33. )
  34. def is_protected_type(obj):
  35. """Determine if the object instance is of a protected type.
  36. Objects of protected types are preserved as-is when passed to
  37. force_str(strings_only=True).
  38. """
  39. return isinstance(obj, _PROTECTED_TYPES)
  40. def force_str(s, encoding="utf-8", strings_only=False, errors="strict"):
  41. """
  42. Similar to smart_str(), except that lazy instances are resolved to
  43. strings, rather than kept as lazy objects.
  44. If strings_only is True, don't convert (some) non-string-like objects.
  45. """
  46. # Handle the common case first for performance reasons.
  47. if issubclass(type(s), str):
  48. return s
  49. if strings_only and is_protected_type(s):
  50. return s
  51. try:
  52. if isinstance(s, bytes):
  53. s = str(s, encoding, errors)
  54. else:
  55. s = str(s)
  56. except UnicodeDecodeError as e:
  57. raise DjangoUnicodeDecodeError(*e.args) from None
  58. return s
  59. def smart_bytes(s, encoding="utf-8", strings_only=False, errors="strict"):
  60. """
  61. Return a bytestring version of 's', encoded as specified in 'encoding'.
  62. If strings_only is True, don't convert (some) non-string-like objects.
  63. """
  64. if isinstance(s, Promise):
  65. # The input is the result of a gettext_lazy() call.
  66. return s
  67. return force_bytes(s, encoding, strings_only, errors)
  68. def force_bytes(s, encoding="utf-8", strings_only=False, errors="strict"):
  69. """
  70. Similar to smart_bytes, except that lazy instances are resolved to
  71. strings, rather than kept as lazy objects.
  72. If strings_only is True, don't convert (some) non-string-like objects.
  73. """
  74. # Handle the common case first for performance reasons.
  75. if isinstance(s, bytes):
  76. if encoding == "utf-8":
  77. return s
  78. else:
  79. return s.decode("utf-8", errors).encode(encoding, errors)
  80. if strings_only and is_protected_type(s):
  81. return s
  82. if isinstance(s, memoryview):
  83. return bytes(s)
  84. return str(s).encode(encoding, errors)
  85. def iri_to_uri(iri):
  86. """
  87. Convert an Internationalized Resource Identifier (IRI) portion to a URI
  88. portion that is suitable for inclusion in a URL.
  89. This is the algorithm from RFC 3987 Section 3.1, slightly simplified since
  90. the input is assumed to be a string rather than an arbitrary byte stream.
  91. Take an IRI (string or UTF-8 bytes, e.g. '/I ♥ Django/' or
  92. b'/I \xe2\x99\xa5 Django/') and return a string containing the encoded
  93. result with ASCII chars only (e.g. '/I%20%E2%99%A5%20Django/').
  94. """
  95. # The list of safe characters here is constructed from the "reserved" and
  96. # "unreserved" characters specified in RFC 3986 Sections 2.2 and 2.3:
  97. # reserved = gen-delims / sub-delims
  98. # gen-delims = ":" / "/" / "?" / "#" / "[" / "]" / "@"
  99. # sub-delims = "!" / "$" / "&" / "'" / "(" / ")"
  100. # / "*" / "+" / "," / ";" / "="
  101. # unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~"
  102. # Of the unreserved characters, urllib.parse.quote() already considers all
  103. # but the ~ safe.
  104. # The % character is also added to the list of safe characters here, as the
  105. # end of RFC 3987 Section 3.1 specifically mentions that % must not be
  106. # converted.
  107. if iri is None:
  108. return iri
  109. elif isinstance(iri, Promise):
  110. iri = str(iri)
  111. return quote(iri, safe="/#%[]=:;$&()+,!?*@'~")
  112. # List of byte values that uri_to_iri() decodes from percent encoding.
  113. # First, the unreserved characters from RFC 3986:
  114. _ascii_ranges = [[45, 46, 95, 126], range(65, 91), range(97, 123)]
  115. _hextobyte = {
  116. (fmt % char).encode(): bytes((char,))
  117. for ascii_range in _ascii_ranges
  118. for char in ascii_range
  119. for fmt in ["%02x", "%02X"]
  120. }
  121. # And then everything above 128, because bytes ≥ 128 are part of multibyte
  122. # Unicode characters.
  123. _hexdig = "0123456789ABCDEFabcdef"
  124. _hextobyte.update(
  125. {(a + b).encode(): bytes.fromhex(a + b) for a in _hexdig[8:] for b in _hexdig}
  126. )
  127. def uri_to_iri(uri):
  128. """
  129. Convert a Uniform Resource Identifier(URI) into an Internationalized
  130. Resource Identifier(IRI).
  131. This is the algorithm from RFC 3987 Section 3.2, excluding step 4.
  132. Take an URI in ASCII bytes (e.g. '/I%20%E2%99%A5%20Django/') and return
  133. a string containing the encoded result (e.g. '/I%20♥%20Django/').
  134. """
  135. if uri is None:
  136. return uri
  137. uri = force_bytes(uri)
  138. # Fast selective unquote: First, split on '%' and then starting with the
  139. # second block, decode the first 2 bytes if they represent a hex code to
  140. # decode. The rest of the block is the part after '%AB', not containing
  141. # any '%'. Add that to the output without further processing.
  142. bits = uri.split(b"%")
  143. if len(bits) == 1:
  144. iri = uri
  145. else:
  146. parts = [bits[0]]
  147. append = parts.append
  148. hextobyte = _hextobyte
  149. for item in bits[1:]:
  150. hex = item[:2]
  151. if hex in hextobyte:
  152. append(hextobyte[item[:2]])
  153. append(item[2:])
  154. else:
  155. append(b"%")
  156. append(item)
  157. iri = b"".join(parts)
  158. return repercent_broken_unicode(iri).decode()
  159. def escape_uri_path(path):
  160. """
  161. Escape the unsafe characters from the path portion of a Uniform Resource
  162. Identifier (URI).
  163. """
  164. # These are the "reserved" and "unreserved" characters specified in RFC
  165. # 3986 Sections 2.2 and 2.3:
  166. # reserved = ";" | "/" | "?" | ":" | "@" | "&" | "=" | "+" | "$" | ","
  167. # unreserved = alphanum | mark
  168. # mark = "-" | "_" | "." | "!" | "~" | "*" | "'" | "(" | ")"
  169. # The list of safe characters here is constructed subtracting ";", "=",
  170. # and "?" according to RFC 3986 Section 3.3.
  171. # The reason for not subtracting and escaping "/" is that we are escaping
  172. # the entire path, not a path segment.
  173. return quote(path, safe="/:@&+$,-_.!~*'()")
  174. def punycode(domain):
  175. """Return the Punycode of the given domain if it's non-ASCII."""
  176. return domain.encode("idna").decode("ascii")
  177. def repercent_broken_unicode(path):
  178. """
  179. As per RFC 3987 Section 3.2, step three of converting a URI into an IRI,
  180. repercent-encode any octet produced that is not part of a strictly legal
  181. UTF-8 octet sequence.
  182. """
  183. changed_parts = []
  184. while True:
  185. try:
  186. path.decode()
  187. except UnicodeDecodeError as e:
  188. # CVE-2019-14235: A recursion shouldn't be used since the exception
  189. # handling uses massive amounts of memory
  190. repercent = quote(path[e.start : e.end], safe=b"/#%[]=:;$&()+,!?*@'~")
  191. changed_parts.append(path[: e.start] + repercent.encode())
  192. path = path[e.end :]
  193. else:
  194. return b"".join(changed_parts) + path
  195. def filepath_to_uri(path):
  196. """Convert a file system path to a URI portion that is suitable for
  197. inclusion in a URL.
  198. Encode certain chars that would normally be recognized as special chars
  199. for URIs. Do not encode the ' character, as it is a valid character
  200. within URIs. See the encodeURIComponent() JavaScript function for details.
  201. """
  202. if path is None:
  203. return path
  204. # I know about `os.sep` and `os.altsep` but I want to leave
  205. # some flexibility for hardcoding separators.
  206. return quote(str(path).replace("\\", "/"), safe="/~!*()'")
  207. def get_system_encoding():
  208. """
  209. The encoding for the character type functions. Fallback to 'ascii' if the
  210. #encoding is unsupported by Python or could not be determined. See tickets
  211. #10335 and #5846.
  212. """
  213. try:
  214. encoding = locale.getlocale()[1] or "ascii"
  215. codecs.lookup(encoding)
  216. except Exception:
  217. encoding = "ascii"
  218. return encoding
  219. DEFAULT_LOCALE_ENCODING = get_system_encoding()