Coverage for /Users/davegaeddert/Development/dropseed/plain/plain/plain/utils/http.py: 39%

1 import base64

2 import datetime

3 import re

4 import unicodedata

5 from binascii import Error as BinasciiError

6 from email.utils import formatdate

7 from urllib.parse import quote, unquote, urlparse

8 from urllib.parse import urlencode as original_urlencode

9

10 from plain.utils.datastructures import MultiValueDict

11 from plain.utils.regex_helper import _lazy_re_compile

12

13# Based on RFC 9110 Appendix A.

14 ETAG_MATCH = _lazy_re_compile(

15 r"""

16 \A( # start of string and capture group

17 (?:W/)? # optional weak indicator

18 " # opening quote

19 [^"]* # any sequence of non-quote characters

20 " # end quote

21 )\Z # end of string and capture group

22""",

23 re.X,

24)

25

26 MONTHS = "jan feb mar apr may jun jul aug sep oct nov dec".split()

27 __D = r"(?P<day>[0-9]{2})"

28 __D2 = r"(?P<day>[ 0-9][0-9])"

29 __M = r"(?P<mon>\w{3})"

30 __Y = r"(?P<year>[0-9]{4})"

31 __Y2 = r"(?P<year>[0-9]{2})"

32 __T = r"(?P<hour>[0-9]{2}):(?P<min>[0-9]{2}):(?P<sec>[0-9]{2})"

33 RFC1123_DATE = _lazy_re_compile(rf"^\w{ 3} , {__D} {__M} {__Y} {__T} GMT$")

34 RFC850_DATE = _lazy_re_compile(rf"^\w{ 6,9} , {__D}-{__M}-{__Y2} {__T} GMT$")

35 ASCTIME_DATE = _lazy_re_compile(rf"^\w{ 3} {__M} {__D2} {__T} {__Y}$")

36

37 RFC3986_GENDELIMS = ":/?#[]@"

38 RFC3986_SUBDELIMS = "!$&'()*+,;="

39

40

41 def urlencode(query, doseq=False):

42 """

43 A version of Python's urllib.parse.urlencode() function that can operate on

44 MultiValueDict and non-string values.

45 """

46 if isinstance(query, MultiValueDict):

47 query = query.lists()

48 elif hasattr(query, "items"):

49 query = query.items()

50 query_params = []

51 for key, value in query:

52 if value is None:

53 raise TypeError(

54 "Cannot encode None for key '%s' in a query string. Did you "

55 "mean to pass an empty string or omit the value?" % key

56 )

57 elif not doseq or isinstance(value, str | bytes):

58 query_val = value

59 else:

60 try:

61 itr = iter(value)

62 except TypeError:

63 query_val = value

64 else:

65 # Consume generators and iterators, when doseq=True, to

66 # work around https://bugs.python.org/issue31706.

67 query_val = []

68 for item in itr:

69 if item is None:

70 raise TypeError(

71 "Cannot encode None for key '%s' in a query "

72 "string. Did you mean to pass an empty string or "

73 "omit the value?" % key

74 )

75 elif not isinstance(item, bytes):

76 item = str(item)

77 query_val.append(item)

78 query_params.append((key, query_val))

79 return original_urlencode(query_params, doseq)

80

81

82 def http_date(epoch_seconds=None):

83 """

84 Format the time to match the RFC 5322 date format as specified by RFC 9110

85 Section 5.6.7.

86

87 `epoch_seconds` is a floating point number expressed in seconds since the

88 epoch, in UTC - such as that outputted by time.time(). If set to None, it

89 defaults to the current time.

90

91 Output a string in the format 'Wdy, DD Mon YYYY HH:MM:SS GMT'.

92 """

93 return formatdate(epoch_seconds, usegmt=True)

94

95

96 def parse_http_date(date):

97 """

98 Parse a date format as specified by HTTP RFC 9110 Section 5.6.7.

99

100 The three formats allowed by the RFC are accepted, even if only the first

101 one is still in widespread use.

102

103 Return an integer expressed in seconds since the epoch, in UTC.

104 """

105 # email.utils.parsedate() does the job for RFC 1123 dates; unfortunately

106 # RFC 9110 makes it mandatory to support RFC 850 dates too. So we roll

107 # our own RFC-compliant parsing.

108 for regex in RFC1123_DATE, RFC850_DATE, ASCTIME_DATE:

109 m = regex.match(date)

110 if m is not None:

111 break

112 else:

113 raise ValueError("%r is not in a valid HTTP date format" % date)

114 try:

115 tz = datetime.timezone.utc

116 year = int(m["year"])

117 if year < 100:

118 current_year = datetime.datetime.now(tz=tz).year

119 current_century = current_year - (current_year % 100)

120 if year - (current_year % 100) > 50:

121 # year that appears to be more than 50 years in the future are

122 # interpreted as representing the past.

123 year += current_century - 100

124 else:

125 year += current_century

126 month = MONTHS.index(m["mon"].lower()) + 1

127 day = int(m["day"])

128 hour = int(m["hour"])

129 min = int(m["min"])

130 sec = int(m["sec"])

131 result = datetime.datetime(year, month, day, hour, min, sec, tzinfo=tz)

132 return int(result.timestamp())

133 except Exception as exc:

134 raise ValueError("%r is not a valid date" % date) from exc

135

136

137 def parse_http_date_safe(date):

138 """

139 Same as parse_http_date, but return None if the input is invalid.

140 """

141 try:

142 return parse_http_date(date)

143 except Exception:

144 pass

145

146

147# Base 36 functions: useful for generating compact URLs

148

149

150 def base36_to_int(s):

151 """

152 Convert a base 36 string to an int. Raise ValueError if the input won't fit

153 into an int.

154 """

155 # To prevent overconsumption of server resources, reject any

156 # base36 string that is longer than 13 base36 digits (13 digits

157 # is sufficient to base36-encode any 64-bit integer)

158 if len(s) > 13:

159 raise ValueError("Base36 input too large")

160 return int(s, 36)

161

162

163 def int_to_base36(i):

164 """Convert an integer to a base36 string."""

165 char_set = "0123456789abcdefghijklmnopqrstuvwxyz"

166 if i < 0:

167 raise ValueError("Negative base36 conversion input.")

168 if i < 36:

169 return char_set[i]

170 b36 = ""

171 while i != 0:

172 i, n = divmod(i, 36)

173 b36 = char_set[n] + b36

174 return b36

175

176

177 def urlsafe_base64_encode(s):

178 """

179 Encode a bytestring to a base64 string for use in URLs. Strip any trailing

180 equal signs.

181 """

182 return base64.urlsafe_b64encode(s).rstrip(b"\n=").decode("ascii")

183

184

185 def urlsafe_base64_decode(s):

186 """

187 Decode a base64 encoded string. Add back any trailing equal signs that

188 might have been stripped.

189 """

190 s = s.encode()

191 try:

192 return base64.urlsafe_b64decode(s.ljust(len(s) + len(s) % 4, b"="))

193 except (LookupError, BinasciiError) as e:

194 raise ValueError(e)

195

196

197 def parse_etags(etag_str):

198 """

199 Parse a string of ETags given in an If-None-Match or If-Match header as

200 defined by RFC 9110. Return a list of quoted ETags, or ['*'] if all ETags

201 should be matched.

202 """

203 if etag_str.strip() == "*":

204 return ["*"]

205 else:

206 # Parse each ETag individually, and return any that are valid.

207 etag_matches = (ETAG_MATCH.match(etag.strip()) for etag in etag_str.split(","))

208 return [match[1] for match in etag_matches if match]

209

210

211 def quote_etag(etag_str):

212 """

213 If the provided string is already a quoted ETag, return it. Otherwise, wrap

214 the string in quotes, making it a strong ETag.

215 """

216 if ETAG_MATCH.match(etag_str):

217 return etag_str

218 else:

219 return '"%s"' % etag_str

220

221

222 def is_same_domain(host, pattern):

223 """

224 Return ``True`` if the host is either an exact match or a match

225 to the wildcard pattern.

226

227 Any pattern beginning with a period matches a domain and all of its

228 subdomains. (e.g. ``.example.com`` matches ``example.com`` and

229 ``foo.example.com``). Anything else is an exact string match.

230 """

231 if not pattern:

232 return False

233

234 pattern = pattern.lower()

235 return (

236 pattern[0] == "."

237 and (host.endswith(pattern) or host == pattern[1:])

238 or pattern == host

239 )

240

241

242 def url_has_allowed_host_and_scheme(url, allowed_hosts, require_https=False):

243 """

244 Return ``True`` if the url uses an allowed host and a safe scheme.

245

246 Always return ``False`` on an empty url.

247

248 If ``require_https`` is ``True``, only 'https' will be considered a valid

249 scheme, as opposed to 'http' and 'https' with the default, ``False``.

250

251 Note: "True" doesn't entail that a URL is "safe". It may still be e.g.

252 quoted incorrectly. Ensure to also use plain.utils.encoding.iri_to_uri()

253 on the path component of untrusted URLs.

254 """

255 if url is not None:

256 url = url.strip()

257 if not url:

258 return False

259 if allowed_hosts is None:

260 allowed_hosts = set()

261 elif isinstance(allowed_hosts, str):

262 allowed_hosts = {allowed_hosts}

263 # Chrome treats \ completely as / in paths but it could be part of some

264 # basic auth credentials so we need to check both URLs.

265 return _url_has_allowed_host_and_scheme(

266 url, allowed_hosts, require_https=require_https

267 ) and _url_has_allowed_host_and_scheme(

268 url.replace("\\", "/"), allowed_hosts, require_https=require_https

269 )

270

271

272 def _url_has_allowed_host_and_scheme(url, allowed_hosts, require_https=False):

273 # Chrome considers any URL with more than two slashes to be absolute, but

274 # urlparse is not so flexible. Treat any url with three slashes as unsafe.

275 if url.startswith("///"):

276 return False

277 try:

278 url_info = urlparse(url)

279 except ValueError: # e.g. invalid IPv6 addresses

280 return False

281 # Forbid URLs like http:///example.com - with a scheme, but without a hostname.

282 # In that URL, example.com is not the hostname but, a path component. However,

283 # Chrome will still consider example.com to be the hostname, so we must not

284 # allow this syntax.

285 if not url_info.netloc and url_info.scheme:

286 return False

287 # Forbid URLs that start with control characters. Some browsers (like

288 # Chrome) ignore quite a few control characters at the start of a

289 # URL and might consider the URL as scheme relative.

290 if unicodedata.category(url[0])[0] == "C":

291 return False

292 scheme = url_info.scheme

293 # Consider URLs without a scheme (e.g. //example.com/p) to be http.

294 if not url_info.scheme and url_info.netloc:

295 scheme = "http"

296 valid_schemes = ["https"] if require_https else ["http", "https"]

297 return (not url_info.netloc or url_info.netloc in allowed_hosts) and (

298 not scheme or scheme in valid_schemes

299 )

300

301

302 def escape_leading_slashes(url):

303 """

304 If redirecting to an absolute path (two leading slashes), a slash must be

305 escaped to prevent browsers from handling the path as schemaless and

306 redirecting to another host.

307 """

308 if url.startswith("//"):

309 url = "/%2F{}".format(url.removeprefix("//"))

310 return url

311

312

313 def _parseparam(s):

314 while s[:1] == ";":

315 s = s[1:]

316 end = s.find(";")

317 while end > 0 and (s.count('"', 0, end) - s.count('\\"', 0, end)) % 2:

318 end = s.find(";", end + 1)

319 if end < 0:

320 end = len(s)

321 f = s[:end]

322 yield f.strip()

323 s = s[end:]

324

325

326 def parse_header_parameters(line):

327 """

328 Parse a Content-type like header.

329 Return the main content-type and a dictionary of options.

330 """

331 parts = _parseparam(";" + line)

332 key = parts.__next__().lower()

333 pdict = {}

334 for p in parts:

335 i = p.find("=")

336 if i >= 0:

337 has_encoding = False

338 name = p[:i].strip().lower()

339 if name.endswith("*"):

340 # Lang/encoding embedded in the value (like "filename*=UTF-8''file.ext")

341 # https://tools.ietf.org/html/rfc2231#section-4

342 name = name[:-1]

343 if p.count("'") == 2:

344 has_encoding = True

345 value = p[i + 1 :].strip()

346 if len(value) >= 2 and value[0] == value[-1] == '"':

347 value = value[1:-1]

348 value = value.replace("\\\\", "\\").replace('\\"', '"')

349 if has_encoding:

350 encoding, lang, value = value.split("'")

351 value = unquote(value, encoding=encoding)

352 pdict[name] = value

353 return key, pdict

354

355

356 def content_disposition_header(as_attachment, filename):

357 """

358 Construct a Content-Disposition HTTP header value from the given filename

359 as specified by RFC 6266.

360 """

361 if filename:

362 disposition = "attachment" if as_attachment else "inline"

363 try:

364 filename.encode("ascii")

365 file_expr = 'filename="{}"'.format(

366 filename.replace("\\", "\\\\").replace('"', r"\"")

367 )

368 except UnicodeEncodeError:

369 file_expr = f"filename*=utf-8''{quote(filename)}"

370 return f"{disposition}; {file_expr}"

371 elif as_attachment:

372 return "attachment"

373 else:

374 return None