diff --git a/CHANGES/10600.bugfix.rst b/CHANGES/10600.bugfix.rst new file mode 100644 index 00000000000..eba47bf56e6 --- /dev/null +++ b/CHANGES/10600.bugfix.rst @@ -0,0 +1,2 @@ +Fixed http parser not rejecting HTTP/1.1 requests that do not have valid Host header. +-- by :user:`Cycloctane`. diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index 2f528117b1c..245d16d27e3 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -437,6 +437,7 @@ cdef class HttpParser: cdef _on_headers_complete(self): self._process_header() + http_version = self.http_version() should_close = not cparser.llhttp_should_keep_alive(self._cparser) upgrade = self._cparser.upgrade chunked = self._cparser.flags & cparser.F_CHUNKED @@ -453,6 +454,8 @@ cdef class HttpParser: raise BadHttpMessage(f"Duplicate '{bad_hdr}' header found.") if self._cparser.type == cparser.HTTP_REQUEST: + if http_version == HttpVersion11 and hdrs.HOST not in headers: + raise BadHttpMessage("Missing 'Host' header in request.") h_upg = headers.get("upgrade", "") allowed = upgrade and h_upg.isascii() and h_upg.lower() in ALLOWED_UPGRADES if allowed or self._cparser.method == cparser.HTTP_CONNECT: @@ -476,11 +479,11 @@ cdef class HttpParser: method = http_method_str(self._cparser.method) msg = _new_request_message( method, self._path, - self.http_version(), headers, raw_headers, + http_version, headers, raw_headers, should_close, encoding, upgrade, chunked, self._url) else: msg = _new_response_message( - self.http_version(), self._cparser.status_code, self._reason, + http_version, self._cparser.status_code, self._reason, headers, raw_headers, should_close, encoding, upgrade, chunked) diff --git a/aiohttp/http_parser.py b/aiohttp/http_parser.py index d0aee4d75c4..be638f1f965 100644 --- a/aiohttp/http_parser.py +++ b/aiohttp/http_parser.py @@ -41,7 +41,7 @@ LineTooLong, TransferEncodingError, ) -from .http_writer import HttpVersion, HttpVersion10 +from .http_writer import HttpVersion, HttpVersion10, HttpVersion11 from .streams import EMPTY_PAYLOAD, StreamReader from .typedefs import RawHeaders @@ -635,6 +635,9 @@ def parse_message(self, lines: list[bytes]) -> RawRequestMessage: chunked, ) = self.parse_headers(lines[1:]) + if version_o == HttpVersion11 and hdrs.HOST not in headers: + raise BadHttpMessage("Missing 'Host' header in request.") + if close is None: # then the headers weren't set in the request if version_o <= HttpVersion10: # HTTP 1.0 must asks to not close close = True diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 1bf80d271c3..2b647d69149 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -125,6 +125,7 @@ def test_c_parser_loaded() -> None: def test_parse_headers(parser: HttpRequestParser) -> None: text = b"""GET /test HTTP/1.1\r +Host: a\r test: a line\r test2: data\r \r @@ -133,8 +134,16 @@ def test_parse_headers(parser: HttpRequestParser) -> None: assert len(messages) == 1 msg = messages[0][0] - assert list(msg.headers.items()) == [("test", "a line"), ("test2", "data")] - assert msg.raw_headers == ((b"test", b"a line"), (b"test2", b"data")) + assert list(msg.headers.items()) == [ + ("Host", "a"), + ("test", "a line"), + ("test2", "data"), + ] + assert msg.raw_headers == ( + (b"Host", b"a"), + (b"test", b"a line"), + (b"test2", b"data"), + ) assert not msg.should_close assert msg.compression is None assert not msg.upgrade @@ -142,6 +151,7 @@ def test_parse_headers(parser: HttpRequestParser) -> None: def test_reject_obsolete_line_folding(parser: HttpRequestParser) -> None: text = b"""GET /test HTTP/1.1\r +Host: a\r test: line\r Content-Length: 48\r test2: data\r @@ -208,7 +218,7 @@ def test_cve_2023_37276(parser: HttpRequestParser) -> None: def test_bad_header_name( parser: HttpRequestParser, rfc9110_5_6_2_token_delim: str ) -> None: - text = f"POST / HTTP/1.1\r\nhead{rfc9110_5_6_2_token_delim}er: val\r\n\r\n".encode() + text = f"POST / HTTP/1.1\r\nHost: a\r\nhead{rfc9110_5_6_2_token_delim}er: val\r\n\r\n".encode() if rfc9110_5_6_2_token_delim == ":": # Inserting colon into header just splits name/value earlier. parser.feed_data(text) @@ -237,7 +247,7 @@ def test_bad_header_name( ), ) def test_bad_headers(parser: HttpRequestParser, hdr: str) -> None: - text = f"POST / HTTP/1.1\r\n{hdr}\r\n\r\n".encode() + text = f"POST / HTTP/1.1\r\nHost: a\r\n{hdr}\r\n\r\n".encode() with pytest.raises(http_exceptions.BadHttpMessage): parser.feed_data(text) @@ -259,7 +269,7 @@ def test_unpaired_surrogate_in_header_py( max_line_size=8190, max_field_size=8190, ) - text = b"POST / HTTP/1.1\r\n\xff\r\n\r\n" + text = b"POST / HTTP/1.1\r\nHost: a\r\n\xff\r\n\r\n" message = None try: parser.feed_data(text) @@ -318,6 +328,12 @@ def test_duplicate_host_header_rejected(parser: HttpRequestParser) -> None: parser.feed_data(text) +def test_missing_host_header_rejected(parser: HttpRequestParser) -> None: + text = b"GET /admin HTTP/1.1\r\n\r\n" + with pytest.raises(http_exceptions.BadHttpMessage, match="Missing 'Host' header"): + parser.feed_data(text) + + def test_bad_chunked(parser: HttpRequestParser) -> None: """Test that invalid chunked encoding doesn't allow content-length to be used.""" text = ( @@ -329,7 +345,7 @@ def test_bad_chunked(parser: HttpRequestParser) -> None: def test_whitespace_before_header(parser: HttpRequestParser) -> None: - text = b"GET / HTTP/1.1\r\n\tContent-Length: 1\r\n\r\nX" + text = b"GET / HTTP/1.1\r\nHost: a\r\n\tContent-Length: 1\r\n\r\nX" with pytest.raises(http_exceptions.BadHttpMessage): parser.feed_data(text) @@ -360,7 +376,7 @@ def test_parse_unusual_request_line(parser: HttpRequestParser) -> None: def test_parse(parser: HttpRequestParser) -> None: - text = b"GET /test HTTP/1.1\r\n\r\n" + text = b"GET /test HTTP/1.1\r\nHost: a\r\n\r\n" messages, upgrade, tail = parser.feed_data(text) assert len(messages) == 1 msg, _ = messages[0] @@ -369,10 +385,11 @@ def test_parse(parser: HttpRequestParser) -> None: assert msg.method == "GET" assert msg.path == "/test" assert msg.version == (1, 1) + assert msg.headers["Host"] == "a" async def test_parse_body(parser: HttpRequestParser) -> None: - text = b"GET /test HTTP/1.1\r\nContent-Length: 4\r\n\r\nbody" + text = b"GET /test HTTP/1.1\r\nHost: a\r\nContent-Length: 4\r\n\r\nbody" messages, upgrade, tail = parser.feed_data(text) assert len(messages) == 1 _, payload = messages[0] @@ -381,7 +398,7 @@ async def test_parse_body(parser: HttpRequestParser) -> None: async def test_parse_body_with_CRLF(parser: HttpRequestParser) -> None: - text = b"\r\nGET /test HTTP/1.1\r\nContent-Length: 4\r\n\r\nbody" + text = b"\r\nGET /test HTTP/1.1\r\nHost: a\r\nContent-Length: 4\r\n\r\nbody" messages, upgrade, tail = parser.feed_data(text) assert len(messages) == 1 _, payload = messages[0] @@ -390,7 +407,7 @@ async def test_parse_body_with_CRLF(parser: HttpRequestParser) -> None: def test_parse_delayed(parser: HttpRequestParser) -> None: - text = b"GET /test HTTP/1.1\r\n" + text = b"GET /test HTTP/1.1\r\nHost: a\r\n" messages, upgrade, tail = parser.feed_data(text) assert len(messages) == 0 assert not upgrade @@ -403,8 +420,9 @@ def test_parse_delayed(parser: HttpRequestParser) -> None: def test_headers_multi_feed(parser: HttpRequestParser) -> None: text1 = b"GET /test HTTP/1.1\r\n" - text2 = b"test: line" - text3 = b" continue\r\n\r\n" + text2 = b"Host: a\r\n" + text3 = b"test: line" + text4 = b" continue\r\n\r\n" messages, upgrade, tail = parser.feed_data(text1) assert len(messages) == 0 @@ -413,18 +431,21 @@ def test_headers_multi_feed(parser: HttpRequestParser) -> None: assert len(messages) == 0 messages, upgrade, tail = parser.feed_data(text3) + assert len(messages) == 0 + + messages, upgrade, tail = parser.feed_data(text4) assert len(messages) == 1 msg = messages[0][0] - assert list(msg.headers.items()) == [("test", "line continue")] - assert msg.raw_headers == ((b"test", b"line continue"),) + assert list(msg.headers.items()) == [("Host", "a"), ("test", "line continue")] + assert msg.raw_headers == ((b"Host", b"a"), (b"test", b"line continue")) assert not msg.should_close assert msg.compression is None assert not msg.upgrade def test_headers_split_field(parser: HttpRequestParser) -> None: - text1 = b"GET /test HTTP/1.1\r\n" + text1 = b"GET /test HTTP/1.1\r\nHost: a\r\n" text2 = b"t" text3 = b"es" text4 = b"t: value\r\n\r\n" @@ -437,8 +458,8 @@ def test_headers_split_field(parser: HttpRequestParser) -> None: assert len(messages) == 1 msg = messages[0][0] - assert list(msg.headers.items()) == [("test", "value")] - assert msg.raw_headers == ((b"test", b"value"),) + assert list(msg.headers.items()) == [("Host", "a"), ("test", "value")] + assert msg.raw_headers == ((b"Host", b"a"), (b"test", b"value")) assert not msg.should_close assert msg.compression is None assert not msg.upgrade @@ -446,7 +467,7 @@ def test_headers_split_field(parser: HttpRequestParser) -> None: def test_parse_headers_multi(parser: HttpRequestParser) -> None: text = ( - b"GET /test HTTP/1.1\r\n" + b"GET /test HTTP/1.1\r\nHost: a\r\n" b"Set-Cookie: c1=cookie1\r\n" b"Set-Cookie: c2=cookie2\r\n\r\n" ) @@ -456,10 +477,12 @@ def test_parse_headers_multi(parser: HttpRequestParser) -> None: msg = messages[0][0] assert list(msg.headers.items()) == [ + ("Host", "a"), ("Set-Cookie", "c1=cookie1"), ("Set-Cookie", "c2=cookie2"), ] assert msg.raw_headers == ( + (b"Host", b"a"), (b"Set-Cookie", b"c1=cookie1"), (b"Set-Cookie", b"c2=cookie2"), ) @@ -475,14 +498,14 @@ def test_conn_default_1_0(parser: HttpRequestParser) -> None: def test_conn_default_1_1(parser: HttpRequestParser) -> None: - text = b"GET /test HTTP/1.1\r\n\r\n" + text = b"GET /test HTTP/1.1\r\nHost: a\r\n\r\n" messages, upgrade, tail = parser.feed_data(text) msg = messages[0][0] assert not msg.should_close def test_conn_close(parser: HttpRequestParser) -> None: - text = b"GET /test HTTP/1.1\r\nconnection: close\r\n\r\n" + text = b"GET /test HTTP/1.1\r\nHost: a\r\nconnection: close\r\n\r\n" messages, upgrade, tail = parser.feed_data(text) msg = messages[0][0] assert msg.should_close @@ -503,14 +526,14 @@ def test_conn_keep_alive_1_0(parser: HttpRequestParser) -> None: def test_conn_keep_alive_1_1(parser: HttpRequestParser) -> None: - text = b"GET /test HTTP/1.1\r\nconnection: keep-alive\r\n\r\n" + text = b"GET /test HTTP/1.1\r\nHost: a\r\nconnection: keep-alive\r\n\r\n" messages, upgrade, tail = parser.feed_data(text) msg = messages[0][0] assert not msg.should_close def test_conn_close_comma_list(parser: HttpRequestParser) -> None: - text = b"GET /test HTTP/1.1\r\nconnection: close, keep-alive\r\n\r\n" + text = b"GET /test HTTP/1.1\r\nHost: a\r\nconnection: close, keep-alive\r\n\r\n" messages, upgrade, tail = parser.feed_data(text) msg = messages[0][0] assert msg.should_close @@ -519,6 +542,7 @@ def test_conn_close_comma_list(parser: HttpRequestParser) -> None: def test_conn_close_multiple_headers(parser: HttpRequestParser) -> None: text = ( b"GET /test HTTP/1.1\r\n" + b"Host: a\r\n" b"connection: keep-alive\r\n" b"connection: close\r\n\r\n" ) @@ -535,14 +559,14 @@ def test_conn_other_1_0(parser: HttpRequestParser) -> None: def test_conn_other_1_1(parser: HttpRequestParser) -> None: - text = b"GET /test HTTP/1.1\r\nconnection: test\r\n\r\n" + text = b"GET /test HTTP/1.1\r\nHost: a\r\nconnection: test\r\n\r\n" messages, upgrade, tail = parser.feed_data(text) msg = messages[0][0] assert not msg.should_close def test_request_chunked(parser: HttpRequestParser) -> None: - text = b"GET /test HTTP/1.1\r\ntransfer-encoding: chunked\r\n\r\n" + text = b"GET /test HTTP/1.1\r\nHost: a\r\ntransfer-encoding: chunked\r\n\r\n" messages, upgrade, tail = parser.feed_data(text) msg, payload = messages[0] assert msg.chunked @@ -552,14 +576,14 @@ def test_request_chunked(parser: HttpRequestParser) -> None: def test_te_header_non_ascii(parser: HttpRequestParser) -> None: # K = Kelvin sign, not valid ascii. - text = "GET /test HTTP/1.1\r\nTransfer-Encoding: chunKed\r\n\r\n" + text = "GET /test HTTP/1.1\r\nHost: a\r\nTransfer-Encoding: chunKed\r\n\r\n" with pytest.raises(http_exceptions.BadHttpMessage): parser.feed_data(text.encode()) def test_upgrade_header_non_ascii(parser: HttpRequestParser) -> None: # K = Kelvin sign, not valid ascii. - text = "GET /test HTTP/1.1\r\nUpgrade: websocKet\r\n\r\n" + text = "GET /test HTTP/1.1\r\nHost: a\r\nUpgrade: websocKet\r\n\r\n" messages, upgrade, tail = parser.feed_data(text.encode()) assert not upgrade @@ -567,6 +591,7 @@ def test_upgrade_header_non_ascii(parser: HttpRequestParser) -> None: def test_request_te_chunked_with_content_length(parser: HttpRequestParser) -> None: text = ( b"GET /test HTTP/1.1\r\n" + b"Host: a\r\n" b"content-length: 1234\r\n" b"transfer-encoding: chunked\r\n\r\n" ) @@ -578,7 +603,7 @@ def test_request_te_chunked_with_content_length(parser: HttpRequestParser) -> No def test_request_te_chunked123(parser: HttpRequestParser) -> None: - text = b"GET /test HTTP/1.1\r\ntransfer-encoding: chunked123\r\n\r\n" + text = b"GET /test HTTP/1.1\r\nHost: a\r\ntransfer-encoding: chunked123\r\n\r\n" with pytest.raises( http_exceptions.BadHttpMessage, match="Request has invalid `Transfer-Encoding`", @@ -587,14 +612,14 @@ def test_request_te_chunked123(parser: HttpRequestParser) -> None: async def test_request_te_last_chunked(parser: HttpRequestParser) -> None: - text = b"GET /test HTTP/1.1\r\nTransfer-Encoding: not, chunked\r\n\r\n1\r\nT\r\n3\r\nest\r\n0\r\n\r\n" + text = b"GET /test HTTP/1.1\r\nHost: a\r\nTransfer-Encoding: not, chunked\r\n\r\n1\r\nT\r\n3\r\nest\r\n0\r\n\r\n" messages, upgrade, tail = parser.feed_data(text) # https://www.rfc-editor.org/rfc/rfc9112#section-6.3-2.4.3 assert await messages[0][1].read() == b"Test" def test_request_te_first_chunked(parser: HttpRequestParser) -> None: - text = b"GET /test HTTP/1.1\r\nTransfer-Encoding: chunked, not\r\n\r\n1\r\nT\r\n3\r\nest\r\n0\r\n\r\n" + text = b"GET /test HTTP/1.1\r\nHost: a\r\nTransfer-Encoding: chunked, not\r\n\r\n1\r\nT\r\n3\r\nest\r\n0\r\n\r\n" # https://www.rfc-editor.org/rfc/rfc9112#section-6.3-2.4.3 with pytest.raises( http_exceptions.BadHttpMessage, @@ -617,6 +642,7 @@ def test_request_te_duplicate_chunked(parser: HttpRequestParser) -> None: def test_conn_upgrade(parser: HttpRequestParser) -> None: text = ( b"GET /test HTTP/1.1\r\n" + b"Host: a\r\n" b"connection: upgrade\r\n" b"upgrade: websocket\r\n\r\n" ) @@ -630,6 +656,7 @@ def test_conn_upgrade(parser: HttpRequestParser) -> None: def test_conn_upgrade_comma_list(parser: HttpRequestParser) -> None: text = ( b"GET /test HTTP/1.1\r\n" + b"host: a\r\n" b"connection: keep-alive, upgrade\r\n" b"upgrade: websocket\r\n\r\n" ) @@ -643,6 +670,7 @@ def test_conn_upgrade_comma_list(parser: HttpRequestParser) -> None: def test_conn_upgrade_multiple_headers(parser: HttpRequestParser) -> None: text = ( b"GET /test HTTP/1.1\r\n" + b"host: a\r\n" b"connection: keep-alive\r\n" b"connection: upgrade\r\n" b"upgrade: websocket\r\n\r\n" @@ -656,7 +684,7 @@ def test_conn_upgrade_multiple_headers(parser: HttpRequestParser) -> None: def test_bad_upgrade(parser: HttpRequestParser) -> None: """Test not upgraded if missing Upgrade header.""" - text = b"GET /test HTTP/1.1\r\nconnection: upgrade\r\n\r\n" + text = b"GET /test HTTP/1.1\r\nHost: a\r\nconnection: upgrade\r\n\r\n" messages, upgrade, tail = parser.feed_data(text) msg = messages[0][0] assert not msg.upgrade @@ -664,21 +692,21 @@ def test_bad_upgrade(parser: HttpRequestParser) -> None: def test_compression_empty(parser: HttpRequestParser) -> None: - text = b"GET /test HTTP/1.1\r\ncontent-encoding: \r\n\r\n" + text = b"GET /test HTTP/1.1\r\nHost: a\r\ncontent-encoding: \r\n\r\n" messages, upgrade, tail = parser.feed_data(text) msg = messages[0][0] assert msg.compression is None def test_compression_deflate(parser: HttpRequestParser) -> None: - text = b"GET /test HTTP/1.1\r\ncontent-encoding: deflate\r\n\r\n" + text = b"GET /test HTTP/1.1\r\nHost: a\r\ncontent-encoding: deflate\r\n\r\n" messages, upgrade, tail = parser.feed_data(text) msg = messages[0][0] assert msg.compression == "deflate" def test_compression_gzip(parser: HttpRequestParser) -> None: - text = b"GET /test HTTP/1.1\r\ncontent-encoding: gzip\r\n\r\n" + text = b"GET /test HTTP/1.1\r\nHost: a\r\ncontent-encoding: gzip\r\n\r\n" messages, upgrade, tail = parser.feed_data(text) msg = messages[0][0] assert msg.compression == "gzip" @@ -686,7 +714,7 @@ def test_compression_gzip(parser: HttpRequestParser) -> None: @pytest.mark.skipif(brotli is None, reason="brotli is not installed") def test_compression_brotli(parser: HttpRequestParser) -> None: - text = b"GET /test HTTP/1.1\r\ncontent-encoding: br\r\n\r\n" + text = b"GET /test HTTP/1.1\r\nHost: a\r\ncontent-encoding: br\r\n\r\n" messages, upgrade, tail = parser.feed_data(text) msg = messages[0][0] assert msg.compression == "br" @@ -694,7 +722,7 @@ def test_compression_brotli(parser: HttpRequestParser) -> None: @pytest.mark.skipif(zstandard is None, reason="zstandard is not installed") def test_compression_zstd(parser: HttpRequestParser) -> None: - text = b"GET /test HTTP/1.1\r\ncontent-encoding: zstd\r\n\r\n" + text = b"GET /test HTTP/1.1\r\nHost: a\r\ncontent-encoding: zstd\r\n\r\n" messages, upgrade, tail = parser.feed_data(text) msg = messages[0][0] assert msg.compression == "zstd" @@ -708,7 +736,7 @@ def test_compression_zstd(parser: HttpRequestParser) -> None: ), ) def test_compression_non_ascii(parser: HttpRequestParser, enc: bytes) -> None: - text = b"GET /test HTTP/1.1\r\ncontent-encoding: " + enc + b"\r\n\r\n" + text = b"GET /test HTTP/1.1\r\nHost: a\r\ncontent-encoding: " + enc + b"\r\n\r\n" messages, upgrade, tail = parser.feed_data(text) msg = messages[0][0] # Non-ascii input should not evaluate to a valid encoding scheme. @@ -716,14 +744,14 @@ def test_compression_non_ascii(parser: HttpRequestParser, enc: bytes) -> None: def test_compression_unknown(parser: HttpRequestParser) -> None: - text = b"GET /test HTTP/1.1\r\ncontent-encoding: compress\r\n\r\n" + text = b"GET /test HTTP/1.1\r\nHost: a\r\ncontent-encoding: compress\r\n\r\n" messages, upgrade, tail = parser.feed_data(text) msg = messages[0][0] assert msg.compression is None def test_url_connect(parser: HttpRequestParser) -> None: - text = b"CONNECT www.google.com HTTP/1.1\r\ncontent-length: 0\r\n\r\n" + text = b"CONNECT www.google.com HTTP/1.1\r\nHost: a\r\ncontent-length: 0\r\n\r\n" messages, upgrade, tail = parser.feed_data(text) msg, payload = messages[0] assert upgrade @@ -731,7 +759,7 @@ def test_url_connect(parser: HttpRequestParser) -> None: def test_headers_connect(parser: HttpRequestParser) -> None: - text = b"CONNECT www.google.com HTTP/1.1\r\ncontent-length: 0\r\n\r\n" + text = b"CONNECT www.google.com HTTP/1.1\r\nHost: a\r\ncontent-length: 0\r\n\r\n" messages, upgrade, tail = parser.feed_data(text) msg, payload = messages[0] assert upgrade @@ -741,6 +769,7 @@ def test_headers_connect(parser: HttpRequestParser) -> None: def test_url_absolute(parser: HttpRequestParser) -> None: text = ( b"GET https://www.google.com/path/to.html HTTP/1.1\r\n" + b"Host: a\r\n" b"content-length: 0\r\n\r\n" ) messages, upgrade, tail = parser.feed_data(text) @@ -751,21 +780,21 @@ def test_url_absolute(parser: HttpRequestParser) -> None: def test_headers_old_websocket_key1(parser: HttpRequestParser) -> None: - text = b"GET /test HTTP/1.1\r\nSEC-WEBSOCKET-KEY1: line\r\n\r\n" + text = b"GET /test HTTP/1.1\r\nHost: a\r\nSEC-WEBSOCKET-KEY1: line\r\n\r\n" with pytest.raises(http_exceptions.BadHttpMessage): parser.feed_data(text) def test_headers_content_length_err_1(parser: HttpRequestParser) -> None: - text = b"GET /test HTTP/1.1\r\ncontent-length: line\r\n\r\n" + text = b"GET /test HTTP/1.1\r\nHost: a\r\ncontent-length: line\r\n\r\n" with pytest.raises(http_exceptions.BadHttpMessage): parser.feed_data(text) def test_headers_content_length_err_2(parser: HttpRequestParser) -> None: - text = b"GET /test HTTP/1.1\r\ncontent-length: -1\r\n\r\n" + text = b"GET /test HTTP/1.1\r\nHost: a\r\ncontent-length: -1\r\n\r\n" with pytest.raises(http_exceptions.BadHttpMessage): parser.feed_data(text) @@ -790,7 +819,7 @@ def test_headers_content_length_err_2(parser: HttpRequestParser) -> None: def test_invalid_header_spacing( parser: HttpRequestParser, pad1: bytes, pad2: bytes, hdr: bytes ) -> None: - text = b"GET /test HTTP/1.1\r\n%s%s%s: value\r\n\r\n" % (pad1, hdr, pad2) + text = b"GET /test HTTP/1.1\r\nHost: a\r\n%s%s%s: value\r\n\r\n" % (pad1, hdr, pad2) if pad1 == pad2 == b"" and hdr != b"": # one entry in param matrix is correct: non-empty name, not padded parser.feed_data(text) @@ -801,19 +830,19 @@ def test_invalid_header_spacing( def test_empty_header_name(parser: HttpRequestParser) -> None: - text = b"GET /test HTTP/1.1\r\n:test\r\n\r\n" + text = b"GET /test HTTP/1.1\r\nHost: a\r\n:test\r\n\r\n" with pytest.raises(http_exceptions.BadHttpMessage): parser.feed_data(text) def test_invalid_header(parser: HttpRequestParser) -> None: - text = b"GET /test HTTP/1.1\r\ntest line\r\n\r\n" + text = b"GET /test HTTP/1.1\r\nHost: a\r\ntest line\r\n\r\n" with pytest.raises(http_exceptions.BadHttpMessage): parser.feed_data(text) def test_invalid_name(parser: HttpRequestParser) -> None: - text = b"GET /test HTTP/1.1\r\ntest[]: line\r\n\r\n" + text = b"GET /test HTTP/1.1\r\nHost: a\r\ntest[]: line\r\n\r\n" with pytest.raises(http_exceptions.BadHttpMessage): parser.feed_data(text) @@ -832,15 +861,15 @@ def test_max_header_field_size(parser: HttpRequestParser, size: int) -> None: def test_max_header_size_under_limit(parser: HttpRequestParser) -> None: name = b"t" * 8185 - text = b"GET /test HTTP/1.1\r\n" + name + b":data\r\n\r\n" + text = b"GET /test HTTP/1.1\r\nHost: a\r\n" + name + b":data\r\n\r\n" messages, upgrade, tail = parser.feed_data(text) msg = messages[0][0] assert msg.method == "GET" assert msg.path == "/test" assert msg.version == (1, 1) - assert msg.headers == CIMultiDict({name.decode(): "data"}) - assert msg.raw_headers == ((name, b"data"),) + assert msg.headers == CIMultiDict([("Host", "a"), (name.decode(), "data")]) + assert msg.raw_headers == ((b"Host", b"a"), (name, b"data")) assert not msg.should_close assert msg.compression is None assert not msg.upgrade @@ -872,7 +901,7 @@ def test_max_header_combined_size(parser: HttpRequestParser) -> None: async def test_max_trailer_size(parser: HttpRequestParser, size: int) -> None: value = b"t" * size text = ( - b"GET /test HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n" + b"GET /test HTTP/1.1\r\nHost: a\r\nTransfer-Encoding: chunked\r\n\r\n" + hex(4000)[2:].encode() + b"\r\n" + b"b" * 4000 @@ -898,7 +927,7 @@ async def test_max_headers( parser: HttpRequestParser, headers: int, trailers: int ) -> None: text = ( - b"GET /test HTTP/1.1\r\nTransfer-Encoding: chunked" + b"GET /test HTTP/1.1\r\nHost: a\r\nTransfer-Encoding: chunked" + b"".join(b"\r\nHeader-%d: Value" % i for i in range(headers)) + b"\r\n\r\n4\r\ntest\r\n0" + b"".join(b"\r\nTrailer-%d: Value" % i for i in range(trailers)) @@ -914,15 +943,15 @@ async def test_max_headers( def test_max_header_value_size_under_limit(parser: HttpRequestParser) -> None: value = b"A" * 8185 - text = b"GET /test HTTP/1.1\r\ndata:" + value + b"\r\n\r\n" + text = b"GET /test HTTP/1.1\r\nHost: a\r\ndata:" + value + b"\r\n\r\n" messages, upgrade, tail = parser.feed_data(text) msg = messages[0][0] assert msg.method == "GET" assert msg.path == "/test" assert msg.version == (1, 1) - assert msg.headers == CIMultiDict({"data": value.decode()}) - assert msg.raw_headers == ((b"data", value),) + assert msg.headers == CIMultiDict([("Host", "a"), ("data", value.decode())]) + assert msg.raw_headers == ((b"Host", b"a"), (b"data", value)) assert not msg.should_close assert msg.compression is None assert not msg.upgrade @@ -963,15 +992,15 @@ def test_max_header_value_size_continuation_under_limit( def test_http_request_parser(parser: HttpRequestParser) -> None: - text = b"GET /path HTTP/1.1\r\n\r\n" + text = b"GET /path HTTP/1.1\r\nHost: a\r\n\r\n" messages, upgrade, tail = parser.feed_data(text) msg = messages[0][0] assert msg.method == "GET" assert msg.path == "/path" assert msg.version == (1, 1) - assert msg.headers == CIMultiDict() - assert msg.raw_headers == () + assert msg.headers == CIMultiDict({"Host": "a"}) + assert msg.raw_headers == ((b"Host", b"a"),) assert not msg.should_close assert msg.compression is None assert not msg.upgrade @@ -1002,7 +1031,7 @@ def test_http_request_bad_status_line(parser: HttpRequestParser) -> None: def test_http_request_bad_status_line_number( parser: HttpRequestParser, nonascii_digit: bytes ) -> None: - text = b"GET /digit HTTP/1." + nonascii_digit + b"\r\n\r\n" + text = b"GET /digit HTTP/1." + nonascii_digit + b"\r\nHost: a\r\n\r\n" with pytest.raises(http_exceptions.BadStatusLine): parser.feed_data(text) @@ -1010,19 +1039,19 @@ def test_http_request_bad_status_line_number( def test_http_request_bad_status_line_separator(parser: HttpRequestParser) -> None: # single code point, old, multibyte NFKC, multibyte NFKD utf8sep = "\N{ARABIC LIGATURE SALLALLAHOU ALAYHE WASALLAM}".encode() - text = b"GET /ligature HTTP/1" + utf8sep + b"1\r\n\r\n" + text = b"GET /ligature HTTP/1" + utf8sep + b"1\r\nHost: a\r\n\r\n" with pytest.raises(http_exceptions.BadStatusLine): parser.feed_data(text) def test_http_request_bad_status_line_whitespace(parser: HttpRequestParser) -> None: - text = b"GET\n/path\fHTTP/1.1\r\n\r\n" + text = b"GET\n/path\fHTTP/1.1\r\nHost: a\r\n\r\n" with pytest.raises(http_exceptions.BadStatusLine): parser.feed_data(text) def test_http_request_message_after_close(parser: HttpRequestParser) -> None: - text = b"GET / HTTP/1.1\r\nConnection: close\r\n\r\nInvalid\r\n\r\n" + text = b"GET / HTTP/1.1\r\nHost: a\r\nConnection: close\r\n\r\nInvalid\r\n\r\n" with pytest.raises( http_exceptions.BadHttpMessage, match="Data after `Connection: close`" ): @@ -1030,7 +1059,7 @@ def test_http_request_message_after_close(parser: HttpRequestParser) -> None: def test_http_request_message_after_close_comma_list(parser: HttpRequestParser) -> None: - text = b"GET / HTTP/1.1\r\nConnection: close, keep-alive\r\n\r\nInvalid\r\n\r\n" + text = b"GET / HTTP/1.1\r\nHost: a\r\nConnection: close, keep-alive\r\n\r\nInvalid\r\n\r\n" with pytest.raises( http_exceptions.BadHttpMessage, match="Data after `Connection: close`" ): @@ -1040,6 +1069,7 @@ def test_http_request_message_after_close_comma_list(parser: HttpRequestParser) def test_http_request_upgrade(parser: HttpRequestParser) -> None: text = ( b"GET /test HTTP/1.1\r\n" + b"Host: a\r\n" b"connection: upgrade\r\n" b"upgrade: websocket\r\n\r\n" b"some raw data" @@ -1055,6 +1085,7 @@ def test_http_request_upgrade(parser: HttpRequestParser) -> None: async def test_http_request_upgrade_unknown(parser: HttpRequestParser) -> None: text = ( b"POST / HTTP/1.1\r\n" + b"Host: a\r\n" b"Connection: Upgrade\r\n" b"Content-Length: 2\r\n" b"Upgrade: unknown\r\n" @@ -1088,7 +1119,7 @@ def xfail_c_parser_url(request: pytest.FixtureRequest) -> None: def test_http_request_parser_utf8_request_line(parser: HttpRequestParser) -> None: messages, upgrade, tail = parser.feed_data( # note the truncated unicode sequence - b"GET /P\xc3\xbcnktchen\xa0\xef\xb7 HTTP/1.1\r\n" + + b"GET /P\xc3\xbcnktchen\xa0\xef\xb7 HTTP/1.1\r\nHost: a\r\n" + # for easier grep: ASCII 0xA0 more commonly known as non-breaking space # note the leading and trailing spaces "sTeP: \N{LATIN SMALL LETTER SHARP S}nek\t\N{NO-BREAK SPACE} " @@ -1099,8 +1130,8 @@ def test_http_request_parser_utf8_request_line(parser: HttpRequestParser) -> Non assert msg.method == "GET" assert msg.path == "/Pünktchen\udca0\udcef\udcb7" assert msg.version == (1, 1) - assert msg.headers == CIMultiDict([("STEP", "ßnek\t\xa0")]) - assert msg.raw_headers == ((b"sTeP", "ßnek\t\xa0".encode()),) + assert msg.headers == CIMultiDict([("Host", "a"), ("STEP", "ßnek\t\xa0")]) + assert msg.raw_headers == ((b"Host", b"a"), (b"sTeP", "ßnek\t\xa0".encode())) assert not msg.should_close assert msg.compression is None assert not msg.upgrade @@ -1111,15 +1142,15 @@ def test_http_request_parser_utf8_request_line(parser: HttpRequestParser) -> Non def test_http_request_parser_utf8(parser: HttpRequestParser) -> None: - text = "GET /path HTTP/1.1\r\nx-test:тест\r\n\r\n".encode() + text = "GET /path HTTP/1.1\r\nHost: a\r\nx-test:тест\r\n\r\n".encode() messages, upgrade, tail = parser.feed_data(text) msg = messages[0][0] assert msg.method == "GET" assert msg.path == "/path" assert msg.version == (1, 1) - assert msg.headers == CIMultiDict([("X-TEST", "тест")]) - assert msg.raw_headers == ((b"x-test", "тест".encode()),) + assert msg.headers == CIMultiDict([("Host", "a"), ("X-TEST", "тест")]) + assert msg.raw_headers == ((b"Host", b"a"), (b"x-test", "тест".encode())) assert not msg.should_close assert msg.compression is None assert not msg.upgrade @@ -1128,16 +1159,19 @@ def test_http_request_parser_utf8(parser: HttpRequestParser) -> None: def test_http_request_parser_non_utf8(parser: HttpRequestParser) -> None: - text = "GET /path HTTP/1.1\r\nx-test:тест\r\n\r\n".encode("cp1251") + text = "GET /path HTTP/1.1\r\nHost: a\r\nx-test:тест\r\n\r\n".encode("cp1251") msg = parser.feed_data(text)[0][0][0] assert msg.method == "GET" assert msg.path == "/path" assert msg.version == (1, 1) assert msg.headers == CIMultiDict( - [("X-TEST", "тест".encode("cp1251").decode("utf8", "surrogateescape"))] + [ + ("Host", "a"), + ("X-TEST", "тест".encode("cp1251").decode("utf8", "surrogateescape")), + ] ) - assert msg.raw_headers == ((b"x-test", "тест".encode("cp1251")),) + assert msg.raw_headers == ((b"Host", b"a"), (b"x-test", "тест".encode("cp1251"))) assert not msg.should_close assert msg.compression is None assert not msg.upgrade @@ -1146,7 +1180,7 @@ def test_http_request_parser_non_utf8(parser: HttpRequestParser) -> None: def test_http_request_parser_two_slashes(parser: HttpRequestParser) -> None: - text = b"GET //path HTTP/1.1\r\n\r\n" + text = b"GET //path HTTP/1.1\r\nHost: a\r\n\r\n" msg = parser.feed_data(text)[0][0][0] assert msg.method == "GET" @@ -1167,17 +1201,19 @@ def test_http_request_parser_bad_method( parser: HttpRequestParser, rfc9110_5_6_2_token_delim: bytes ) -> None: with pytest.raises(http_exceptions.BadHttpMethod): - parser.feed_data(rfc9110_5_6_2_token_delim + b'ET" /get HTTP/1.1\r\n\r\n') + parser.feed_data( + rfc9110_5_6_2_token_delim + b'ET" /get HTTP/1.1\r\nHost: a\r\n\r\n' + ) def test_http_request_parser_bad_version(parser: HttpRequestParser) -> None: with pytest.raises(http_exceptions.BadHttpMessage): - parser.feed_data(b"GET //get HT/11\r\n\r\n") + parser.feed_data(b"GET //get HT/11\r\nHost: a\r\n\r\n") def test_http_request_parser_bad_version_number(parser: HttpRequestParser) -> None: with pytest.raises(http_exceptions.BadHttpMessage): - parser.feed_data(b"GET /test HTTP/1.32\r\n\r\n") + parser.feed_data(b"GET /test HTTP/1.32\r\nHost: a\r\n\r\n") def test_http_request_parser_bad_ascii_uri(parser: HttpRequestParser) -> None: @@ -1201,15 +1237,15 @@ def test_http_request_max_status_line(parser: HttpRequestParser, size: int) -> N def test_http_request_max_status_line_under_limit(parser: HttpRequestParser) -> None: path = b"t" * 8172 messages, upgraded, tail = parser.feed_data( - b"GET /path" + path + b" HTTP/1.1\r\n\r\n" + b"GET /path" + path + b" HTTP/1.1\r\nHost: a\r\n\r\n" ) msg = messages[0][0] assert msg.method == "GET" assert msg.path == "/path" + path.decode() assert msg.version == (1, 1) - assert msg.headers == CIMultiDict() - assert msg.raw_headers == () + assert msg.headers == CIMultiDict({"Host": "a"}) + assert msg.raw_headers == ((b"Host", b"a"),) assert not msg.should_close assert msg.compression is None assert not msg.upgrade @@ -1452,7 +1488,7 @@ def test_http_response_parser_code_not_ascii( def test_http_request_chunked_payload(parser: HttpRequestParser) -> None: - text = b"GET /test HTTP/1.1\r\ntransfer-encoding: chunked\r\n\r\n" + text = b"GET /test HTTP/1.1\r\nHost: a\r\ntransfer-encoding: chunked\r\n\r\n" msg, payload = parser.feed_data(text)[0][0] assert msg.chunked @@ -1470,12 +1506,13 @@ def test_http_request_chunked_payload(parser: HttpRequestParser) -> None: def test_http_request_chunked_payload_and_next_message( parser: HttpRequestParser, ) -> None: - text = b"GET /test HTTP/1.1\r\ntransfer-encoding: chunked\r\n\r\n" + text = b"GET /test HTTP/1.1\r\nHost: a\r\ntransfer-encoding: chunked\r\n\r\n" msg, payload = parser.feed_data(text)[0][0] messages, upgraded, tail = parser.feed_data( b"4\r\ndata\r\n4\r\nline\r\n0\r\n\r\n" b"POST /test2 HTTP/1.1\r\n" + b"Host: a\r\n" b"transfer-encoding: chunked\r\n\r\n" ) @@ -1493,7 +1530,7 @@ def test_http_request_chunked_payload_and_next_message( def test_http_request_chunked_payload_chunks(parser: HttpRequestParser) -> None: - text = b"GET /test HTTP/1.1\r\ntransfer-encoding: chunked\r\n\r\n" + text = b"GET /test HTTP/1.1\r\nHost: a\r\ntransfer-encoding: chunked\r\n\r\n" msg, payload = parser.feed_data(text)[0][0] parser.feed_data(b"4\r\ndata\r") @@ -1516,7 +1553,7 @@ def test_http_request_chunked_payload_chunks(parser: HttpRequestParser) -> None: def test_parse_chunked_payload_chunk_extension(parser: HttpRequestParser) -> None: - text = b"GET /test HTTP/1.1\r\ntransfer-encoding: chunked\r\n\r\n" + text = b"GET /test HTTP/1.1\r\nHost: a\r\ntransfer-encoding: chunked\r\n\r\n" msg, payload = parser.feed_data(text)[0][0] parser.feed_data(b"4;test\r\ndata\r\n4\r\nline\r\n0\r\ntest: test\r\n\r\n") @@ -1528,7 +1565,7 @@ def test_parse_chunked_payload_chunk_extension(parser: HttpRequestParser) -> Non async def test_request_chunked_with_trailer(parser: HttpRequestParser) -> None: - text = b"GET /test HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n4\r\ntest\r\n0\r\ntest: trailer\r\nsecond: test trailer\r\n\r\n" + text = b"GET /test HTTP/1.1\r\nHost: a\r\nTransfer-Encoding: chunked\r\n\r\n4\r\ntest\r\n0\r\ntest: trailer\r\nsecond: test trailer\r\n\r\n" messages, upgraded, tail = parser.feed_data(text) assert not tail msg, payload = messages[0] @@ -1538,7 +1575,7 @@ async def test_request_chunked_with_trailer(parser: HttpRequestParser) -> None: async def test_request_chunked_reject_bad_trailer(parser: HttpRequestParser) -> None: - text = b"GET /test HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n0\r\nbad\ntrailer\r\n\r\n" + text = b"GET /test HTTP/1.1\r\nHost: a\r\nTransfer-Encoding: chunked\r\n\r\n0\r\nbad\ntrailer\r\n\r\n" with pytest.raises(http_exceptions.BadHttpMessage, match=r"b'bad\\ntrailer'"): parser.feed_data(text) @@ -1549,7 +1586,7 @@ def test_parse_no_length_or_te_on_post( request_cls: type[HttpRequestParser], ) -> None: parser = request_cls(protocol, loop, limit=2**16) - text = b"POST /test HTTP/1.1\r\n\r\n" + text = b"POST /test HTTP/1.1\r\nHost: a\r\n\r\n" msg, payload = parser.feed_data(text)[0][0] assert payload.is_eof() @@ -1581,7 +1618,7 @@ def test_parse_length_payload(response: HttpResponseParser) -> None: def test_parse_no_length_payload(parser: HttpRequestParser) -> None: - text = b"PUT / HTTP/1.1\r\n\r\n" + text = b"PUT / HTTP/1.1\r\nHost: a\r\n\r\n" msg, payload = parser.feed_data(text)[0][0] assert payload.is_eof() @@ -1753,7 +1790,7 @@ async def test_parse_chunked_payload_with_lf_in_extensions( def test_partial_url(parser: HttpRequestParser) -> None: messages, upgrade, tail = parser.feed_data(b"GET /te") assert len(messages) == 0 - messages, upgrade, tail = parser.feed_data(b"st HTTP/1.1\r\n\r\n") + messages, upgrade, tail = parser.feed_data(b"st HTTP/1.1\r\nHost: a\r\n\r\n") assert len(messages) == 1 msg, payload = messages[0] @@ -1778,7 +1815,7 @@ def test_partial_url(parser: HttpRequestParser) -> None: def test_parse_uri_percent_encoded( parser: HttpRequestParser, uri: str, path: str, query: dict[str, str], fragment: str ) -> None: - text = (f"GET {uri} HTTP/1.1\r\n\r\n").encode() + text = (f"GET {uri} HTTP/1.1\r\nHost: a\r\n\r\n").encode() messages, upgrade, tail = parser.feed_data(text) msg = messages[0][0] @@ -1792,7 +1829,7 @@ def test_parse_uri_percent_encoded( def test_parse_uri_utf8(parser: HttpRequestParser) -> None: if not isinstance(parser, HttpRequestParserPy): pytest.xfail("Not valid HTTP. Maybe update py-parser to reject later.") - text = ("GET /путь?ключ=знач#фраг HTTP/1.1\r\n\r\n").encode() + text = ("GET /путь?ключ=знач#фраг HTTP/1.1\r\nHost: a\r\n\r\n").encode() messages, upgrade, tail = parser.feed_data(text) msg = messages[0][0] @@ -1804,7 +1841,8 @@ def test_parse_uri_utf8(parser: HttpRequestParser) -> None: def test_parse_uri_utf8_percent_encoded(parser: HttpRequestParser) -> None: text = ( - "GET %s HTTP/1.1\r\n\r\n" % quote("/путь?ключ=знач#фраг", safe="/?=#") + "GET %s HTTP/1.1\r\nHost: a\r\n\r\n" + % quote("/путь?ключ=знач#фраг", safe="/?=#") ).encode() messages, upgrade, tail = parser.feed_data(text) msg = messages[0][0] diff --git a/tests/test_web_urldispatcher.py b/tests/test_web_urldispatcher.py index 144bd9cd03e..da6a0c38b37 100644 --- a/tests/test_web_urldispatcher.py +++ b/tests/test_web_urldispatcher.py @@ -248,7 +248,7 @@ async def test_follow_symlink_directory_traversal( # We need to use a raw socket to test this, as the client will normalize # the path before sending it to the server. reader, writer = await asyncio.open_connection(client.host, client.port) - writer.write(b"GET /../private_file HTTP/1.1\r\n\r\n") + writer.write(b"GET /../private_file HTTP/1.1\r\nHost: a\r\n\r\n") response = await reader.readuntil(b"\r\n\r\n") assert b"404 Not Found" in response writer.close() @@ -300,14 +300,14 @@ async def test_follow_symlink_directory_traversal_after_normalization( # We need to use a raw socket to test this, as the client will normalize # the path before sending it to the server. reader, writer = await asyncio.open_connection(client.host, client.port) - writer.write(b"GET /my_symlink/../private_file HTTP/1.1\r\n\r\n") + writer.write(b"GET /my_symlink/../private_file HTTP/1.1\r\nHost: a\r\n\r\n") response = await reader.readuntil(b"\r\n\r\n") assert b"404 Not Found" in response writer.close() await writer.wait_closed() reader, writer = await asyncio.open_connection(client.host, client.port) - writer.write(b"GET /my_symlink/symlink_target_file HTTP/1.1\r\n\r\n") + writer.write(b"GET /my_symlink/symlink_target_file HTTP/1.1\r\nHost: a\r\n\r\n") response = await reader.readuntil(b"\r\n\r\n") assert b"200 OK" in response response = await reader.readuntil(b"readable")