shengyudong/baijiahao_data_crawl
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

2025-12-25 upload

Browse Source
This commit is contained in:
“shengyudong”
2025-12-25 11:16:59 +08:00
commit 322ac74336
2241 changed files with 639966 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1210
venv/Lib/site-packages/mitmproxy/proxy/layers/http/__init__.py Normal file
View File

File diff suppressed because it is too large Load Diff

61
venv/Lib/site-packages/mitmproxy/proxy/layers/http/_base.py Normal file
View File

@@ -0,0 +1,61 @@
import html
import textwrap
from dataclasses import dataclass
from mitmproxy import http
from mitmproxy.connection import Connection
from mitmproxy.proxy import commands
from mitmproxy.proxy import events
from mitmproxy.proxy import layer
from mitmproxy.proxy.context import Context
StreamId = int
@dataclass
class HttpEvent(events.Event):
# we need stream ids on every event to avoid race conditions
stream_id: StreamId
class HttpConnection(layer.Layer):
conn: Connection
def __init__(self, context: Context, conn: Connection):
super().__init__(context)
self.conn = conn
class HttpCommand(commands.Command):
pass
class ReceiveHttp(HttpCommand):
event: HttpEvent
def __init__(self, event: HttpEvent):
self.event = event
def __repr__(self) -> str:
return f"Receive({self.event})"
def format_error(status_code: int, message: str) -> bytes:
reason = http.status_codes.RESPONSES.get(status_code, "Unknown")
return (
textwrap.dedent(
f"""
<html>
<head>
<title>{status_code} {reason}</title>
</head>
<body>
<h1>{status_code} {reason}</h1>
<p>{html.escape(message)}</p>
</body>
</html>
"""
)
.strip()
.encode("utf8", "replace")
)

167
venv/Lib/site-packages/mitmproxy/proxy/layers/http/_events.py Normal file
View File

@@ -0,0 +1,167 @@
import enum
import typing
from dataclasses import dataclass
from ._base import HttpEvent
from mitmproxy import http
from mitmproxy.http import HTTPFlow
from mitmproxy.net.http import status_codes
@dataclass
class RequestHeaders(HttpEvent):
request: http.Request
end_stream: bool
"""
If True, we already know at this point that there is no message body. This is useful for HTTP/2, where it allows
us to set END_STREAM on headers already (and some servers - Akamai - implicitly expect that).
In either case, this event will nonetheless be followed by RequestEndOfMessage.
"""
replay_flow: HTTPFlow | None = None
"""If set, the current request headers belong to a replayed flow, which should be reused."""
@dataclass
class ResponseHeaders(HttpEvent):
response: http.Response
end_stream: bool = False
# explicit constructors below to facilitate type checking in _http1/_http2
@dataclass
class RequestData(HttpEvent):
data: bytes
def __init__(self, stream_id: int, data: bytes):
self.stream_id = stream_id
self.data = data
@dataclass
class ResponseData(HttpEvent):
data: bytes
def __init__(self, stream_id: int, data: bytes):
self.stream_id = stream_id
self.data = data
@dataclass
class RequestTrailers(HttpEvent):
trailers: http.Headers
def __init__(self, stream_id: int, trailers: http.Headers):
self.stream_id = stream_id
self.trailers = trailers
@dataclass
class ResponseTrailers(HttpEvent):
trailers: http.Headers
def __init__(self, stream_id: int, trailers: http.Headers):
self.stream_id = stream_id
self.trailers = trailers
@dataclass
class RequestEndOfMessage(HttpEvent):
def __init__(self, stream_id: int):
self.stream_id = stream_id
@dataclass
class ResponseEndOfMessage(HttpEvent):
def __init__(self, stream_id: int):
self.stream_id = stream_id
class ErrorCode(enum.Enum):
GENERIC_CLIENT_ERROR = 1
GENERIC_SERVER_ERROR = 2
REQUEST_TOO_LARGE = 3
RESPONSE_TOO_LARGE = 4
CONNECT_FAILED = 5
PASSTHROUGH_CLOSE = 6
KILL = 7
HTTP_1_1_REQUIRED = 8
"""Client should fall back to HTTP/1.1 to perform request."""
DESTINATION_UNKNOWN = 9
"""Proxy does not know where to send request to."""
CLIENT_DISCONNECTED = 10
"""Client disconnected before receiving entire response."""
CANCEL = 11
"""Client or server cancelled h2/h3 stream."""
REQUEST_VALIDATION_FAILED = 12
RESPONSE_VALIDATION_FAILED = 13
def http_status_code(self) -> int | None:
match self:
# Client Errors
case (
ErrorCode.GENERIC_CLIENT_ERROR
| ErrorCode.REQUEST_VALIDATION_FAILED
| ErrorCode.DESTINATION_UNKNOWN
):
return status_codes.BAD_REQUEST
case ErrorCode.REQUEST_TOO_LARGE:
return status_codes.PAYLOAD_TOO_LARGE
case (
ErrorCode.CONNECT_FAILED
| ErrorCode.GENERIC_SERVER_ERROR
| ErrorCode.RESPONSE_VALIDATION_FAILED
| ErrorCode.RESPONSE_TOO_LARGE
):
return status_codes.BAD_GATEWAY
case (
ErrorCode.PASSTHROUGH_CLOSE
| ErrorCode.KILL
| ErrorCode.HTTP_1_1_REQUIRED
| ErrorCode.CLIENT_DISCONNECTED
| ErrorCode.CANCEL
):
return None
case other: # pragma: no cover
typing.assert_never(other)
@dataclass
class RequestProtocolError(HttpEvent):
message: str
code: ErrorCode = ErrorCode.GENERIC_CLIENT_ERROR
def __init__(self, stream_id: int, message: str, code: ErrorCode):
assert isinstance(code, ErrorCode)
self.stream_id = stream_id
self.message = message
self.code = code
@dataclass
class ResponseProtocolError(HttpEvent):
message: str
code: ErrorCode = ErrorCode.GENERIC_SERVER_ERROR
def __init__(self, stream_id: int, message: str, code: ErrorCode):
assert isinstance(code, ErrorCode)
self.stream_id = stream_id
self.message = message
self.code = code
__all__ = [
"ErrorCode",
"HttpEvent",
"RequestHeaders",
"RequestData",
"RequestEndOfMessage",
"ResponseHeaders",
"ResponseData",
"RequestTrailers",
"ResponseTrailers",
"ResponseEndOfMessage",
"RequestProtocolError",
"ResponseProtocolError",
]

122
venv/Lib/site-packages/mitmproxy/proxy/layers/http/_hooks.py Normal file
View File

@@ -0,0 +1,122 @@
from dataclasses import dataclass
from mitmproxy import http
from mitmproxy.proxy import commands
@dataclass
class HttpRequestHeadersHook(commands.StartHook):
"""
HTTP request headers were successfully read. At this point, the body is empty.
"""
name = "requestheaders"
flow: http.HTTPFlow
@dataclass
class HttpRequestHook(commands.StartHook):
"""
The full HTTP request has been read.
Note: If request streaming is active, this event fires after the entire body has been streamed.
HTTP trailers, if present, have not been transmitted to the server yet and can still be modified.
Enabling streaming may cause unexpected event sequences: For example, `response` may now occur
before `request` because the server replied with "413 Payload Too Large" during upload.
"""
name = "request"
flow: http.HTTPFlow
@dataclass
class HttpResponseHeadersHook(commands.StartHook):
"""
HTTP response headers were successfully read. At this point, the body is empty.
"""
name = "responseheaders"
flow: http.HTTPFlow
@dataclass
class HttpResponseHook(commands.StartHook):
"""
The full HTTP response has been read.
Note: If response streaming is active, this event fires after the entire body has been streamed.
HTTP trailers, if present, have not been transmitted to the client yet and can still be modified.
"""
name = "response"
flow: http.HTTPFlow
@dataclass
class HttpErrorHook(commands.StartHook):
"""
An HTTP error has occurred, e.g. invalid server responses, or
interrupted connections. This is distinct from a valid server HTTP
error response, which is simply a response with an HTTP error code.
Every flow will receive either an error or an response event, but not both.
"""
name = "error"
flow: http.HTTPFlow
@dataclass
class HttpConnectHook(commands.StartHook):
"""
An HTTP CONNECT request was received. This event can be ignored for most practical purposes.
This event only occurs in regular and upstream proxy modes
when the client instructs mitmproxy to open a connection to an upstream host.
Setting a non 2xx response on the flow will return the response to the client and abort the connection.
CONNECT requests are HTTP proxy instructions for mitmproxy itself
and not forwarded. They do not generate the usual HTTP handler events,
but all requests going over the newly opened connection will.
"""
flow: http.HTTPFlow
@dataclass
class HttpConnectUpstreamHook(commands.StartHook):
"""
An HTTP CONNECT request is about to be sent to an upstream proxy.
This event can be ignored for most practical purposes.
This event can be used to set custom authentication headers for upstream proxies.
CONNECT requests do not generate the usual HTTP handler events,
but all requests going over the newly opened connection will.
"""
flow: http.HTTPFlow
@dataclass
class HttpConnectedHook(commands.StartHook):
"""
HTTP CONNECT was successful
> [!WARNING]
> This may fire before an upstream connection has been established
> if `connection_strategy` is set to `lazy` (default)
"""
flow: http.HTTPFlow
@dataclass
class HttpConnectErrorHook(commands.StartHook):
"""
HTTP CONNECT has failed.
This can happen when the upstream server is unreachable or proxy authentication is required.
In contrast to the `error` hook, `flow.error` is not guaranteed to be set.
"""
flow: http.HTTPFlow

502
venv/Lib/site-packages/mitmproxy/proxy/layers/http/_http1.py Normal file
View File

@@ -0,0 +1,502 @@
import abc
from collections.abc import Callable
from typing import Union
import h11
from h11._readers import ChunkedReader
from h11._readers import ContentLengthReader
from h11._readers import Http10Reader
from h11._receivebuffer import ReceiveBuffer
from ...context import Context
from ._base import format_error
from ._base import HttpConnection
from ._events import ErrorCode
from ._events import HttpEvent
from ._events import RequestData
from ._events import RequestEndOfMessage
from ._events import RequestHeaders
from ._events import RequestProtocolError
from ._events import ResponseData
from ._events import ResponseEndOfMessage
from ._events import ResponseHeaders
from ._events import ResponseProtocolError
from mitmproxy import http
from mitmproxy import version
from mitmproxy.connection import Connection
from mitmproxy.connection import ConnectionState
from mitmproxy.net.http import http1
from mitmproxy.net.http import status_codes
from mitmproxy.proxy import commands
from mitmproxy.proxy import events
from mitmproxy.proxy import layer
from mitmproxy.proxy.layers.http._base import ReceiveHttp
from mitmproxy.proxy.layers.http._base import StreamId
from mitmproxy.proxy.utils import expect
from mitmproxy.utils import human
TBodyReader = Union[ChunkedReader, Http10Reader, ContentLengthReader]
class Http1Connection(HttpConnection, metaclass=abc.ABCMeta):
stream_id: StreamId | None = None
request: http.Request | None = None
response: http.Response | None = None
request_done: bool = False
response_done: bool = False
# this is a bit of a hack to make both mypy and PyCharm happy.
state: Callable[[events.Event], layer.CommandGenerator[None]] | Callable
body_reader: TBodyReader
buf: ReceiveBuffer
ReceiveProtocolError: type[RequestProtocolError | ResponseProtocolError]
ReceiveData: type[RequestData | ResponseData]
ReceiveEndOfMessage: type[RequestEndOfMessage | ResponseEndOfMessage]
def __init__(self, context: Context, conn: Connection):
super().__init__(context, conn)
self.buf = ReceiveBuffer()
@abc.abstractmethod
def send(self, event: HttpEvent) -> layer.CommandGenerator[None]:
yield from () # pragma: no cover
@abc.abstractmethod
def read_headers(
self, event: events.ConnectionEvent
) -> layer.CommandGenerator[None]:
yield from () # pragma: no cover
def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]:
if isinstance(event, HttpEvent):
yield from self.send(event)
else:
if (
isinstance(event, events.DataReceived)
and self.state != self.passthrough
):
self.buf += event.data
yield from self.state(event)
@expect(events.Start)
def start(self, _) -> layer.CommandGenerator[None]:
self.state = self.read_headers
yield from ()
state = start
def read_body(self, event: events.Event) -> layer.CommandGenerator[None]:
assert self.stream_id is not None
while True:
try:
if isinstance(event, events.DataReceived):
h11_event = self.body_reader(self.buf)
elif isinstance(event, events.ConnectionClosed):
h11_event = self.body_reader.read_eof()
else:
raise AssertionError(f"Unexpected event: {event}")
except h11.ProtocolError as e:
yield commands.CloseConnection(self.conn)
yield ReceiveHttp(
self.ReceiveProtocolError(
self.stream_id,
f"HTTP/1 protocol error: {e}",
code=self.ReceiveProtocolError.code,
)
)
return
if h11_event is None:
return
elif isinstance(h11_event, h11.Data):
data: bytes = bytes(h11_event.data)
if data:
yield ReceiveHttp(self.ReceiveData(self.stream_id, data))
elif isinstance(h11_event, h11.EndOfMessage):
assert self.request
if h11_event.headers:
raise NotImplementedError(f"HTTP trailers are not implemented yet.")
if self.request.data.method.upper() != b"CONNECT":
yield ReceiveHttp(self.ReceiveEndOfMessage(self.stream_id))
is_request = isinstance(self, Http1Server)
yield from self.mark_done(request=is_request, response=not is_request)
return
def wait(self, event: events.Event) -> layer.CommandGenerator[None]:
"""
We wait for the current flow to be finished before parsing the next message,
as we may want to upgrade to WebSocket or plain TCP before that.
"""
assert self.stream_id
if isinstance(event, events.DataReceived):
return
elif isinstance(event, events.ConnectionClosed):
# for practical purposes, we assume that a peer which sent at least a FIN
# is not interested in any more data from us, see
# see https://github.com/httpwg/http-core/issues/22
if event.connection.state is not ConnectionState.CLOSED:
yield commands.CloseConnection(event.connection)
yield ReceiveHttp(
self.ReceiveProtocolError(
self.stream_id,
f"Client disconnected.",
code=ErrorCode.CLIENT_DISCONNECTED,
)
)
else: # pragma: no cover
raise AssertionError(f"Unexpected event: {event}")
def done(self, event: events.ConnectionEvent) -> layer.CommandGenerator[None]:
yield from () # pragma: no cover
def make_pipe(self) -> layer.CommandGenerator[None]:
self.state = self.passthrough
if self.buf:
already_received = self.buf.maybe_extract_at_most(len(self.buf)) or b""
# Some clients send superfluous newlines after CONNECT, we want to eat those.
already_received = already_received.lstrip(b"\r\n")
if already_received:
yield from self.state(events.DataReceived(self.conn, already_received))
def passthrough(self, event: events.Event) -> layer.CommandGenerator[None]:
assert self.stream_id
if isinstance(event, events.DataReceived):
yield ReceiveHttp(self.ReceiveData(self.stream_id, event.data))
elif isinstance(event, events.ConnectionClosed):
if isinstance(self, Http1Server):
yield ReceiveHttp(RequestEndOfMessage(self.stream_id))
else:
yield ReceiveHttp(ResponseEndOfMessage(self.stream_id))
def mark_done(
self, *, request: bool = False, response: bool = False
) -> layer.CommandGenerator[None]:
if request:
self.request_done = True
if response:
self.response_done = True
if self.request_done and self.response_done:
assert self.request
assert self.response
if should_make_pipe(self.request, self.response):
yield from self.make_pipe()
return
try:
read_until_eof_semantics = (
http1.expected_http_body_size(self.request, self.response) == -1
)
except ValueError:
# this may raise only now (and not earlier) because an addon set invalid headers,
# in which case it's not really clear what we are supposed to do.
read_until_eof_semantics = False
connection_done = (
read_until_eof_semantics
or http1.connection_close(
self.request.http_version, self.request.headers
)
or http1.connection_close(
self.response.http_version, self.response.headers
)
# If we proxy HTTP/2 to HTTP/1, we only use upstream connections for one request.
# This simplifies our connection management quite a bit as we can rely on
# the proxyserver's max-connection-per-server throttling.
or (
(self.request.is_http2 or self.request.is_http3)
and isinstance(self, Http1Client)
)
)
if connection_done:
yield commands.CloseConnection(self.conn)
self.state = self.done
return
self.request_done = self.response_done = False
self.request = self.response = None
if isinstance(self, Http1Server):
self.stream_id += 2
else:
self.stream_id = None
self.state = self.read_headers
if self.buf:
yield from self.state(events.DataReceived(self.conn, b""))
class Http1Server(Http1Connection):
"""A simple HTTP/1 server with no pipelining support."""
ReceiveProtocolError = RequestProtocolError
ReceiveData = RequestData
ReceiveEndOfMessage = RequestEndOfMessage
stream_id: int
def __init__(self, context: Context):
super().__init__(context, context.client)
self.stream_id = 1
def send(self, event: HttpEvent) -> layer.CommandGenerator[None]:
assert event.stream_id == self.stream_id
if isinstance(event, ResponseHeaders):
self.response = response = event.response
if response.is_http2 or response.is_http3:
response = response.copy()
# Convert to an HTTP/1 response.
response.http_version = "HTTP/1.1"
# not everyone supports empty reason phrases, so we better make up one.
response.reason = status_codes.RESPONSES.get(response.status_code, "")
# Shall we set a Content-Length header here if there is none?
# For now, let's try to modify as little as possible.
raw = http1.assemble_response_head(response)
yield commands.SendData(self.conn, raw)
elif isinstance(event, ResponseData):
assert self.response
if "chunked" in self.response.headers.get("transfer-encoding", "").lower():
raw = b"%x\r\n%s\r\n" % (len(event.data), event.data)
else:
raw = event.data
if raw:
yield commands.SendData(self.conn, raw)
elif isinstance(event, ResponseEndOfMessage):
assert self.request
assert self.response
if (
self.request.method.upper() != "HEAD"
and "chunked"
in self.response.headers.get("transfer-encoding", "").lower()
):
yield commands.SendData(self.conn, b"0\r\n\r\n")
yield from self.mark_done(response=True)
elif isinstance(event, ResponseProtocolError):
if not (self.conn.state & ConnectionState.CAN_WRITE):
return
status = event.code.http_status_code()
if not self.response and status is not None:
yield commands.SendData(
self.conn, make_error_response(status, event.message)
)
yield commands.CloseConnection(self.conn)
else:
raise AssertionError(f"Unexpected event: {event}")
def read_headers(
self, event: events.ConnectionEvent
) -> layer.CommandGenerator[None]:
if isinstance(event, events.DataReceived):
request_head = self.buf.maybe_extract_lines()
if request_head:
try:
self.request = http1.read_request_head(
[bytes(x) for x in request_head]
)
expected_body_size = http1.expected_http_body_size(self.request)
except ValueError as e:
yield commands.SendData(self.conn, make_error_response(400, str(e)))
yield commands.CloseConnection(self.conn)
if self.request:
# we have headers that we can show in the ui
yield ReceiveHttp(
RequestHeaders(self.stream_id, self.request, False)
)
yield ReceiveHttp(
RequestProtocolError(
self.stream_id, str(e), ErrorCode.GENERIC_CLIENT_ERROR
)
)
else:
yield commands.Log(
f"{human.format_address(self.conn.peername)}: {e}"
)
self.state = self.done
return
yield ReceiveHttp(
RequestHeaders(
self.stream_id, self.request, expected_body_size == 0
)
)
self.body_reader = make_body_reader(expected_body_size)
self.state = self.read_body
yield from self.state(event)
else:
pass # FIXME: protect against header size DoS
elif isinstance(event, events.ConnectionClosed):
buf = bytes(self.buf)
if buf.strip():
yield commands.Log(
f"Client closed connection before completing request headers: {buf!r}"
)
yield commands.CloseConnection(self.conn)
else:
raise AssertionError(f"Unexpected event: {event}")
def mark_done(
self, *, request: bool = False, response: bool = False
) -> layer.CommandGenerator[None]:
yield from super().mark_done(request=request, response=response)
if self.request_done and not self.response_done:
self.state = self.wait
class Http1Client(Http1Connection):
"""A simple HTTP/1 client with no pipelining support."""
ReceiveProtocolError = ResponseProtocolError
ReceiveData = ResponseData
ReceiveEndOfMessage = ResponseEndOfMessage
def __init__(self, context: Context):
super().__init__(context, context.server)
def send(self, event: HttpEvent) -> layer.CommandGenerator[None]:
if isinstance(event, RequestProtocolError):
yield commands.CloseConnection(self.conn)
return
if self.stream_id is None:
assert isinstance(event, RequestHeaders)
self.stream_id = event.stream_id
self.request = event.request
assert self.stream_id == event.stream_id
if isinstance(event, RequestHeaders):
request = event.request
if request.is_http2 or request.is_http3:
# Convert to an HTTP/1 request.
request = (
request.copy()
) # (we could probably be a bit more efficient here.)
request.http_version = "HTTP/1.1"
if "Host" not in request.headers and request.authority:
request.headers.insert(0, "Host", request.authority)
request.authority = ""
cookie_headers = request.headers.get_all("Cookie")
if len(cookie_headers) > 1:
# Only HTTP/2 supports multiple cookie headers, HTTP/1.x does not.
# see: https://www.rfc-editor.org/rfc/rfc6265#section-5.4
# https://www.rfc-editor.org/rfc/rfc7540#section-8.1.2.5
request.headers["Cookie"] = "; ".join(cookie_headers)
raw = http1.assemble_request_head(request)
yield commands.SendData(self.conn, raw)
elif isinstance(event, RequestData):
assert self.request
if "chunked" in self.request.headers.get("transfer-encoding", "").lower():
raw = b"%x\r\n%s\r\n" % (len(event.data), event.data)
else:
raw = event.data
if raw:
yield commands.SendData(self.conn, raw)
elif isinstance(event, RequestEndOfMessage):
assert self.request
if "chunked" in self.request.headers.get("transfer-encoding", "").lower():
yield commands.SendData(self.conn, b"0\r\n\r\n")
elif http1.expected_http_body_size(self.request, self.response) == -1:
yield commands.CloseTcpConnection(self.conn, half_close=True)
yield from self.mark_done(request=True)
else:
raise AssertionError(f"Unexpected event: {event}")
def read_headers(
self, event: events.ConnectionEvent
) -> layer.CommandGenerator[None]:
if isinstance(event, events.DataReceived):
if not self.request:
# we just received some data for an unknown request.
yield commands.Log(f"Unexpected data from server: {bytes(self.buf)!r}")
yield commands.CloseConnection(self.conn)
return
assert self.stream_id is not None
response_head = self.buf.maybe_extract_lines()
if response_head:
try:
self.response = http1.read_response_head(
[bytes(x) for x in response_head]
)
expected_size = http1.expected_http_body_size(
self.request, self.response
)
except ValueError as e:
yield commands.CloseConnection(self.conn)
yield ReceiveHttp(
ResponseProtocolError(
self.stream_id,
f"Cannot parse HTTP response: {e}",
ErrorCode.GENERIC_SERVER_ERROR,
)
)
return
yield ReceiveHttp(
ResponseHeaders(self.stream_id, self.response, expected_size == 0)
)
self.body_reader = make_body_reader(expected_size)
self.state = self.read_body
yield from self.state(event)
else:
pass # FIXME: protect against header size DoS
elif isinstance(event, events.ConnectionClosed):
if self.conn.state & ConnectionState.CAN_WRITE:
yield commands.CloseConnection(self.conn)
if self.stream_id:
if self.buf:
yield ReceiveHttp(
ResponseProtocolError(
self.stream_id,
f"unexpected server response: {bytes(self.buf)!r}",
ErrorCode.GENERIC_SERVER_ERROR,
)
)
else:
# The server has closed the connection to prevent us from continuing.
# We need to signal that to the stream.
# https://tools.ietf.org/html/rfc7231#section-6.5.11
yield ReceiveHttp(
ResponseProtocolError(
self.stream_id,
"server closed connection",
ErrorCode.GENERIC_SERVER_ERROR,
)
)
else:
return
else:
raise AssertionError(f"Unexpected event: {event}")
def should_make_pipe(request: http.Request, response: http.Response) -> bool:
if response.status_code == 101:
return True
elif response.status_code == 200 and request.method.upper() == "CONNECT":
return True
else:
return False
def make_body_reader(expected_size: int | None) -> TBodyReader:
if expected_size is None:
return ChunkedReader()
elif expected_size == -1:
return Http10Reader()
else:
return ContentLengthReader(expected_size)
def make_error_response(
status_code: int,
message: str = "",
) -> bytes:
resp = http.Response.make(
status_code,
format_error(status_code, message),
http.Headers(
Server=version.MITMPROXY,
Connection="close",
Content_Type="text/html",
),
)
return http1.assemble_response(resp)
__all__ = [
"Http1Client",
"Http1Server",
]

714
venv/Lib/site-packages/mitmproxy/proxy/layers/http/_http2.py Normal file
View File

@@ -0,0 +1,714 @@
import collections
import time
from collections.abc import Sequence
from enum import Enum
from logging import DEBUG
from logging import ERROR
from typing import Any
from typing import assert_never
from typing import ClassVar
import h2.config
import h2.connection
import h2.errors
import h2.events
import h2.exceptions
import h2.settings
import h2.stream
import h2.utilities
from ...commands import CloseConnection
from ...commands import Log
from ...commands import RequestWakeup
from ...commands import SendData
from ...context import Context
from ...events import ConnectionClosed
from ...events import DataReceived
from ...events import Event
from ...events import Start
from ...events import Wakeup
from ...layer import CommandGenerator
from ...utils import expect
from . import ErrorCode
from . import RequestData
from . import RequestEndOfMessage
from . import RequestHeaders
from . import RequestProtocolError
from . import RequestTrailers
from . import ResponseData
from . import ResponseEndOfMessage
from . import ResponseHeaders
from . import ResponseProtocolError
from . import ResponseTrailers
from ._base import format_error
from ._base import HttpConnection
from ._base import HttpEvent
from ._base import ReceiveHttp
from ._http_h2 import BufferedH2Connection
from ._http_h2 import H2ConnectionLogger
from mitmproxy import http
from mitmproxy import version
from mitmproxy.connection import Connection
from mitmproxy.net.http import status_codes
from mitmproxy.net.http import url
from mitmproxy.utils import human
class StreamState(Enum):
EXPECTING_HEADERS = 1
HEADERS_RECEIVED = 2
CATCH_HYPER_H2_ERRORS = (ValueError, IndexError)
class Http2Connection(HttpConnection):
h2_conf: ClassVar[h2.config.H2Configuration]
h2_conf_defaults: dict[str, Any] = dict(
header_encoding=False,
validate_outbound_headers=False,
# validate_inbound_headers is controlled by the validate_inbound_headers option.
normalize_inbound_headers=False, # changing this to True is required to pass h2spec
normalize_outbound_headers=False,
)
h2_conn: BufferedH2Connection
streams: dict[int, StreamState]
"""keep track of all active stream ids to send protocol errors on teardown"""
ReceiveProtocolError: type[RequestProtocolError | ResponseProtocolError]
ReceiveData: type[RequestData | ResponseData]
ReceiveTrailers: type[RequestTrailers | ResponseTrailers]
ReceiveEndOfMessage: type[RequestEndOfMessage | ResponseEndOfMessage]
def __init__(self, context: Context, conn: Connection):
super().__init__(context, conn)
if self.debug:
self.h2_conf.logger = H2ConnectionLogger(
self.context.client.peername, self.__class__.__name__
)
self.h2_conf.validate_inbound_headers = (
self.context.options.validate_inbound_headers
)
self.h2_conn = BufferedH2Connection(self.h2_conf)
self.streams = {}
def is_closed(self, stream_id: int) -> bool:
"""Check if a non-idle stream is closed"""
stream = self.h2_conn.streams.get(stream_id, None)
if (
stream is not None
and stream.state_machine.state is not h2.stream.StreamState.CLOSED
and self.h2_conn.state_machine.state
is not h2.connection.ConnectionState.CLOSED
):
return False
else:
return True
def is_open_for_us(self, stream_id: int) -> bool:
"""Check if we can write to a non-idle stream."""
stream = self.h2_conn.streams.get(stream_id, None)
if (
stream is not None
and stream.state_machine.state
is not h2.stream.StreamState.HALF_CLOSED_LOCAL
and stream.state_machine.state is not h2.stream.StreamState.CLOSED
and self.h2_conn.state_machine.state
is not h2.connection.ConnectionState.CLOSED
):
return True
else:
return False
def _handle_event(self, event: Event) -> CommandGenerator[None]:
if isinstance(event, Start):
self.h2_conn.initiate_connection()
yield SendData(self.conn, self.h2_conn.data_to_send())
elif isinstance(event, HttpEvent):
if isinstance(event, (RequestData, ResponseData)):
if self.is_open_for_us(event.stream_id):
self.h2_conn.send_data(event.stream_id, event.data)
elif isinstance(event, (RequestTrailers, ResponseTrailers)):
if self.is_open_for_us(event.stream_id):
trailers = [*event.trailers.fields]
self.h2_conn.send_trailers(event.stream_id, trailers)
elif isinstance(event, (RequestEndOfMessage, ResponseEndOfMessage)):
if self.is_open_for_us(event.stream_id):
self.h2_conn.end_stream(event.stream_id)
elif isinstance(event, (RequestProtocolError, ResponseProtocolError)):
if not self.is_closed(event.stream_id):
stream: h2.stream.H2Stream = self.h2_conn.streams[event.stream_id]
status = event.code.http_status_code()
if (
isinstance(event, ResponseProtocolError)
and self.is_open_for_us(event.stream_id)
and not stream.state_machine.headers_sent
and status is not None
):
self.h2_conn.send_headers(
event.stream_id,
[
(b":status", b"%d" % status),
(b"server", version.MITMPROXY.encode()),
(b"content-type", b"text/html"),
],
)
self.h2_conn.send_data(
event.stream_id,
format_error(status, event.message),
end_stream=True,
)
else:
match event.code:
case ErrorCode.CANCEL | ErrorCode.CLIENT_DISCONNECTED:
error_code = h2.errors.ErrorCodes.CANCEL
case ErrorCode.KILL:
# XXX: Debateable whether this is the best error code.
error_code = h2.errors.ErrorCodes.INTERNAL_ERROR
case ErrorCode.HTTP_1_1_REQUIRED:
error_code = h2.errors.ErrorCodes.HTTP_1_1_REQUIRED
case ErrorCode.PASSTHROUGH_CLOSE:
# FIXME: This probably shouldn't be a protocol error, but an EOM event.
error_code = h2.errors.ErrorCodes.CANCEL
case (
ErrorCode.GENERIC_CLIENT_ERROR
| ErrorCode.GENERIC_SERVER_ERROR
| ErrorCode.REQUEST_TOO_LARGE
| ErrorCode.RESPONSE_TOO_LARGE
| ErrorCode.CONNECT_FAILED
| ErrorCode.DESTINATION_UNKNOWN
| ErrorCode.REQUEST_VALIDATION_FAILED
| ErrorCode.RESPONSE_VALIDATION_FAILED
):
error_code = h2.errors.ErrorCodes.INTERNAL_ERROR
case other: # pragma: no cover
assert_never(other)
self.h2_conn.reset_stream(event.stream_id, error_code.value)
else:
raise AssertionError(f"Unexpected event: {event}")
data_to_send = self.h2_conn.data_to_send()
if data_to_send:
yield SendData(self.conn, data_to_send)
elif isinstance(event, DataReceived):
try:
try:
events = self.h2_conn.receive_data(event.data)
except CATCH_HYPER_H2_ERRORS as e: # pragma: no cover
# this should never raise a ValueError, but we triggered one while fuzzing:
# https://github.com/python-hyper/hyper-h2/issues/1231
# this stays here as defense-in-depth.
raise h2.exceptions.ProtocolError(
f"uncaught hyper-h2 error: {e}"
) from e
except h2.exceptions.ProtocolError as e:
events = [e]
for h2_event in events:
if self.debug:
yield Log(f"{self.debug}[h2] {h2_event}", DEBUG)
if (yield from self.handle_h2_event(h2_event)):
if self.debug:
yield Log(f"{self.debug}[h2] done", DEBUG)
return
data_to_send = self.h2_conn.data_to_send()
if data_to_send:
yield SendData(self.conn, data_to_send)
elif isinstance(event, ConnectionClosed):
yield from self.close_connection("peer closed connection")
else:
raise AssertionError(f"Unexpected event: {event!r}")
def handle_h2_event(self, event: h2.events.Event) -> CommandGenerator[bool]:
"""returns true if further processing should be stopped."""
if isinstance(event, h2.events.DataReceived):
state = self.streams.get(event.stream_id, None)
if state is StreamState.HEADERS_RECEIVED:
is_empty_eos_data_frame = event.stream_ended and not event.data
if not is_empty_eos_data_frame:
yield ReceiveHttp(self.ReceiveData(event.stream_id, event.data))
elif state is StreamState.EXPECTING_HEADERS:
yield from self.protocol_error(
f"Received HTTP/2 data frame, expected headers."
)
return True
self.h2_conn.acknowledge_received_data(
event.flow_controlled_length, event.stream_id
)
elif isinstance(event, h2.events.TrailersReceived):
trailers = http.Headers(event.headers)
yield ReceiveHttp(self.ReceiveTrailers(event.stream_id, trailers))
elif isinstance(event, h2.events.StreamEnded):
state = self.streams.get(event.stream_id, None)
if state is StreamState.HEADERS_RECEIVED:
yield ReceiveHttp(self.ReceiveEndOfMessage(event.stream_id))
elif state is StreamState.EXPECTING_HEADERS:
raise AssertionError("unreachable")
if self.is_closed(event.stream_id):
self.streams.pop(event.stream_id, None)
elif isinstance(event, h2.events.StreamReset):
if event.stream_id in self.streams:
try:
err_str = h2.errors.ErrorCodes(event.error_code).name
except ValueError:
err_str = str(event.error_code)
match event.error_code:
case h2.errors.ErrorCodes.CANCEL:
err_code = ErrorCode.CANCEL
case h2.errors.ErrorCodes.HTTP_1_1_REQUIRED:
err_code = ErrorCode.HTTP_1_1_REQUIRED
case _:
err_code = self.ReceiveProtocolError.code
yield ReceiveHttp(
self.ReceiveProtocolError(
event.stream_id,
f"stream reset by client ({err_str})",
code=err_code,
)
)
self.streams.pop(event.stream_id)
else:
pass # We don't track priority frames which could be followed by a stream reset here.
elif isinstance(event, h2.exceptions.ProtocolError):
yield from self.protocol_error(f"HTTP/2 protocol error: {event}")
return True
elif isinstance(event, h2.events.ConnectionTerminated):
yield from self.close_connection(f"HTTP/2 connection closed: {event!r}")
return True
# The implementation above isn't really ideal, we should probably only terminate streams > last_stream_id?
# We currently lack a mechanism to signal that connections are still active but cannot be reused.
# for stream_id in self.streams:
# if stream_id > event.last_stream_id:
# yield ReceiveHttp(self.ReceiveProtocolError(stream_id, f"HTTP/2 connection closed: {event!r}"))
# self.streams.pop(stream_id)
elif isinstance(event, h2.events.RemoteSettingsChanged):
pass
elif isinstance(event, h2.events.SettingsAcknowledged):
pass
elif isinstance(event, h2.events.PriorityUpdated):
pass
elif isinstance(event, h2.events.PingReceived):
pass
elif isinstance(event, h2.events.PingAckReceived):
pass
elif isinstance(event, h2.events.PushedStreamReceived):
yield Log(
"Received HTTP/2 push promise, even though we signalled no support.",
ERROR,
)
elif isinstance(event, h2.events.UnknownFrameReceived):
# https://http2.github.io/http2-spec/#rfc.section.4.1
# Implementations MUST ignore and discard any frame that has a type that is unknown.
yield Log(f"Ignoring unknown HTTP/2 frame type: {event.frame.type}")
elif isinstance(event, h2.events.AlternativeServiceAvailable):
yield Log(
"Received HTTP/2 Alt-Svc frame, which will not be forwarded.", DEBUG
)
else:
raise AssertionError(f"Unexpected event: {event!r}")
return False
def protocol_error(
self,
message: str,
error_code: int = h2.errors.ErrorCodes.PROTOCOL_ERROR,
) -> CommandGenerator[None]:
yield Log(f"{human.format_address(self.conn.peername)}: {message}")
self.h2_conn.close_connection(error_code, message.encode())
yield SendData(self.conn, self.h2_conn.data_to_send())
yield from self.close_connection(message)
def close_connection(self, msg: str) -> CommandGenerator[None]:
yield CloseConnection(self.conn)
for stream_id in self.streams:
yield ReceiveHttp(
self.ReceiveProtocolError(
stream_id, msg, self.ReceiveProtocolError.code
)
)
self.streams.clear()
self._handle_event = self.done # type: ignore
@expect(DataReceived, HttpEvent, ConnectionClosed, Wakeup)
def done(self, _) -> CommandGenerator[None]:
yield from ()
def normalize_h1_headers(
headers: list[tuple[bytes, bytes]], is_client: bool
) -> list[tuple[bytes, bytes]]:
# HTTP/1 servers commonly send capitalized headers (Content-Length vs content-length),
# which isn't valid HTTP/2. As such we normalize.
# Make sure that this is not just an iterator but an iterable,
# otherwise hyper-h2 will silently drop headers.
return list(
h2.utilities.normalize_outbound_headers(
headers,
h2.utilities.HeaderValidationFlags(is_client, False, not is_client, False),
)
)
def normalize_h2_headers(headers: list[tuple[bytes, bytes]]) -> CommandGenerator[None]:
for i in range(len(headers)):
if not headers[i][0].islower():
yield Log(
f"Lowercased {repr(headers[i][0]).lstrip('b')} header as uppercase is not allowed with HTTP/2."
)
headers[i] = (headers[i][0].lower(), headers[i][1])
def format_h2_request_headers(
context: Context,
event: RequestHeaders,
) -> CommandGenerator[list[tuple[bytes, bytes]]]:
pseudo_headers = [
(b":method", event.request.data.method),
(b":scheme", event.request.data.scheme),
(b":path", event.request.data.path),
]
if event.request.authority:
pseudo_headers.append((b":authority", event.request.data.authority))
if event.request.is_http2 or event.request.is_http3:
hdrs = list(event.request.headers.fields)
if context.options.normalize_outbound_headers:
yield from normalize_h2_headers(hdrs)
else:
headers = event.request.headers
if not event.request.authority and "host" in headers:
headers = headers.copy()
pseudo_headers.append((b":authority", headers.pop(b"host")))
hdrs = normalize_h1_headers(list(headers.fields), True)
return pseudo_headers + hdrs
def format_h2_response_headers(
context: Context,
event: ResponseHeaders,
) -> CommandGenerator[list[tuple[bytes, bytes]]]:
headers = [
(b":status", b"%d" % event.response.status_code),
*event.response.headers.fields,
]
if event.response.is_http2 or event.response.is_http3:
if context.options.normalize_outbound_headers:
yield from normalize_h2_headers(headers)
else:
headers = normalize_h1_headers(headers, False)
return headers
class Http2Server(Http2Connection):
h2_conf = h2.config.H2Configuration(
**Http2Connection.h2_conf_defaults,
client_side=False,
)
ReceiveProtocolError = RequestProtocolError
ReceiveData = RequestData
ReceiveTrailers = RequestTrailers
ReceiveEndOfMessage = RequestEndOfMessage
def __init__(self, context: Context):
super().__init__(context, context.client)
def _handle_event(self, event: Event) -> CommandGenerator[None]:
if isinstance(event, ResponseHeaders):
if self.is_open_for_us(event.stream_id):
self.h2_conn.send_headers(
event.stream_id,
headers=(
yield from format_h2_response_headers(self.context, event)
),
end_stream=event.end_stream,
)
yield SendData(self.conn, self.h2_conn.data_to_send())
else:
yield from super()._handle_event(event)
def handle_h2_event(self, event: h2.events.Event) -> CommandGenerator[bool]:
if isinstance(event, h2.events.RequestReceived):
try:
(
host,
port,
method,
scheme,
authority,
path,
headers,
) = parse_h2_request_headers(event.headers)
except ValueError as e:
yield from self.protocol_error(f"Invalid HTTP/2 request headers: {e}")
return True
request = http.Request(
host=host,
port=port,
method=method,
scheme=scheme,
authority=authority,
path=path,
http_version=b"HTTP/2.0",
headers=headers,
content=None,
trailers=None,
timestamp_start=time.time(),
timestamp_end=None,
)
self.streams[event.stream_id] = StreamState.HEADERS_RECEIVED
yield ReceiveHttp(
RequestHeaders(
event.stream_id, request, end_stream=bool(event.stream_ended)
)
)
return False
else:
return (yield from super().handle_h2_event(event))
class Http2Client(Http2Connection):
h2_conf = h2.config.H2Configuration(
**Http2Connection.h2_conf_defaults,
client_side=True,
)
ReceiveProtocolError = ResponseProtocolError
ReceiveData = ResponseData
ReceiveTrailers = ResponseTrailers
ReceiveEndOfMessage = ResponseEndOfMessage
our_stream_id: dict[int, int]
their_stream_id: dict[int, int]
stream_queue: collections.defaultdict[int, list[Event]]
"""Queue of streams that we haven't sent yet because we have reached MAX_CONCURRENT_STREAMS"""
provisional_max_concurrency: int | None = 10
"""A provisional currency limit before we get the server's first settings frame."""
last_activity: float
"""Timestamp of when we've last seen network activity on this connection."""
def __init__(self, context: Context):
super().__init__(context, context.server)
# Disable HTTP/2 push for now to keep things simple.
# don't send here, that is done as part of initiate_connection().
self.h2_conn.local_settings.enable_push = 0
# hyper-h2 pitfall: we need to acknowledge here, otherwise its sends out the old settings.
self.h2_conn.local_settings.acknowledge()
self.our_stream_id = {}
self.their_stream_id = {}
self.stream_queue = collections.defaultdict(list)
def _handle_event(self, event: Event) -> CommandGenerator[None]:
# We can't reuse stream ids from the client because they may arrived reordered here
# and HTTP/2 forbids opening a stream on a lower id than what was previously sent (see test_stream_concurrency).
# To mitigate this, we transparently map the outside's stream id to our stream id.
if isinstance(event, HttpEvent):
ours = self.our_stream_id.get(event.stream_id, None)
if ours is None:
no_free_streams = self.h2_conn.open_outbound_streams >= (
self.provisional_max_concurrency
or self.h2_conn.remote_settings.max_concurrent_streams
)
if no_free_streams:
self.stream_queue[event.stream_id].append(event)
return
ours = self.h2_conn.get_next_available_stream_id()
self.our_stream_id[event.stream_id] = ours
self.their_stream_id[ours] = event.stream_id
event.stream_id = ours
for cmd in self._handle_event2(event):
if isinstance(cmd, ReceiveHttp):
cmd.event.stream_id = self.their_stream_id[cmd.event.stream_id]
yield cmd
can_resume_queue = self.stream_queue and self.h2_conn.open_outbound_streams < (
self.provisional_max_concurrency
or self.h2_conn.remote_settings.max_concurrent_streams
)
if can_resume_queue:
# popitem would be LIFO, but we want FIFO.
events = self.stream_queue.pop(next(iter(self.stream_queue)))
for event in events:
yield from self._handle_event(event)
def _handle_event2(self, event: Event) -> CommandGenerator[None]:
if isinstance(event, Wakeup):
send_ping_now = (
# add one second to avoid unnecessary roundtrip, we don't need to be super correct here.
time.time() - self.last_activity + 1
> self.context.options.http2_ping_keepalive
)
if send_ping_now:
# PING frames MUST contain 8 octets of opaque data in the payload.
# A sender can include any value it chooses and use those octets in any fashion.
self.last_activity = time.time()
self.h2_conn.ping(b"0" * 8)
data = self.h2_conn.data_to_send()
if data is not None:
yield Log(
f"Send HTTP/2 keep-alive PING to {human.format_address(self.conn.peername)}",
DEBUG,
)
yield SendData(self.conn, data)
time_until_next_ping = self.context.options.http2_ping_keepalive - (
time.time() - self.last_activity
)
yield RequestWakeup(time_until_next_ping)
return
self.last_activity = time.time()
if isinstance(event, Start):
if self.context.options.http2_ping_keepalive > 0:
yield RequestWakeup(self.context.options.http2_ping_keepalive)
yield from super()._handle_event(event)
elif isinstance(event, RequestHeaders):
self.h2_conn.send_headers(
event.stream_id,
headers=(yield from format_h2_request_headers(self.context, event)),
end_stream=event.end_stream,
)
self.streams[event.stream_id] = StreamState.EXPECTING_HEADERS
yield SendData(self.conn, self.h2_conn.data_to_send())
else:
yield from super()._handle_event(event)
def handle_h2_event(self, event: h2.events.Event) -> CommandGenerator[bool]:
if isinstance(event, h2.events.ResponseReceived):
if (
self.streams.get(event.stream_id, None)
is not StreamState.EXPECTING_HEADERS
):
yield from self.protocol_error(f"Received unexpected HTTP/2 response.")
return True
try:
status_code, headers = parse_h2_response_headers(event.headers)
except ValueError as e:
yield from self.protocol_error(f"Invalid HTTP/2 response headers: {e}")
return True
response = http.Response(
http_version=b"HTTP/2.0",
status_code=status_code,
reason=b"",
headers=headers,
content=None,
trailers=None,
timestamp_start=time.time(),
timestamp_end=None,
)
self.streams[event.stream_id] = StreamState.HEADERS_RECEIVED
yield ReceiveHttp(
ResponseHeaders(event.stream_id, response, bool(event.stream_ended))
)
return False
elif isinstance(event, h2.events.InformationalResponseReceived):
# We violate the spec here ("A proxy MUST forward 1xx responses", RFC 7231),
# but that's probably fine:
# - 100 Continue is sent by mitmproxy to clients (irrespective of what the server does).
# - 101 Switching Protocols is not allowed for HTTP/2.
# - 102 Processing is WebDAV only and also ignorable.
# - 103 Early Hints is not mission-critical.
headers = http.Headers(event.headers)
status: str | int = "<unknown status>"
try:
status = int(headers[":status"])
reason = status_codes.RESPONSES.get(status, "")
except (KeyError, ValueError):
reason = ""
yield Log(f"Swallowing HTTP/2 informational response: {status} {reason}")
return False
elif isinstance(event, h2.events.RequestReceived):
yield from self.protocol_error(
f"HTTP/2 protocol error: received request from server"
)
return True
elif isinstance(event, h2.events.RemoteSettingsChanged):
# We have received at least one settings from now,
# which means we can rely on the max concurrency in remote_settings
self.provisional_max_concurrency = None
return (yield from super().handle_h2_event(event))
else:
return (yield from super().handle_h2_event(event))
def split_pseudo_headers(
h2_headers: Sequence[tuple[bytes, bytes]],
) -> tuple[dict[bytes, bytes], http.Headers]:
pseudo_headers: dict[bytes, bytes] = {}
i = 0
for header, value in h2_headers:
if header.startswith(b":"):
if header in pseudo_headers:
raise ValueError(f"Duplicate HTTP/2 pseudo header: {header!r}")
pseudo_headers[header] = value
i += 1
else:
# Pseudo-headers must be at the start, we are done here.
break
headers = http.Headers(h2_headers[i:])
return pseudo_headers, headers
def parse_h2_request_headers(
h2_headers: Sequence[tuple[bytes, bytes]],
) -> tuple[str, int, bytes, bytes, bytes, bytes, http.Headers]:
"""Split HTTP/2 pseudo-headers from the actual headers and parse them."""
pseudo_headers, headers = split_pseudo_headers(h2_headers)
try:
method: bytes = pseudo_headers.pop(b":method")
scheme: bytes = pseudo_headers.pop(
b":scheme"
) # this raises for HTTP/2 CONNECT requests
path: bytes = pseudo_headers.pop(b":path")
authority: bytes = pseudo_headers.pop(b":authority", b"")
except KeyError as e:
raise ValueError(f"Required pseudo header is missing: {e}")
if pseudo_headers:
raise ValueError(f"Unknown pseudo headers: {pseudo_headers}")
if authority:
host, port = url.parse_authority(authority, check=True)
if port is None:
port = 80 if scheme == b"http" else 443
else:
host = ""
port = 0
return host, port, method, scheme, authority, path, headers
def parse_h2_response_headers(
h2_headers: Sequence[tuple[bytes, bytes]],
) -> tuple[int, http.Headers]:
"""Split HTTP/2 pseudo-headers from the actual headers and parse them."""
pseudo_headers, headers = split_pseudo_headers(h2_headers)
try:
status_code: int = int(pseudo_headers.pop(b":status"))
except KeyError as e:
raise ValueError(f"Required pseudo header is missing: {e}")
if pseudo_headers:
raise ValueError(f"Unknown pseudo headers: {pseudo_headers}")
return status_code, headers
__all__ = [
"format_h2_request_headers",
"format_h2_response_headers",
"parse_h2_request_headers",
"parse_h2_response_headers",
"Http2Client",
"Http2Server",
]

309
venv/Lib/site-packages/mitmproxy/proxy/layers/http/_http3.py Normal file
View File

@@ -0,0 +1,309 @@
import time
from abc import abstractmethod
from typing import assert_never
from aioquic.h3.connection import ErrorCode as H3ErrorCode
from aioquic.h3.connection import FrameUnexpected as H3FrameUnexpected
from aioquic.h3.events import DataReceived
from aioquic.h3.events import HeadersReceived
from aioquic.h3.events import PushPromiseReceived
from . import ErrorCode
from . import RequestData
from . import RequestEndOfMessage
from . import RequestHeaders
from . import RequestProtocolError
from . import RequestTrailers
from . import ResponseData
from . import ResponseEndOfMessage
from . import ResponseHeaders
from . import ResponseProtocolError
from . import ResponseTrailers
from ._base import format_error
from ._base import HttpConnection
from ._base import HttpEvent
from ._base import ReceiveHttp
from ._http2 import format_h2_request_headers
from ._http2 import format_h2_response_headers
from ._http2 import parse_h2_request_headers
from ._http2 import parse_h2_response_headers
from ._http_h3 import LayeredH3Connection
from ._http_h3 import StreamClosed
from ._http_h3 import TrailersReceived
from mitmproxy import connection
from mitmproxy import http
from mitmproxy import version
from mitmproxy.proxy import commands
from mitmproxy.proxy import context
from mitmproxy.proxy import events
from mitmproxy.proxy import layer
from mitmproxy.proxy.layers.quic import error_code_to_str
from mitmproxy.proxy.layers.quic import QuicConnectionClosed
from mitmproxy.proxy.layers.quic import QuicStreamEvent
from mitmproxy.proxy.utils import expect
class Http3Connection(HttpConnection):
h3_conn: LayeredH3Connection
ReceiveData: type[RequestData | ResponseData]
ReceiveEndOfMessage: type[RequestEndOfMessage | ResponseEndOfMessage]
ReceiveProtocolError: type[RequestProtocolError | ResponseProtocolError]
ReceiveTrailers: type[RequestTrailers | ResponseTrailers]
def __init__(self, context: context.Context, conn: connection.Connection):
super().__init__(context, conn)
self.h3_conn = LayeredH3Connection(
self.conn, is_client=self.conn is self.context.server
)
def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]:
if isinstance(event, events.Start):
yield from self.h3_conn.transmit()
# send mitmproxy HTTP events over the H3 connection
elif isinstance(event, HttpEvent):
try:
if isinstance(event, (RequestData, ResponseData)):
self.h3_conn.send_data(event.stream_id, event.data)
elif isinstance(event, (RequestHeaders, ResponseHeaders)):
headers = yield from (
format_h2_request_headers(self.context, event)
if isinstance(event, RequestHeaders)
else format_h2_response_headers(self.context, event)
)
self.h3_conn.send_headers(
event.stream_id, headers, end_stream=event.end_stream
)
elif isinstance(event, (RequestTrailers, ResponseTrailers)):
self.h3_conn.send_trailers(
event.stream_id, [*event.trailers.fields]
)
elif isinstance(event, (RequestEndOfMessage, ResponseEndOfMessage)):
self.h3_conn.end_stream(event.stream_id)
elif isinstance(event, (RequestProtocolError, ResponseProtocolError)):
status = event.code.http_status_code()
if (
isinstance(event, ResponseProtocolError)
and not self.h3_conn.has_sent_headers(event.stream_id)
and status is not None
):
self.h3_conn.send_headers(
event.stream_id,
[
(b":status", b"%d" % status),
(b"server", version.MITMPROXY.encode()),
(b"content-type", b"text/html"),
],
)
self.h3_conn.send_data(
event.stream_id,
format_error(status, event.message),
end_stream=True,
)
else:
match event.code:
case ErrorCode.CANCEL | ErrorCode.CLIENT_DISCONNECTED:
error_code = H3ErrorCode.H3_REQUEST_CANCELLED
case ErrorCode.KILL:
error_code = H3ErrorCode.H3_INTERNAL_ERROR
case ErrorCode.HTTP_1_1_REQUIRED:
error_code = H3ErrorCode.H3_VERSION_FALLBACK
case ErrorCode.PASSTHROUGH_CLOSE:
# FIXME: This probably shouldn't be a protocol error, but an EOM event.
error_code = H3ErrorCode.H3_REQUEST_CANCELLED
case (
ErrorCode.GENERIC_CLIENT_ERROR
| ErrorCode.GENERIC_SERVER_ERROR
| ErrorCode.REQUEST_TOO_LARGE
| ErrorCode.RESPONSE_TOO_LARGE
| ErrorCode.CONNECT_FAILED
| ErrorCode.DESTINATION_UNKNOWN
| ErrorCode.REQUEST_VALIDATION_FAILED
| ErrorCode.RESPONSE_VALIDATION_FAILED
):
error_code = H3ErrorCode.H3_INTERNAL_ERROR
case other: # pragma: no cover
assert_never(other)
self.h3_conn.close_stream(event.stream_id, error_code.value)
else: # pragma: no cover
raise AssertionError(f"Unexpected event: {event!r}")
except H3FrameUnexpected as e:
# Http2Connection also ignores HttpEvents that violate the current stream state
yield commands.Log(f"Received {event!r} unexpectedly: {e}")
else:
# transmit buffered data
yield from self.h3_conn.transmit()
# forward stream messages from the QUIC layer to the H3 connection
elif isinstance(event, QuicStreamEvent):
h3_events = self.h3_conn.handle_stream_event(event)
for h3_event in h3_events:
if isinstance(h3_event, StreamClosed):
err_str = error_code_to_str(h3_event.error_code)
match h3_event.error_code:
case H3ErrorCode.H3_REQUEST_CANCELLED:
err_code = ErrorCode.CANCEL
case H3ErrorCode.H3_VERSION_FALLBACK:
err_code = ErrorCode.HTTP_1_1_REQUIRED
case _:
err_code = self.ReceiveProtocolError.code
yield ReceiveHttp(
self.ReceiveProtocolError(
h3_event.stream_id,
f"stream closed by client ({err_str})",
code=err_code,
)
)
elif isinstance(h3_event, DataReceived):
if h3_event.data:
yield ReceiveHttp(
self.ReceiveData(h3_event.stream_id, h3_event.data)
)
if h3_event.stream_ended:
yield ReceiveHttp(self.ReceiveEndOfMessage(h3_event.stream_id))
elif isinstance(h3_event, HeadersReceived):
try:
receive_event = self.parse_headers(h3_event)
except ValueError as e:
self.h3_conn.close_connection(
error_code=H3ErrorCode.H3_GENERAL_PROTOCOL_ERROR,
reason_phrase=f"Invalid HTTP/3 request headers: {e}",
)
else:
yield ReceiveHttp(receive_event)
if h3_event.stream_ended:
yield ReceiveHttp(
self.ReceiveEndOfMessage(h3_event.stream_id)
)
elif isinstance(h3_event, TrailersReceived):
yield ReceiveHttp(
self.ReceiveTrailers(
h3_event.stream_id, http.Headers(h3_event.trailers)
)
)
if h3_event.stream_ended:
yield ReceiveHttp(self.ReceiveEndOfMessage(h3_event.stream_id))
elif isinstance(h3_event, PushPromiseReceived): # pragma: no cover
self.h3_conn.close_connection(
error_code=H3ErrorCode.H3_GENERAL_PROTOCOL_ERROR,
reason_phrase=f"Received HTTP/3 push promise, even though we signalled no support.",
)
else: # pragma: no cover
raise AssertionError(f"Unexpected event: {event!r}")
yield from self.h3_conn.transmit()
# report a protocol error for all remaining open streams when a connection is closed
elif isinstance(event, QuicConnectionClosed):
self._handle_event = self.done # type: ignore
self.h3_conn.handle_connection_closed(event)
msg = event.reason_phrase or error_code_to_str(event.error_code)
for stream_id in self.h3_conn.get_open_stream_ids():
yield ReceiveHttp(
self.ReceiveProtocolError(
stream_id, msg, self.ReceiveProtocolError.code
)
)
else: # pragma: no cover
raise AssertionError(f"Unexpected event: {event!r}")
@expect(HttpEvent, QuicStreamEvent, QuicConnectionClosed)
def done(self, _) -> layer.CommandGenerator[None]:
yield from ()
@abstractmethod
def parse_headers(self, event: HeadersReceived) -> RequestHeaders | ResponseHeaders:
pass # pragma: no cover
class Http3Server(Http3Connection):
ReceiveData = RequestData
ReceiveEndOfMessage = RequestEndOfMessage
ReceiveProtocolError = RequestProtocolError
ReceiveTrailers = RequestTrailers
def __init__(self, context: context.Context):
super().__init__(context, context.client)
def parse_headers(self, event: HeadersReceived) -> RequestHeaders | ResponseHeaders:
# same as HTTP/2
(
host,
port,
method,
scheme,
authority,
path,
headers,
) = parse_h2_request_headers(event.headers)
request = http.Request(
host=host,
port=port,
method=method,
scheme=scheme,
authority=authority,
path=path,
http_version=b"HTTP/3",
headers=headers,
content=None,
trailers=None,
timestamp_start=time.time(),
timestamp_end=None,
)
return RequestHeaders(event.stream_id, request, end_stream=event.stream_ended)
class Http3Client(Http3Connection):
ReceiveData = ResponseData
ReceiveEndOfMessage = ResponseEndOfMessage
ReceiveProtocolError = ResponseProtocolError
ReceiveTrailers = ResponseTrailers
our_stream_id: dict[int, int]
their_stream_id: dict[int, int]
def __init__(self, context: context.Context):
super().__init__(context, context.server)
self.our_stream_id = {}
self.their_stream_id = {}
def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]:
# QUIC and HTTP/3 would actually allow for direct stream ID mapping, but since we want
# to support H2<->H3, we need to translate IDs.
# NOTE: We always create bidirectional streams, as we can't safely infer unidirectionality.
if isinstance(event, HttpEvent):
ours = self.our_stream_id.get(event.stream_id, None)
if ours is None:
ours = self.h3_conn.get_next_available_stream_id()
self.our_stream_id[event.stream_id] = ours
self.their_stream_id[ours] = event.stream_id
event.stream_id = ours
for cmd in super()._handle_event(event):
if isinstance(cmd, ReceiveHttp):
cmd.event.stream_id = self.their_stream_id[cmd.event.stream_id]
yield cmd
def parse_headers(self, event: HeadersReceived) -> RequestHeaders | ResponseHeaders:
# same as HTTP/2
status_code, headers = parse_h2_response_headers(event.headers)
response = http.Response(
http_version=b"HTTP/3",
status_code=status_code,
reason=b"",
headers=headers,
content=None,
trailers=None,
timestamp_start=time.time(),
timestamp_end=None,
)
return ResponseHeaders(event.stream_id, response, event.stream_ended)
__all__ = [
"Http3Client",
"Http3Server",
]

207
venv/Lib/site-packages/mitmproxy/proxy/layers/http/_http_h2.py Normal file
View File

@@ -0,0 +1,207 @@
import collections
import logging
from typing import NamedTuple
import h2.config
import h2.connection
import h2.events
import h2.exceptions
import h2.settings
import h2.stream
logger = logging.getLogger(__name__)
class H2ConnectionLogger(h2.config.DummyLogger):
def __init__(self, peername: tuple, conn_type: str):
super().__init__()
self.peername = peername
self.conn_type = conn_type
def debug(self, fmtstr, *args):
logger.debug(
f"{self.conn_type} {fmtstr}", *args, extra={"client": self.peername}
)
def trace(self, fmtstr, *args):
logger.log(
logging.DEBUG - 1,
f"{self.conn_type} {fmtstr}",
*args,
extra={"client": self.peername},
)
class SendH2Data(NamedTuple):
data: bytes
end_stream: bool
class BufferedH2Connection(h2.connection.H2Connection):
"""
This class wrap's hyper-h2's H2Connection and adds internal send buffers.
To simplify implementation, padding is unsupported.
"""
stream_buffers: collections.defaultdict[int, collections.deque[SendH2Data]]
stream_trailers: dict[int, list[tuple[bytes, bytes]]]
def __init__(self, config: h2.config.H2Configuration):
super().__init__(config)
self.local_settings.initial_window_size = 2**31 - 1
self.local_settings.max_frame_size = 2**17
self.max_inbound_frame_size = 2**17
# hyper-h2 pitfall: we need to acknowledge here, otherwise its sends out the old settings.
self.local_settings.acknowledge()
self.stream_buffers = collections.defaultdict(collections.deque)
self.stream_trailers = {}
def initiate_connection(self):
super().initiate_connection()
# We increase the flow-control window for new streams with a setting,
# but we need to increase the overall connection flow-control window as well.
self.increment_flow_control_window(
2**31 - 1 - self.inbound_flow_control_window
) # maximum - default
def send_data(
self,
stream_id: int,
data: bytes,
end_stream: bool = False,
pad_length: None = None,
) -> None:
"""
Send data on a given stream.
In contrast to plain hyper-h2, this method will not raise if the data cannot be sent immediately.
Data is split up and buffered internally.
"""
frame_size = len(data)
assert pad_length is None
if frame_size > self.max_outbound_frame_size:
for start in range(0, frame_size, self.max_outbound_frame_size):
chunk = data[start : start + self.max_outbound_frame_size]
self.send_data(stream_id, chunk, end_stream=False)
return
if self.stream_buffers.get(stream_id, None):
# We already have some data buffered, let's append.
self.stream_buffers[stream_id].append(SendH2Data(data, end_stream))
else:
available_window = self.local_flow_control_window(stream_id)
if frame_size <= available_window:
super().send_data(stream_id, data, end_stream)
else:
if available_window:
can_send_now = data[:available_window]
super().send_data(stream_id, can_send_now, end_stream=False)
data = data[available_window:]
# We can't send right now, so we buffer.
self.stream_buffers[stream_id].append(SendH2Data(data, end_stream))
def send_trailers(self, stream_id: int, trailers: list[tuple[bytes, bytes]]):
if self.stream_buffers.get(stream_id, None):
# Though trailers are not subject to flow control, we need to queue them and send strictly after data frames
self.stream_trailers[stream_id] = trailers
else:
self.send_headers(stream_id, trailers, end_stream=True)
def end_stream(self, stream_id: int) -> None:
if stream_id in self.stream_trailers:
return # we already have trailers queued up that will end the stream.
self.send_data(stream_id, b"", end_stream=True)
def reset_stream(self, stream_id: int, error_code: int = 0) -> None:
self.stream_buffers.pop(stream_id, None)
super().reset_stream(stream_id, error_code)
def receive_data(self, data: bytes):
events = super().receive_data(data)
ret = []
for event in events:
if isinstance(event, h2.events.WindowUpdated):
if event.stream_id == 0:
self.connection_window_updated()
else:
self.stream_window_updated(event.stream_id)
continue
elif isinstance(event, h2.events.RemoteSettingsChanged):
if (
h2.settings.SettingCodes.INITIAL_WINDOW_SIZE
in event.changed_settings
):
self.connection_window_updated()
elif isinstance(event, h2.events.StreamReset):
self.stream_buffers.pop(event.stream_id, None)
elif isinstance(event, h2.events.ConnectionTerminated):
self.stream_buffers.clear()
ret.append(event)
return ret
def stream_window_updated(self, stream_id: int) -> bool:
"""
The window for a specific stream has updated. Send as much buffered data as possible.
"""
# If the stream has been reset in the meantime, we just clear the buffer.
try:
stream: h2.stream.H2Stream = self.streams[stream_id]
except KeyError:
stream_was_reset = True
else:
stream_was_reset = stream.state_machine.state not in (
h2.stream.StreamState.OPEN,
h2.stream.StreamState.HALF_CLOSED_REMOTE,
)
if stream_was_reset:
self.stream_buffers.pop(stream_id, None)
return False
available_window = self.local_flow_control_window(stream_id)
sent_any_data = False
while available_window > 0 and stream_id in self.stream_buffers:
chunk: SendH2Data = self.stream_buffers[stream_id].popleft()
if len(chunk.data) > available_window:
# We can't send the entire chunk, so we have to put some bytes back into the buffer.
self.stream_buffers[stream_id].appendleft(
SendH2Data(
data=chunk.data[available_window:],
end_stream=chunk.end_stream,
)
)
chunk = SendH2Data(
data=chunk.data[:available_window],
end_stream=False,
)
super().send_data(stream_id, data=chunk.data, end_stream=chunk.end_stream)
available_window -= len(chunk.data)
if not self.stream_buffers[stream_id]:
del self.stream_buffers[stream_id]
if stream_id in self.stream_trailers:
self.send_headers(
stream_id, self.stream_trailers.pop(stream_id), end_stream=True
)
sent_any_data = True
return sent_any_data
def connection_window_updated(self) -> None:
"""
The connection window has updated. Send data from buffers in a round-robin fashion.
"""
sent_any_data = True
while sent_any_data:
sent_any_data = False
for stream_id in list(self.stream_buffers):
self.stream_buffers[stream_id] = self.stream_buffers.pop(
stream_id
) # move to end of dict
if self.stream_window_updated(stream_id):
sent_any_data = True
if self.outbound_flow_control_window == 0:
return

321
venv/Lib/site-packages/mitmproxy/proxy/layers/http/_http_h3.py Normal file
View File

@@ -0,0 +1,321 @@
from collections.abc import Iterable
from dataclasses import dataclass
from aioquic.h3.connection import FrameUnexpected
from aioquic.h3.connection import H3Connection
from aioquic.h3.connection import H3Event
from aioquic.h3.connection import H3Stream
from aioquic.h3.connection import Headers
from aioquic.h3.connection import HeadersState
from aioquic.h3.events import HeadersReceived
from aioquic.quic.configuration import QuicConfiguration
from aioquic.quic.events import StreamDataReceived
from aioquic.quic.packet import QuicErrorCode
from mitmproxy import connection
from mitmproxy.proxy import commands
from mitmproxy.proxy import layer
from mitmproxy.proxy.layers.quic import CloseQuicConnection
from mitmproxy.proxy.layers.quic import QuicConnectionClosed
from mitmproxy.proxy.layers.quic import QuicStreamDataReceived
from mitmproxy.proxy.layers.quic import QuicStreamEvent
from mitmproxy.proxy.layers.quic import QuicStreamReset
from mitmproxy.proxy.layers.quic import QuicStreamStopSending
from mitmproxy.proxy.layers.quic import ResetQuicStream
from mitmproxy.proxy.layers.quic import SendQuicStreamData
from mitmproxy.proxy.layers.quic import StopSendingQuicStream
@dataclass
class TrailersReceived(H3Event):
"""
The TrailersReceived event is fired whenever trailers are received.
"""
trailers: Headers
"The trailers."
stream_id: int
"The ID of the stream the trailers were received for."
stream_ended: bool
"Whether the STREAM frame had the FIN bit set."
@dataclass
class StreamClosed(H3Event):
"""
The StreamReset event is fired when the peer is sending a CLOSE_STREAM
or a STOP_SENDING frame. For HTTP/3, we don't differentiate between the two.
"""
stream_id: int
"The ID of the stream that was reset."
error_code: int
"""The error code indicating why the stream was closed."""
class MockQuic:
"""
aioquic intermingles QUIC and HTTP/3. This is something we don't want to do because that makes testing much harder.
Instead, we mock our QUIC connection object here and then take out the wire data to be sent.
"""
def __init__(self, conn: connection.Connection, is_client: bool) -> None:
self.conn = conn
self.pending_commands: list[commands.Command] = []
self._next_stream_id: list[int] = [0, 1, 2, 3]
self._is_client = is_client
# the following fields are accessed by H3Connection
self.configuration = QuicConfiguration(is_client=is_client)
self._quic_logger = None
self._remote_max_datagram_frame_size = 0
def close(
self,
error_code: int = QuicErrorCode.NO_ERROR,
frame_type: int | None = None,
reason_phrase: str = "",
) -> None:
# we'll get closed if a protocol error occurs in `H3Connection.handle_event`
# we note the error on the connection and yield a CloseConnection
# this will then call `QuicConnection.close` with the proper values
# once the `Http3Connection` receives `ConnectionClosed`, it will send out `ProtocolError`
self.pending_commands.append(
CloseQuicConnection(self.conn, error_code, frame_type, reason_phrase)
)
def get_next_available_stream_id(self, is_unidirectional: bool = False) -> int:
# since we always reserve the ID, we have to "find" the next ID like `QuicConnection` does
index = (int(is_unidirectional) << 1) | int(not self._is_client)
stream_id = self._next_stream_id[index]
self._next_stream_id[index] = stream_id + 4
return stream_id
def reset_stream(self, stream_id: int, error_code: int) -> None:
self.pending_commands.append(ResetQuicStream(self.conn, stream_id, error_code))
def stop_send(self, stream_id: int, error_code: int) -> None:
self.pending_commands.append(
StopSendingQuicStream(self.conn, stream_id, error_code)
)
def send_stream_data(
self, stream_id: int, data: bytes, end_stream: bool = False
) -> None:
self.pending_commands.append(
SendQuicStreamData(self.conn, stream_id, data, end_stream)
)
class LayeredH3Connection(H3Connection):
"""
Creates a H3 connection using a fake QUIC connection, which allows layer separation.
Also ensures that headers, data and trailers are sent in that order.
"""
def __init__(
self,
conn: connection.Connection,
is_client: bool,
enable_webtransport: bool = False,
) -> None:
self._mock = MockQuic(conn, is_client)
self._closed_streams: set[int] = set()
"""
We keep track of all stream IDs for which we have requested
STOP_SENDING to silently discard incoming data.
"""
super().__init__(self._mock, enable_webtransport) # type: ignore
# aioquic's constructor sets and then uses _max_push_id.
# This is a hack to forcibly disable it.
@property
def _max_push_id(self) -> int | None:
return None
@_max_push_id.setter
def _max_push_id(self, value):
pass
def _after_send(self, stream_id: int, end_stream: bool) -> None:
# if the stream ended, `QuicConnection` has an assert that no further data is being sent
# to catch this more early on, we set the header state on the `H3Stream`
if end_stream:
self._stream[stream_id].headers_send_state = HeadersState.AFTER_TRAILERS
def _handle_request_or_push_frame(
self,
frame_type: int,
frame_data: bytes | None,
stream: H3Stream,
stream_ended: bool,
) -> list[H3Event]:
# turn HeadersReceived into TrailersReceived for trailers
events = super()._handle_request_or_push_frame(
frame_type, frame_data, stream, stream_ended
)
for index, event in enumerate(events):
if (
isinstance(event, HeadersReceived)
and self._stream[event.stream_id].headers_recv_state
== HeadersState.AFTER_TRAILERS
):
events[index] = TrailersReceived(
event.headers, event.stream_id, event.stream_ended
)
return events
def close_connection(
self,
error_code: int = QuicErrorCode.NO_ERROR,
frame_type: int | None = None,
reason_phrase: str = "",
) -> None:
"""Closes the underlying QUIC connection and ignores any incoming events."""
self._is_done = True
self._quic.close(error_code, frame_type, reason_phrase)
def end_stream(self, stream_id: int) -> None:
"""Ends the given stream if not already done so."""
stream = self._get_or_create_stream(stream_id)
if stream.headers_send_state != HeadersState.AFTER_TRAILERS:
super().send_data(stream_id, b"", end_stream=True)
stream.headers_send_state = HeadersState.AFTER_TRAILERS
def get_next_available_stream_id(self, is_unidirectional: bool = False):
"""Reserves and returns the next available stream ID."""
return self._quic.get_next_available_stream_id(is_unidirectional)
def get_open_stream_ids(self) -> Iterable[int]:
"""Iterates over all non-special open streams"""
return (
stream.stream_id
for stream in self._stream.values()
if (
stream.stream_type is None
and not (
stream.headers_recv_state == HeadersState.AFTER_TRAILERS
and stream.headers_send_state == HeadersState.AFTER_TRAILERS
)
)
)
def handle_connection_closed(self, event: QuicConnectionClosed) -> None:
self._is_done = True
def handle_stream_event(self, event: QuicStreamEvent) -> list[H3Event]:
# don't do anything if we're done
if self._is_done:
return []
elif isinstance(event, (QuicStreamReset, QuicStreamStopSending)):
self.close_stream(
event.stream_id,
event.error_code,
stop_send=isinstance(event, QuicStreamStopSending),
)
stream = self._get_or_create_stream(event.stream_id)
stream.ended = True
stream.headers_recv_state = HeadersState.AFTER_TRAILERS
return [StreamClosed(event.stream_id, event.error_code)]
# convert data events from the QUIC layer back to aioquic events
elif isinstance(event, QuicStreamDataReceived):
# Discard contents if we have already sent STOP_SENDING on this stream.
if event.stream_id in self._closed_streams:
return []
elif self._get_or_create_stream(event.stream_id).ended:
# aioquic will not send us any data events once a stream has ended.
# Instead, it will close the connection. We simulate this here for H3 tests.
self.close_connection(
error_code=QuicErrorCode.PROTOCOL_VIOLATION,
reason_phrase="stream already ended",
)
return []
else:
return self.handle_event(
StreamDataReceived(event.data, event.end_stream, event.stream_id)
)
# should never happen
else: # pragma: no cover
raise AssertionError(f"Unexpected event: {event!r}")
def has_sent_headers(self, stream_id: int) -> bool:
"""Indicates whether headers have been sent over the given stream."""
try:
return self._stream[stream_id].headers_send_state != HeadersState.INITIAL
except KeyError:
return False
def close_stream(
self, stream_id: int, error_code: int, stop_send: bool = True
) -> None:
"""Close a stream that hasn't been closed locally yet."""
if stream_id not in self._closed_streams:
self._closed_streams.add(stream_id)
stream = self._get_or_create_stream(stream_id)
stream.headers_send_state = HeadersState.AFTER_TRAILERS
# https://www.rfc-editor.org/rfc/rfc9000.html#section-3.5-8
# An endpoint that wishes to terminate both directions of
# a bidirectional stream can terminate one direction by
# sending a RESET_STREAM frame, and it can encourage prompt
# termination in the opposite direction by sending a
# STOP_SENDING frame.
self._mock.reset_stream(stream_id=stream_id, error_code=error_code)
if stop_send:
self._mock.stop_send(stream_id=stream_id, error_code=error_code)
def send_data(self, stream_id: int, data: bytes, end_stream: bool = False) -> None:
"""Sends data over the given stream."""
super().send_data(stream_id, data, end_stream)
self._after_send(stream_id, end_stream)
def send_datagram(self, flow_id: int, data: bytes) -> None:
# supporting datagrams would require additional information from the underlying QUIC connection
raise NotImplementedError() # pragma: no cover
def send_headers(
self, stream_id: int, headers: Headers, end_stream: bool = False
) -> None:
"""Sends headers over the given stream."""
# ensure we haven't sent something before
stream = self._get_or_create_stream(stream_id)
if stream.headers_send_state != HeadersState.INITIAL:
raise FrameUnexpected("initial HEADERS frame is not allowed in this state")
super().send_headers(stream_id, headers, end_stream)
self._after_send(stream_id, end_stream)
def send_trailers(self, stream_id: int, trailers: Headers) -> None:
"""Sends trailers over the given stream and ends it."""
# ensure we got some headers first
stream = self._get_or_create_stream(stream_id)
if stream.headers_send_state != HeadersState.AFTER_HEADERS:
raise FrameUnexpected("trailing HEADERS frame is not allowed in this state")
super().send_headers(stream_id, trailers, end_stream=True)
self._after_send(stream_id, end_stream=True)
def transmit(self) -> layer.CommandGenerator[None]:
"""Yields all pending commands for the upper QUIC layer."""
while self._mock.pending_commands:
yield self._mock.pending_commands.pop(0)
__all__ = [
"LayeredH3Connection",
"StreamClosed",
"TrailersReceived",
]

105
venv/Lib/site-packages/mitmproxy/proxy/layers/http/_upstream_proxy.py Normal file
View File

@@ -0,0 +1,105 @@
import time
from logging import DEBUG
from h11._receivebuffer import ReceiveBuffer
from mitmproxy import connection
from mitmproxy import http
from mitmproxy.net.http import http1
from mitmproxy.proxy import commands
from mitmproxy.proxy import context
from mitmproxy.proxy import layer
from mitmproxy.proxy import tunnel
from mitmproxy.proxy.layers import tls
from mitmproxy.proxy.layers.http._hooks import HttpConnectUpstreamHook
from mitmproxy.utils import human
class HttpUpstreamProxy(tunnel.TunnelLayer):
buf: ReceiveBuffer
send_connect: bool
conn: connection.Server
tunnel_connection: connection.Server
def __init__(
self, ctx: context.Context, tunnel_conn: connection.Server, send_connect: bool
):
super().__init__(ctx, tunnel_connection=tunnel_conn, conn=ctx.server)
self.buf = ReceiveBuffer()
self.send_connect = send_connect
@classmethod
def make(cls, ctx: context.Context, send_connect: bool) -> tunnel.LayerStack:
assert ctx.server.via
scheme, address = ctx.server.via
assert scheme in ("http", "https")
http_proxy = connection.Server(address=address)
stack = tunnel.LayerStack()
if scheme == "https":
http_proxy.alpn_offers = tls.HTTP1_ALPNS
http_proxy.sni = address[0]
stack /= tls.ServerTLSLayer(ctx, http_proxy)
stack /= cls(ctx, http_proxy, send_connect)
return stack
def start_handshake(self) -> layer.CommandGenerator[None]:
if not self.send_connect:
return (yield from super().start_handshake())
assert self.conn.address
flow = http.HTTPFlow(self.context.client, self.tunnel_connection)
authority = (
self.conn.address[0].encode("idna") + f":{self.conn.address[1]}".encode()
)
headers = http.Headers()
if self.context.options.http_connect_send_host_header:
headers.insert(0, b"Host", authority)
flow.request = http.Request(
host=self.conn.address[0],
port=self.conn.address[1],
method=b"CONNECT",
scheme=b"",
authority=authority,
path=b"",
http_version=b"HTTP/1.1",
headers=headers,
content=b"",
trailers=None,
timestamp_start=time.time(),
timestamp_end=time.time(),
)
yield HttpConnectUpstreamHook(flow)
raw = http1.assemble_request(flow.request)
yield commands.SendData(self.tunnel_connection, raw)
def receive_handshake_data(
self, data: bytes
) -> layer.CommandGenerator[tuple[bool, str | None]]:
if not self.send_connect:
return (yield from super().receive_handshake_data(data))
self.buf += data
response_head = self.buf.maybe_extract_lines()
if response_head:
try:
response = http1.read_response_head([bytes(x) for x in response_head])
except ValueError as e:
proxyaddr = human.format_address(self.tunnel_connection.address)
yield commands.Log(f"{proxyaddr}: {e}")
return False, f"Error connecting to {proxyaddr}: {e}"
if 200 <= response.status_code < 300:
if self.buf:
yield from self.receive_data(bytes(self.buf))
del self.buf
return True, None
else:
proxyaddr = human.format_address(self.tunnel_connection.address)
raw_resp = b"\n".join(response_head)
yield commands.Log(f"{proxyaddr}: {raw_resp!r}", DEBUG)
return (
False,
f"Upstream proxy {proxyaddr} refused HTTP CONNECT request: {response.status_code} {response.reason}",
)
else:
return False, None