diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/__init__.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/__init__.py new file mode 100644 index 0000000000..8110d5952a --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/__init__.py @@ -0,0 +1 @@ +from . import localpaths as _localpaths # noqa: F401 diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/localpaths.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/localpaths.py new file mode 100644 index 0000000000..34e997fc5a --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/localpaths.py @@ -0,0 +1,36 @@ +import os +import sys + +here = os.path.abspath(os.path.dirname(__file__)) +repo_root = os.path.abspath(os.path.join(here, os.pardir)) + +sys.path.insert(0, os.path.join(here)) +sys.path.insert(0, os.path.join(here, "wptserve")) +sys.path.insert(0, os.path.join(here, "third_party", "pywebsocket3")) +sys.path.insert(0, os.path.join(here, "third_party", "atomicwrites")) +sys.path.insert(0, os.path.join(here, "third_party", "attrs", "src")) +sys.path.insert(0, os.path.join(here, "third_party", "html5lib")) +sys.path.insert(0, os.path.join(here, "third_party", "zipp")) +sys.path.insert(0, os.path.join(here, "third_party", "exceptiongroup", "src")) +sys.path.insert(0, os.path.join(here, "third_party", "more-itertools")) +sys.path.insert(0, os.path.join(here, "third_party", "packaging")) +sys.path.insert(0, os.path.join(here, "third_party", "pathlib2")) +sys.path.insert(0, os.path.join(here, "third_party", "pluggy", "src")) +sys.path.insert(0, os.path.join(here, "third_party", "py")) +sys.path.insert(0, os.path.join(here, "third_party", "pytest")) +sys.path.insert(0, os.path.join(here, "third_party", "pytest", "src")) +sys.path.insert(0, os.path.join(here, "third_party", "pytest-asyncio")) +sys.path.insert(0, os.path.join(here, "third_party", "six")) +sys.path.insert(0, os.path.join(here, "third_party", "webencodings")) +sys.path.insert(0, os.path.join(here, "third_party", "h2", "src")) +sys.path.insert(0, os.path.join(here, "third_party", "hpack", "src")) +sys.path.insert(0, os.path.join(here, "third_party", "hyperframe", "src")) +sys.path.insert(0, os.path.join(here, "third_party", "certifi")) +sys.path.insert(0, os.path.join(here, "third_party", "hyper")) +sys.path.insert(0, os.path.join(here, "third_party", "websockets", "src")) +sys.path.insert(0, os.path.join(here, "third_party", "iniconfig", "src")) +if sys.version_info < (3, 8): + sys.path.insert(0, os.path.join(here, "third_party", "importlib_metadata")) +sys.path.insert(0, os.path.join(here, "webdriver")) +sys.path.insert(0, os.path.join(here, "wptrunner")) +sys.path.insert(0, os.path.join(here, "webtransport")) diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/h2/src/h2/__init__.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/h2/src/h2/__init__.py new file mode 100644 index 0000000000..d3fd0cb674 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/h2/src/h2/__init__.py @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- +""" +h2 +~~ + +A HTTP/2 implementation. +""" +__version__ = '4.1.0' diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/h2/src/h2/config.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/h2/src/h2/config.py new file mode 100644 index 0000000000..915df55c15 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/h2/src/h2/config.py @@ -0,0 +1,195 @@ +# -*- coding: utf-8 -*- +""" +h2/config +~~~~~~~~~ + +Objects for controlling the configuration of the HTTP/2 stack. +""" + +import sys + + +class _BooleanConfigOption: + """ + Descriptor for handling a boolean config option. This will block + attempts to set boolean config options to non-bools. + """ + def __init__(self, name): + self.name = name + self.attr_name = '_%s' % self.name + + def __get__(self, instance, owner): + return getattr(instance, self.attr_name) + + def __set__(self, instance, value): + if not isinstance(value, bool): + raise ValueError("%s must be a bool" % self.name) + setattr(instance, self.attr_name, value) + + +class DummyLogger: + """ + A Logger object that does not actual logging, hence a DummyLogger. + + For the class the log operation is merely a no-op. The intent is to avoid + conditionals being sprinkled throughout the h2 code for calls to + logging functions when no logger is passed into the corresponding object. + """ + def __init__(self, *vargs): + pass + + def debug(self, *vargs, **kwargs): + """ + No-op logging. Only level needed for now. + """ + pass + + def trace(self, *vargs, **kwargs): + """ + No-op logging. Only level needed for now. + """ + pass + + +class OutputLogger: + """ + A Logger object that prints to stderr or any other file-like object. + + This class is provided for convenience and not part of the stable API. + + :param file: A file-like object passed to the print function. + Defaults to ``sys.stderr``. + :param trace: Enables trace-level output. Defaults to ``False``. + """ + def __init__(self, file=None, trace_level=False): + super().__init__() + self.file = file or sys.stderr + self.trace_level = trace_level + + def debug(self, fmtstr, *args): + print(f"h2 (debug): {fmtstr % args}", file=self.file) + + def trace(self, fmtstr, *args): + if self.trace_level: + print(f"h2 (trace): {fmtstr % args}", file=self.file) + + +class H2Configuration: + """ + An object that controls the way a single HTTP/2 connection behaves. + + This object allows the users to customize behaviour. In particular, it + allows users to enable or disable optional features, or to otherwise handle + various unusual behaviours. + + This object has very little behaviour of its own: it mostly just ensures + that configuration is self-consistent. + + :param client_side: Whether this object is to be used on the client side of + a connection, or on the server side. Affects the logic used by the + state machine, the default settings values, the allowable stream IDs, + and several other properties. Defaults to ``True``. + :type client_side: ``bool`` + + :param header_encoding: Controls whether the headers emitted by this object + in events are transparently decoded to ``unicode`` strings, and what + encoding is used to do that decoding. This defaults to ``None``, + meaning that headers will be returned as bytes. To automatically + decode headers (that is, to return them as unicode strings), this can + be set to the string name of any encoding, e.g. ``'utf-8'``. + + .. versionchanged:: 3.0.0 + Changed default value from ``'utf-8'`` to ``None`` + + :type header_encoding: ``str``, ``False``, or ``None`` + + :param validate_outbound_headers: Controls whether the headers emitted + by this object are validated against the rules in RFC 7540. + Disabling this setting will cause outbound header validation to + be skipped, and allow the object to emit headers that may be illegal + according to RFC 7540. Defaults to ``True``. + :type validate_outbound_headers: ``bool`` + + :param normalize_outbound_headers: Controls whether the headers emitted + by this object are normalized before sending. Disabling this setting + will cause outbound header normalization to be skipped, and allow + the object to emit headers that may be illegal according to + RFC 7540. Defaults to ``True``. + :type normalize_outbound_headers: ``bool`` + + :param validate_inbound_headers: Controls whether the headers received + by this object are validated against the rules in RFC 7540. + Disabling this setting will cause inbound header validation to + be skipped, and allow the object to receive headers that may be illegal + according to RFC 7540. Defaults to ``True``. + :type validate_inbound_headers: ``bool`` + + :param normalize_inbound_headers: Controls whether the headers received by + this object are normalized according to the rules of RFC 7540. + Disabling this setting may lead to h2 emitting header blocks that + some RFCs forbid, e.g. with multiple cookie fields. + + .. versionadded:: 3.0.0 + + :type normalize_inbound_headers: ``bool`` + + :param logger: A logger that conforms to the requirements for this module, + those being no I/O and no context switches, which is needed in order + to run in asynchronous operation. + + .. versionadded:: 2.6.0 + + :type logger: ``logging.Logger`` + """ + client_side = _BooleanConfigOption('client_side') + validate_outbound_headers = _BooleanConfigOption( + 'validate_outbound_headers' + ) + normalize_outbound_headers = _BooleanConfigOption( + 'normalize_outbound_headers' + ) + validate_inbound_headers = _BooleanConfigOption( + 'validate_inbound_headers' + ) + normalize_inbound_headers = _BooleanConfigOption( + 'normalize_inbound_headers' + ) + + def __init__(self, + client_side=True, + header_encoding=None, + validate_outbound_headers=True, + normalize_outbound_headers=True, + validate_inbound_headers=True, + normalize_inbound_headers=True, + logger=None): + self.client_side = client_side + self.header_encoding = header_encoding + self.validate_outbound_headers = validate_outbound_headers + self.normalize_outbound_headers = normalize_outbound_headers + self.validate_inbound_headers = validate_inbound_headers + self.normalize_inbound_headers = normalize_inbound_headers + self.logger = logger or DummyLogger(__name__) + + @property + def header_encoding(self): + """ + Controls whether the headers emitted by this object in events are + transparently decoded to ``unicode`` strings, and what encoding is used + to do that decoding. This defaults to ``None``, meaning that headers + will be returned as bytes. To automatically decode headers (that is, to + return them as unicode strings), this can be set to the string name of + any encoding, e.g. ``'utf-8'``. + """ + return self._header_encoding + + @header_encoding.setter + def header_encoding(self, value): + """ + Enforces constraints on the value of header encoding. + """ + if not isinstance(value, (bool, str, type(None))): + raise ValueError("header_encoding must be bool, string, or None") + if value is True: + raise ValueError("header_encoding cannot be True") + self._header_encoding = value diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/h2/src/h2/connection.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/h2/src/h2/connection.py new file mode 100644 index 0000000000..25251e20a3 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/h2/src/h2/connection.py @@ -0,0 +1,2048 @@ +# -*- coding: utf-8 -*- +""" +h2/connection +~~~~~~~~~~~~~ + +An implementation of a HTTP/2 connection. +""" +import base64 + +from enum import Enum, IntEnum + +from hyperframe.exceptions import InvalidPaddingError +from hyperframe.frame import ( + GoAwayFrame, WindowUpdateFrame, HeadersFrame, DataFrame, PingFrame, + PushPromiseFrame, SettingsFrame, RstStreamFrame, PriorityFrame, + ContinuationFrame, AltSvcFrame, ExtensionFrame +) +from hpack.hpack import Encoder, Decoder +from hpack.exceptions import HPACKError, OversizedHeaderListError + +from .config import H2Configuration +from .errors import ErrorCodes, _error_code_from_int +from .events import ( + WindowUpdated, RemoteSettingsChanged, PingReceived, PingAckReceived, + SettingsAcknowledged, ConnectionTerminated, PriorityUpdated, + AlternativeServiceAvailable, UnknownFrameReceived +) +from .exceptions import ( + ProtocolError, NoSuchStreamError, FlowControlError, FrameTooLargeError, + TooManyStreamsError, StreamClosedError, StreamIDTooLowError, + NoAvailableStreamIDError, RFC1122Error, DenialOfServiceError +) +from .frame_buffer import FrameBuffer +from .settings import Settings, SettingCodes +from .stream import H2Stream, StreamClosedBy +from .utilities import SizeLimitDict, guard_increment_window +from .windows import WindowManager + + +class ConnectionState(Enum): + IDLE = 0 + CLIENT_OPEN = 1 + SERVER_OPEN = 2 + CLOSED = 3 + + +class ConnectionInputs(Enum): + SEND_HEADERS = 0 + SEND_PUSH_PROMISE = 1 + SEND_DATA = 2 + SEND_GOAWAY = 3 + SEND_WINDOW_UPDATE = 4 + SEND_PING = 5 + SEND_SETTINGS = 6 + SEND_RST_STREAM = 7 + SEND_PRIORITY = 8 + RECV_HEADERS = 9 + RECV_PUSH_PROMISE = 10 + RECV_DATA = 11 + RECV_GOAWAY = 12 + RECV_WINDOW_UPDATE = 13 + RECV_PING = 14 + RECV_SETTINGS = 15 + RECV_RST_STREAM = 16 + RECV_PRIORITY = 17 + SEND_ALTERNATIVE_SERVICE = 18 # Added in 2.3.0 + RECV_ALTERNATIVE_SERVICE = 19 # Added in 2.3.0 + + +class AllowedStreamIDs(IntEnum): + EVEN = 0 + ODD = 1 + + +class H2ConnectionStateMachine: + """ + A single HTTP/2 connection state machine. + + This state machine, while defined in its own class, is logically part of + the H2Connection class also defined in this file. The state machine itself + maintains very little state directly, instead focusing entirely on managing + state transitions. + """ + # For the purposes of this state machine we treat HEADERS and their + # associated CONTINUATION frames as a single jumbo frame. The protocol + # allows/requires this by preventing other frames from being interleved in + # between HEADERS/CONTINUATION frames. + # + # The _transitions dictionary contains a mapping of tuples of + # (state, input) to tuples of (side_effect_function, end_state). This map + # contains all allowed transitions: anything not in this map is invalid + # and immediately causes a transition to ``closed``. + + _transitions = { + # State: idle + (ConnectionState.IDLE, ConnectionInputs.SEND_HEADERS): + (None, ConnectionState.CLIENT_OPEN), + (ConnectionState.IDLE, ConnectionInputs.RECV_HEADERS): + (None, ConnectionState.SERVER_OPEN), + (ConnectionState.IDLE, ConnectionInputs.SEND_SETTINGS): + (None, ConnectionState.IDLE), + (ConnectionState.IDLE, ConnectionInputs.RECV_SETTINGS): + (None, ConnectionState.IDLE), + (ConnectionState.IDLE, ConnectionInputs.SEND_WINDOW_UPDATE): + (None, ConnectionState.IDLE), + (ConnectionState.IDLE, ConnectionInputs.RECV_WINDOW_UPDATE): + (None, ConnectionState.IDLE), + (ConnectionState.IDLE, ConnectionInputs.SEND_PING): + (None, ConnectionState.IDLE), + (ConnectionState.IDLE, ConnectionInputs.RECV_PING): + (None, ConnectionState.IDLE), + (ConnectionState.IDLE, ConnectionInputs.SEND_GOAWAY): + (None, ConnectionState.CLOSED), + (ConnectionState.IDLE, ConnectionInputs.RECV_GOAWAY): + (None, ConnectionState.CLOSED), + (ConnectionState.IDLE, ConnectionInputs.SEND_PRIORITY): + (None, ConnectionState.IDLE), + (ConnectionState.IDLE, ConnectionInputs.RECV_PRIORITY): + (None, ConnectionState.IDLE), + (ConnectionState.IDLE, ConnectionInputs.SEND_ALTERNATIVE_SERVICE): + (None, ConnectionState.SERVER_OPEN), + (ConnectionState.IDLE, ConnectionInputs.RECV_ALTERNATIVE_SERVICE): + (None, ConnectionState.CLIENT_OPEN), + + # State: open, client side. + (ConnectionState.CLIENT_OPEN, ConnectionInputs.SEND_HEADERS): + (None, ConnectionState.CLIENT_OPEN), + (ConnectionState.CLIENT_OPEN, ConnectionInputs.SEND_DATA): + (None, ConnectionState.CLIENT_OPEN), + (ConnectionState.CLIENT_OPEN, ConnectionInputs.SEND_GOAWAY): + (None, ConnectionState.CLOSED), + (ConnectionState.CLIENT_OPEN, ConnectionInputs.SEND_WINDOW_UPDATE): + (None, ConnectionState.CLIENT_OPEN), + (ConnectionState.CLIENT_OPEN, ConnectionInputs.SEND_PING): + (None, ConnectionState.CLIENT_OPEN), + (ConnectionState.CLIENT_OPEN, ConnectionInputs.SEND_SETTINGS): + (None, ConnectionState.CLIENT_OPEN), + (ConnectionState.CLIENT_OPEN, ConnectionInputs.SEND_PRIORITY): + (None, ConnectionState.CLIENT_OPEN), + (ConnectionState.CLIENT_OPEN, ConnectionInputs.RECV_HEADERS): + (None, ConnectionState.CLIENT_OPEN), + (ConnectionState.CLIENT_OPEN, ConnectionInputs.RECV_PUSH_PROMISE): + (None, ConnectionState.CLIENT_OPEN), + (ConnectionState.CLIENT_OPEN, ConnectionInputs.RECV_DATA): + (None, ConnectionState.CLIENT_OPEN), + (ConnectionState.CLIENT_OPEN, ConnectionInputs.RECV_GOAWAY): + (None, ConnectionState.CLOSED), + (ConnectionState.CLIENT_OPEN, ConnectionInputs.RECV_WINDOW_UPDATE): + (None, ConnectionState.CLIENT_OPEN), + (ConnectionState.CLIENT_OPEN, ConnectionInputs.RECV_PING): + (None, ConnectionState.CLIENT_OPEN), + (ConnectionState.CLIENT_OPEN, ConnectionInputs.RECV_SETTINGS): + (None, ConnectionState.CLIENT_OPEN), + (ConnectionState.CLIENT_OPEN, ConnectionInputs.SEND_RST_STREAM): + (None, ConnectionState.CLIENT_OPEN), + (ConnectionState.CLIENT_OPEN, ConnectionInputs.RECV_RST_STREAM): + (None, ConnectionState.CLIENT_OPEN), + (ConnectionState.CLIENT_OPEN, ConnectionInputs.RECV_PRIORITY): + (None, ConnectionState.CLIENT_OPEN), + (ConnectionState.CLIENT_OPEN, + ConnectionInputs.RECV_ALTERNATIVE_SERVICE): + (None, ConnectionState.CLIENT_OPEN), + + # State: open, server side. + (ConnectionState.SERVER_OPEN, ConnectionInputs.SEND_HEADERS): + (None, ConnectionState.SERVER_OPEN), + (ConnectionState.SERVER_OPEN, ConnectionInputs.SEND_PUSH_PROMISE): + (None, ConnectionState.SERVER_OPEN), + (ConnectionState.SERVER_OPEN, ConnectionInputs.SEND_DATA): + (None, ConnectionState.SERVER_OPEN), + (ConnectionState.SERVER_OPEN, ConnectionInputs.SEND_GOAWAY): + (None, ConnectionState.CLOSED), + (ConnectionState.SERVER_OPEN, ConnectionInputs.SEND_WINDOW_UPDATE): + (None, ConnectionState.SERVER_OPEN), + (ConnectionState.SERVER_OPEN, ConnectionInputs.SEND_PING): + (None, ConnectionState.SERVER_OPEN), + (ConnectionState.SERVER_OPEN, ConnectionInputs.SEND_SETTINGS): + (None, ConnectionState.SERVER_OPEN), + (ConnectionState.SERVER_OPEN, ConnectionInputs.SEND_PRIORITY): + (None, ConnectionState.SERVER_OPEN), + (ConnectionState.SERVER_OPEN, ConnectionInputs.RECV_HEADERS): + (None, ConnectionState.SERVER_OPEN), + (ConnectionState.SERVER_OPEN, ConnectionInputs.RECV_DATA): + (None, ConnectionState.SERVER_OPEN), + (ConnectionState.SERVER_OPEN, ConnectionInputs.RECV_GOAWAY): + (None, ConnectionState.CLOSED), + (ConnectionState.SERVER_OPEN, ConnectionInputs.RECV_WINDOW_UPDATE): + (None, ConnectionState.SERVER_OPEN), + (ConnectionState.SERVER_OPEN, ConnectionInputs.RECV_PING): + (None, ConnectionState.SERVER_OPEN), + (ConnectionState.SERVER_OPEN, ConnectionInputs.RECV_SETTINGS): + (None, ConnectionState.SERVER_OPEN), + (ConnectionState.SERVER_OPEN, ConnectionInputs.RECV_PRIORITY): + (None, ConnectionState.SERVER_OPEN), + (ConnectionState.SERVER_OPEN, ConnectionInputs.SEND_RST_STREAM): + (None, ConnectionState.SERVER_OPEN), + (ConnectionState.SERVER_OPEN, ConnectionInputs.RECV_RST_STREAM): + (None, ConnectionState.SERVER_OPEN), + (ConnectionState.SERVER_OPEN, + ConnectionInputs.SEND_ALTERNATIVE_SERVICE): + (None, ConnectionState.SERVER_OPEN), + (ConnectionState.SERVER_OPEN, + ConnectionInputs.RECV_ALTERNATIVE_SERVICE): + (None, ConnectionState.SERVER_OPEN), + + # State: closed + (ConnectionState.CLOSED, ConnectionInputs.SEND_GOAWAY): + (None, ConnectionState.CLOSED), + (ConnectionState.CLOSED, ConnectionInputs.RECV_GOAWAY): + (None, ConnectionState.CLOSED), + } + + def __init__(self): + self.state = ConnectionState.IDLE + + def process_input(self, input_): + """ + Process a specific input in the state machine. + """ + if not isinstance(input_, ConnectionInputs): + raise ValueError("Input must be an instance of ConnectionInputs") + + try: + func, target_state = self._transitions[(self.state, input_)] + except KeyError: + old_state = self.state + self.state = ConnectionState.CLOSED + raise ProtocolError( + "Invalid input %s in state %s" % (input_, old_state) + ) + else: + self.state = target_state + if func is not None: # pragma: no cover + return func() + + return [] + + +class H2Connection: + """ + A low-level HTTP/2 connection object. This handles building and receiving + frames and maintains both connection and per-stream state for all streams + on this connection. + + This wraps a HTTP/2 Connection state machine implementation, ensuring that + frames can only be sent/received when the connection is in a valid state. + It also builds stream state machines on demand to ensure that the + constraints of those state machines are met as well. Attempts to create + frames that cannot be sent will raise a ``ProtocolError``. + + .. versionchanged:: 2.3.0 + Added the ``header_encoding`` keyword argument. + + .. versionchanged:: 2.5.0 + Added the ``config`` keyword argument. Deprecated the ``client_side`` + and ``header_encoding`` parameters. + + .. versionchanged:: 3.0.0 + Removed deprecated parameters and properties. + + :param config: The configuration for the HTTP/2 connection. + + .. versionadded:: 2.5.0 + + :type config: :class:`H2Configuration ` + """ + # The initial maximum outbound frame size. This can be changed by receiving + # a settings frame. + DEFAULT_MAX_OUTBOUND_FRAME_SIZE = 65535 + + # The initial maximum inbound frame size. This is somewhat arbitrarily + # chosen. + DEFAULT_MAX_INBOUND_FRAME_SIZE = 2**24 + + # The highest acceptable stream ID. + HIGHEST_ALLOWED_STREAM_ID = 2**31 - 1 + + # The largest acceptable window increment. + MAX_WINDOW_INCREMENT = 2**31 - 1 + + # The initial default value of SETTINGS_MAX_HEADER_LIST_SIZE. + DEFAULT_MAX_HEADER_LIST_SIZE = 2**16 + + # Keep in memory limited amount of results for streams closes + MAX_CLOSED_STREAMS = 2**16 + + def __init__(self, config=None): + self.state_machine = H2ConnectionStateMachine() + self.streams = {} + self.highest_inbound_stream_id = 0 + self.highest_outbound_stream_id = 0 + self.encoder = Encoder() + self.decoder = Decoder() + + # This won't always actually do anything: for versions of HPACK older + # than 2.3.0 it does nothing. However, we have to try! + self.decoder.max_header_list_size = self.DEFAULT_MAX_HEADER_LIST_SIZE + + #: The configuration for this HTTP/2 connection object. + #: + #: .. versionadded:: 2.5.0 + self.config = config + if self.config is None: + self.config = H2Configuration( + client_side=True, + ) + + # Objects that store settings, including defaults. + # + # We set the MAX_CONCURRENT_STREAMS value to 100 because its default is + # unbounded, and that's a dangerous default because it allows + # essentially unbounded resources to be allocated regardless of how + # they will be used. 100 should be suitable for the average + # application. This default obviously does not apply to the remote + # peer's settings: the remote peer controls them! + # + # We also set MAX_HEADER_LIST_SIZE to a reasonable value. This is to + # advertise our defence against CVE-2016-6581. However, not all + # versions of HPACK will let us do it. That's ok: we should at least + # suggest that we're not vulnerable. + self.local_settings = Settings( + client=self.config.client_side, + initial_values={ + SettingCodes.MAX_CONCURRENT_STREAMS: 100, + SettingCodes.MAX_HEADER_LIST_SIZE: + self.DEFAULT_MAX_HEADER_LIST_SIZE, + } + ) + self.remote_settings = Settings(client=not self.config.client_side) + + # The current value of the connection flow control windows on the + # connection. + self.outbound_flow_control_window = ( + self.remote_settings.initial_window_size + ) + + #: The maximum size of a frame that can be emitted by this peer, in + #: bytes. + self.max_outbound_frame_size = self.remote_settings.max_frame_size + + #: The maximum size of a frame that can be received by this peer, in + #: bytes. + self.max_inbound_frame_size = self.local_settings.max_frame_size + + # Buffer for incoming data. + self.incoming_buffer = FrameBuffer(server=not self.config.client_side) + + # A private variable to store a sequence of received header frames + # until completion. + self._header_frames = [] + + # Data that needs to be sent. + self._data_to_send = bytearray() + + # Keeps track of how streams are closed. + # Used to ensure that we don't blow up in the face of frames that were + # in flight when a RST_STREAM was sent. + # Also used to determine whether we should consider a frame received + # while a stream is closed as either a stream error or a connection + # error. + self._closed_streams = SizeLimitDict( + size_limit=self.MAX_CLOSED_STREAMS + ) + + # The flow control window manager for the connection. + self._inbound_flow_control_window_manager = WindowManager( + max_window_size=self.local_settings.initial_window_size + ) + + # When in doubt use dict-dispatch. + self._frame_dispatch_table = { + HeadersFrame: self._receive_headers_frame, + PushPromiseFrame: self._receive_push_promise_frame, + SettingsFrame: self._receive_settings_frame, + DataFrame: self._receive_data_frame, + WindowUpdateFrame: self._receive_window_update_frame, + PingFrame: self._receive_ping_frame, + RstStreamFrame: self._receive_rst_stream_frame, + PriorityFrame: self._receive_priority_frame, + GoAwayFrame: self._receive_goaway_frame, + ContinuationFrame: self._receive_naked_continuation, + AltSvcFrame: self._receive_alt_svc_frame, + ExtensionFrame: self._receive_unknown_frame + } + + def _prepare_for_sending(self, frames): + if not frames: + return + self._data_to_send += b''.join(f.serialize() for f in frames) + assert all(f.body_len <= self.max_outbound_frame_size for f in frames) + + def _open_streams(self, remainder): + """ + A common method of counting number of open streams. Returns the number + of streams that are open *and* that have (stream ID % 2) == remainder. + While it iterates, also deletes any closed streams. + """ + count = 0 + to_delete = [] + + for stream_id, stream in self.streams.items(): + if stream.open and (stream_id % 2 == remainder): + count += 1 + elif stream.closed: + to_delete.append(stream_id) + + for stream_id in to_delete: + stream = self.streams.pop(stream_id) + self._closed_streams[stream_id] = stream.closed_by + + return count + + @property + def open_outbound_streams(self): + """ + The current number of open outbound streams. + """ + outbound_numbers = int(self.config.client_side) + return self._open_streams(outbound_numbers) + + @property + def open_inbound_streams(self): + """ + The current number of open inbound streams. + """ + inbound_numbers = int(not self.config.client_side) + return self._open_streams(inbound_numbers) + + @property + def inbound_flow_control_window(self): + """ + The size of the inbound flow control window for the connection. This is + rarely publicly useful: instead, use :meth:`remote_flow_control_window + `. This + shortcut is largely present to provide a shortcut to this data. + """ + return self._inbound_flow_control_window_manager.current_window_size + + def _begin_new_stream(self, stream_id, allowed_ids): + """ + Initiate a new stream. + + .. versionchanged:: 2.0.0 + Removed this function from the public API. + + :param stream_id: The ID of the stream to open. + :param allowed_ids: What kind of stream ID is allowed. + """ + self.config.logger.debug( + "Attempting to initiate stream ID %d", stream_id + ) + outbound = self._stream_id_is_outbound(stream_id) + highest_stream_id = ( + self.highest_outbound_stream_id if outbound else + self.highest_inbound_stream_id + ) + + if stream_id <= highest_stream_id: + raise StreamIDTooLowError(stream_id, highest_stream_id) + + if (stream_id % 2) != int(allowed_ids): + raise ProtocolError( + "Invalid stream ID for peer." + ) + + s = H2Stream( + stream_id, + config=self.config, + inbound_window_size=self.local_settings.initial_window_size, + outbound_window_size=self.remote_settings.initial_window_size + ) + self.config.logger.debug("Stream ID %d created", stream_id) + s.max_inbound_frame_size = self.max_inbound_frame_size + s.max_outbound_frame_size = self.max_outbound_frame_size + + self.streams[stream_id] = s + self.config.logger.debug("Current streams: %s", self.streams.keys()) + + if outbound: + self.highest_outbound_stream_id = stream_id + else: + self.highest_inbound_stream_id = stream_id + + return s + + def initiate_connection(self): + """ + Provides any data that needs to be sent at the start of the connection. + Must be called for both clients and servers. + """ + self.config.logger.debug("Initializing connection") + self.state_machine.process_input(ConnectionInputs.SEND_SETTINGS) + if self.config.client_side: + preamble = b'PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n' + else: + preamble = b'' + + f = SettingsFrame(0) + for setting, value in self.local_settings.items(): + f.settings[setting] = value + self.config.logger.debug( + "Send Settings frame: %s", self.local_settings + ) + + self._data_to_send += preamble + f.serialize() + + def initiate_upgrade_connection(self, settings_header=None): + """ + Call to initialise the connection object for use with an upgraded + HTTP/2 connection (i.e. a connection negotiated using the + ``Upgrade: h2c`` HTTP header). + + This method differs from :meth:`initiate_connection + ` in several ways. + Firstly, it handles the additional SETTINGS frame that is sent in the + ``HTTP2-Settings`` header field. When called on a client connection, + this method will return a bytestring that the caller can put in the + ``HTTP2-Settings`` field they send on their initial request. When + called on a server connection, the user **must** provide the value they + received from the client in the ``HTTP2-Settings`` header field to the + ``settings_header`` argument, which will be used appropriately. + + Additionally, this method sets up stream 1 in a half-closed state + appropriate for this side of the connection, to reflect the fact that + the request is already complete. + + Finally, this method also prepares the appropriate preamble to be sent + after the upgrade. + + .. versionadded:: 2.3.0 + + :param settings_header: (optional, server-only): The value of the + ``HTTP2-Settings`` header field received from the client. + :type settings_header: ``bytes`` + + :returns: For clients, a bytestring to put in the ``HTTP2-Settings``. + For servers, returns nothing. + :rtype: ``bytes`` or ``None`` + """ + self.config.logger.debug( + "Upgrade connection. Current settings: %s", self.local_settings + ) + + frame_data = None + # Begin by getting the preamble in place. + self.initiate_connection() + + if self.config.client_side: + f = SettingsFrame(0) + for setting, value in self.local_settings.items(): + f.settings[setting] = value + + frame_data = f.serialize_body() + frame_data = base64.urlsafe_b64encode(frame_data) + elif settings_header: + # We have a settings header from the client. This needs to be + # applied, but we want to throw away the ACK. We do this by + # inserting the data into a Settings frame and then passing it to + # the state machine, but ignoring the return value. + settings_header = base64.urlsafe_b64decode(settings_header) + f = SettingsFrame(0) + f.parse_body(settings_header) + self._receive_settings_frame(f) + + # Set up appropriate state. Stream 1 in a half-closed state: + # half-closed(local) for clients, half-closed(remote) for servers. + # Additionally, we need to set up the Connection state machine. + connection_input = ( + ConnectionInputs.SEND_HEADERS if self.config.client_side + else ConnectionInputs.RECV_HEADERS + ) + self.config.logger.debug("Process input %s", connection_input) + self.state_machine.process_input(connection_input) + + # Set up stream 1. + self._begin_new_stream(stream_id=1, allowed_ids=AllowedStreamIDs.ODD) + self.streams[1].upgrade(self.config.client_side) + return frame_data + + def _get_or_create_stream(self, stream_id, allowed_ids): + """ + Gets a stream by its stream ID. Will create one if one does not already + exist. Use allowed_ids to circumvent the usual stream ID rules for + clients and servers. + + .. versionchanged:: 2.0.0 + Removed this function from the public API. + """ + try: + return self.streams[stream_id] + except KeyError: + return self._begin_new_stream(stream_id, allowed_ids) + + def _get_stream_by_id(self, stream_id): + """ + Gets a stream by its stream ID. Raises NoSuchStreamError if the stream + ID does not correspond to a known stream and is higher than the current + maximum: raises if it is lower than the current maximum. + + .. versionchanged:: 2.0.0 + Removed this function from the public API. + """ + try: + return self.streams[stream_id] + except KeyError: + outbound = self._stream_id_is_outbound(stream_id) + highest_stream_id = ( + self.highest_outbound_stream_id if outbound else + self.highest_inbound_stream_id + ) + + if stream_id > highest_stream_id: + raise NoSuchStreamError(stream_id) + else: + raise StreamClosedError(stream_id) + + def get_next_available_stream_id(self): + """ + Returns an integer suitable for use as the stream ID for the next + stream created by this endpoint. For server endpoints, this stream ID + will be even. For client endpoints, this stream ID will be odd. If no + stream IDs are available, raises :class:`NoAvailableStreamIDError + `. + + .. warning:: The return value from this function does not change until + the stream ID has actually been used by sending or pushing + headers on that stream. For that reason, it should be + called as close as possible to the actual use of the + stream ID. + + .. versionadded:: 2.0.0 + + :raises: :class:`NoAvailableStreamIDError + ` + :returns: The next free stream ID this peer can use to initiate a + stream. + :rtype: ``int`` + """ + # No streams have been opened yet, so return the lowest allowed stream + # ID. + if not self.highest_outbound_stream_id: + next_stream_id = 1 if self.config.client_side else 2 + else: + next_stream_id = self.highest_outbound_stream_id + 2 + self.config.logger.debug( + "Next available stream ID %d", next_stream_id + ) + if next_stream_id > self.HIGHEST_ALLOWED_STREAM_ID: + raise NoAvailableStreamIDError("Exhausted allowed stream IDs") + + return next_stream_id + + def send_headers(self, stream_id, headers, end_stream=False, + priority_weight=None, priority_depends_on=None, + priority_exclusive=None): + """ + Send headers on a given stream. + + This function can be used to send request or response headers: the kind + that are sent depends on whether this connection has been opened as a + client or server connection, and whether the stream was opened by the + remote peer or not. + + If this is a client connection, calling ``send_headers`` will send the + headers as a request. It will also implicitly open the stream being + used. If this is a client connection and ``send_headers`` has *already* + been called, this will send trailers instead. + + If this is a server connection, calling ``send_headers`` will send the + headers as a response. It is a protocol error for a server to open a + stream by sending headers. If this is a server connection and + ``send_headers`` has *already* been called, this will send trailers + instead. + + When acting as a server, you may call ``send_headers`` any number of + times allowed by the following rules, in this order: + + - zero or more times with ``(':status', '1XX')`` (where ``1XX`` is a + placeholder for any 100-level status code). + - once with any other status header. + - zero or one time for trailers. + + That is, you are allowed to send as many informational responses as you + like, followed by one complete response and zero or one HTTP trailer + blocks. + + Clients may send one or two header blocks: one request block, and + optionally one trailer block. + + If it is important to send HPACK "never indexed" header fields (as + defined in `RFC 7451 Section 7.1.3 + `_), the user may + instead provide headers using the HPACK library's :class:`HeaderTuple + ` and :class:`NeverIndexedHeaderTuple + ` objects. + + This method also allows users to prioritize the stream immediately, + by sending priority information on the HEADERS frame directly. To do + this, any one of ``priority_weight``, ``priority_depends_on``, or + ``priority_exclusive`` must be set to a value that is not ``None``. For + more information on the priority fields, see :meth:`prioritize + `. + + .. warning:: In HTTP/2, it is mandatory that all the HTTP/2 special + headers (that is, ones whose header keys begin with ``:``) appear + at the start of the header block, before any normal headers. + + .. versionchanged:: 2.3.0 + Added support for using :class:`HeaderTuple + ` objects to store headers. + + .. versionchanged:: 2.4.0 + Added the ability to provide priority keyword arguments: + ``priority_weight``, ``priority_depends_on``, and + ``priority_exclusive``. + + :param stream_id: The stream ID to send the headers on. If this stream + does not currently exist, it will be created. + :type stream_id: ``int`` + + :param headers: The request/response headers to send. + :type headers: An iterable of two tuples of bytestrings or + :class:`HeaderTuple ` objects. + + :param end_stream: Whether this headers frame should end the stream + immediately (that is, whether no more data will be sent after this + frame). Defaults to ``False``. + :type end_stream: ``bool`` + + :param priority_weight: Sets the priority weight of the stream. See + :meth:`prioritize ` for more + about how this field works. Defaults to ``None``, which means that + no priority information will be sent. + :type priority_weight: ``int`` or ``None`` + + :param priority_depends_on: Sets which stream this one depends on for + priority purposes. See :meth:`prioritize + ` for more about how this + field works. Defaults to ``None``, which means that no priority + information will be sent. + :type priority_depends_on: ``int`` or ``None`` + + :param priority_exclusive: Sets whether this stream exclusively depends + on the stream given in ``priority_depends_on`` for priority + purposes. See :meth:`prioritize + ` for more about how this + field workds. Defaults to ``None``, which means that no priority + information will be sent. + :type priority_depends_on: ``bool`` or ``None`` + + :returns: Nothing + """ + self.config.logger.debug( + "Send headers on stream ID %d", stream_id + ) + + # Check we can open the stream. + if stream_id not in self.streams: + max_open_streams = self.remote_settings.max_concurrent_streams + if (self.open_outbound_streams + 1) > max_open_streams: + raise TooManyStreamsError( + "Max outbound streams is %d, %d open" % + (max_open_streams, self.open_outbound_streams) + ) + + self.state_machine.process_input(ConnectionInputs.SEND_HEADERS) + stream = self._get_or_create_stream( + stream_id, AllowedStreamIDs(self.config.client_side) + ) + frames = stream.send_headers( + headers, self.encoder, end_stream + ) + + # We may need to send priority information. + priority_present = ( + (priority_weight is not None) or + (priority_depends_on is not None) or + (priority_exclusive is not None) + ) + + if priority_present: + if not self.config.client_side: + raise RFC1122Error("Servers SHOULD NOT prioritize streams.") + + headers_frame = frames[0] + headers_frame.flags.add('PRIORITY') + frames[0] = _add_frame_priority( + headers_frame, + priority_weight, + priority_depends_on, + priority_exclusive + ) + + self._prepare_for_sending(frames) + + def send_data(self, stream_id, data, end_stream=False, pad_length=None): + """ + Send data on a given stream. + + This method does no breaking up of data: if the data is larger than the + value returned by :meth:`local_flow_control_window + ` for this stream + then a :class:`FlowControlError ` will + be raised. If the data is larger than :data:`max_outbound_frame_size + ` then a + :class:`FrameTooLargeError ` will be + raised. + + h2 does this to avoid buffering the data internally. If the user + has more data to send than h2 will allow, consider breaking it up + and buffering it externally. + + :param stream_id: The ID of the stream on which to send the data. + :type stream_id: ``int`` + :param data: The data to send on the stream. + :type data: ``bytes`` + :param end_stream: (optional) Whether this is the last data to be sent + on the stream. Defaults to ``False``. + :type end_stream: ``bool`` + :param pad_length: (optional) Length of the padding to apply to the + data frame. Defaults to ``None`` for no use of padding. Note that + a value of ``0`` results in padding of length ``0`` + (with the "padding" flag set on the frame). + + .. versionadded:: 2.6.0 + + :type pad_length: ``int`` + :returns: Nothing + """ + self.config.logger.debug( + "Send data on stream ID %d with len %d", stream_id, len(data) + ) + frame_size = len(data) + if pad_length is not None: + if not isinstance(pad_length, int): + raise TypeError("pad_length must be an int") + if pad_length < 0 or pad_length > 255: + raise ValueError("pad_length must be within range: [0, 255]") + # Account for padding bytes plus the 1-byte padding length field. + frame_size += pad_length + 1 + self.config.logger.debug( + "Frame size on stream ID %d is %d", stream_id, frame_size + ) + + if frame_size > self.local_flow_control_window(stream_id): + raise FlowControlError( + "Cannot send %d bytes, flow control window is %d." % + (frame_size, self.local_flow_control_window(stream_id)) + ) + elif frame_size > self.max_outbound_frame_size: + raise FrameTooLargeError( + "Cannot send frame size %d, max frame size is %d" % + (frame_size, self.max_outbound_frame_size) + ) + + self.state_machine.process_input(ConnectionInputs.SEND_DATA) + frames = self.streams[stream_id].send_data( + data, end_stream, pad_length=pad_length + ) + + self._prepare_for_sending(frames) + + self.outbound_flow_control_window -= frame_size + self.config.logger.debug( + "Outbound flow control window size is %d", + self.outbound_flow_control_window + ) + assert self.outbound_flow_control_window >= 0 + + def end_stream(self, stream_id): + """ + Cleanly end a given stream. + + This method ends a stream by sending an empty DATA frame on that stream + with the ``END_STREAM`` flag set. + + :param stream_id: The ID of the stream to end. + :type stream_id: ``int`` + :returns: Nothing + """ + self.config.logger.debug("End stream ID %d", stream_id) + self.state_machine.process_input(ConnectionInputs.SEND_DATA) + frames = self.streams[stream_id].end_stream() + self._prepare_for_sending(frames) + + def increment_flow_control_window(self, increment, stream_id=None): + """ + Increment a flow control window, optionally for a single stream. Allows + the remote peer to send more data. + + .. versionchanged:: 2.0.0 + Rejects attempts to increment the flow control window by out of + range values with a ``ValueError``. + + :param increment: The amount to increment the flow control window by. + :type increment: ``int`` + :param stream_id: (optional) The ID of the stream that should have its + flow control window opened. If not present or ``None``, the + connection flow control window will be opened instead. + :type stream_id: ``int`` or ``None`` + :returns: Nothing + :raises: ``ValueError`` + """ + if not (1 <= increment <= self.MAX_WINDOW_INCREMENT): + raise ValueError( + "Flow control increment must be between 1 and %d" % + self.MAX_WINDOW_INCREMENT + ) + + self.state_machine.process_input(ConnectionInputs.SEND_WINDOW_UPDATE) + + if stream_id is not None: + stream = self.streams[stream_id] + frames = stream.increase_flow_control_window( + increment + ) + + self.config.logger.debug( + "Increase stream ID %d flow control window by %d", + stream_id, increment + ) + else: + self._inbound_flow_control_window_manager.window_opened(increment) + f = WindowUpdateFrame(0) + f.window_increment = increment + frames = [f] + + self.config.logger.debug( + "Increase connection flow control window by %d", increment + ) + + self._prepare_for_sending(frames) + + def push_stream(self, stream_id, promised_stream_id, request_headers): + """ + Push a response to the client by sending a PUSH_PROMISE frame. + + If it is important to send HPACK "never indexed" header fields (as + defined in `RFC 7451 Section 7.1.3 + `_), the user may + instead provide headers using the HPACK library's :class:`HeaderTuple + ` and :class:`NeverIndexedHeaderTuple + ` objects. + + :param stream_id: The ID of the stream that this push is a response to. + :type stream_id: ``int`` + :param promised_stream_id: The ID of the stream that the pushed + response will be sent on. + :type promised_stream_id: ``int`` + :param request_headers: The headers of the request that the pushed + response will be responding to. + :type request_headers: An iterable of two tuples of bytestrings or + :class:`HeaderTuple ` objects. + :returns: Nothing + """ + self.config.logger.debug( + "Send Push Promise frame on stream ID %d", stream_id + ) + + if not self.remote_settings.enable_push: + raise ProtocolError("Remote peer has disabled stream push") + + self.state_machine.process_input(ConnectionInputs.SEND_PUSH_PROMISE) + stream = self._get_stream_by_id(stream_id) + + # We need to prevent users pushing streams in response to streams that + # they themselves have already pushed: see #163 and RFC 7540 ยง 6.6. The + # easiest way to do that is to assert that the stream_id is not even: + # this shortcut works because only servers can push and the state + # machine will enforce this. + if (stream_id % 2) == 0: + raise ProtocolError("Cannot recursively push streams.") + + new_stream = self._begin_new_stream( + promised_stream_id, AllowedStreamIDs.EVEN + ) + self.streams[promised_stream_id] = new_stream + + frames = stream.push_stream_in_band( + promised_stream_id, request_headers, self.encoder + ) + new_frames = new_stream.locally_pushed() + self._prepare_for_sending(frames + new_frames) + + def ping(self, opaque_data): + """ + Send a PING frame. + + :param opaque_data: A bytestring of length 8 that will be sent in the + PING frame. + :returns: Nothing + """ + self.config.logger.debug("Send Ping frame") + + if not isinstance(opaque_data, bytes) or len(opaque_data) != 8: + raise ValueError("Invalid value for ping data: %r" % opaque_data) + + self.state_machine.process_input(ConnectionInputs.SEND_PING) + f = PingFrame(0) + f.opaque_data = opaque_data + self._prepare_for_sending([f]) + + def reset_stream(self, stream_id, error_code=0): + """ + Reset a stream. + + This method forcibly closes a stream by sending a RST_STREAM frame for + a given stream. This is not a graceful closure. To gracefully end a + stream, try the :meth:`end_stream + ` method. + + :param stream_id: The ID of the stream to reset. + :type stream_id: ``int`` + :param error_code: (optional) The error code to use to reset the + stream. Defaults to :data:`ErrorCodes.NO_ERROR + `. + :type error_code: ``int`` + :returns: Nothing + """ + self.config.logger.debug("Reset stream ID %d", stream_id) + self.state_machine.process_input(ConnectionInputs.SEND_RST_STREAM) + stream = self._get_stream_by_id(stream_id) + frames = stream.reset_stream(error_code) + self._prepare_for_sending(frames) + + def close_connection(self, error_code=0, additional_data=None, + last_stream_id=None): + + """ + Close a connection, emitting a GOAWAY frame. + + .. versionchanged:: 2.4.0 + Added ``additional_data`` and ``last_stream_id`` arguments. + + :param error_code: (optional) The error code to send in the GOAWAY + frame. + :param additional_data: (optional) Additional debug data indicating + a reason for closing the connection. Must be a bytestring. + :param last_stream_id: (optional) The last stream which was processed + by the sender. Defaults to ``highest_inbound_stream_id``. + :returns: Nothing + """ + self.config.logger.debug("Close connection") + self.state_machine.process_input(ConnectionInputs.SEND_GOAWAY) + + # Additional_data must be bytes + if additional_data is not None: + assert isinstance(additional_data, bytes) + + if last_stream_id is None: + last_stream_id = self.highest_inbound_stream_id + + f = GoAwayFrame( + stream_id=0, + last_stream_id=last_stream_id, + error_code=error_code, + additional_data=(additional_data or b'') + ) + self._prepare_for_sending([f]) + + def update_settings(self, new_settings): + """ + Update the local settings. This will prepare and emit the appropriate + SETTINGS frame. + + :param new_settings: A dictionary of {setting: new value} + """ + self.config.logger.debug( + "Update connection settings to %s", new_settings + ) + self.state_machine.process_input(ConnectionInputs.SEND_SETTINGS) + self.local_settings.update(new_settings) + s = SettingsFrame(0) + s.settings = new_settings + self._prepare_for_sending([s]) + + def advertise_alternative_service(self, + field_value, + origin=None, + stream_id=None): + """ + Notify a client about an available Alternative Service. + + An Alternative Service is defined in `RFC 7838 + `_. An Alternative Service + notification informs a client that a given origin is also available + elsewhere. + + Alternative Services can be advertised in two ways. Firstly, they can + be advertised explicitly: that is, a server can say "origin X is also + available at Y". To advertise like this, set the ``origin`` argument + and not the ``stream_id`` argument. Alternatively, they can be + advertised implicitly: that is, a server can say "the origin you're + contacting on stream X is also available at Y". To advertise like this, + set the ``stream_id`` argument and not the ``origin`` argument. + + The explicit method of advertising can be done as long as the + connection is active. The implicit method can only be done after the + client has sent the request headers and before the server has sent the + response headers: outside of those points, h2 will forbid sending + the Alternative Service advertisement by raising a ProtocolError. + + The ``field_value`` parameter is specified in RFC 7838. h2 does + not validate or introspect this argument: the user is required to + ensure that it's well-formed. ``field_value`` corresponds to RFC 7838's + "Alternative Service Field Value". + + .. note:: It is strongly preferred to use the explicit method of + advertising Alternative Services. The implicit method of + advertising Alternative Services has a number of subtleties + and can lead to inconsistencies between the server and + client. h2 allows both mechanisms, but caution is + strongly advised. + + .. versionadded:: 2.3.0 + + :param field_value: The RFC 7838 Alternative Service Field Value. This + argument is not introspected by h2: the user is responsible + for ensuring that it is well-formed. + :type field_value: ``bytes`` + + :param origin: The origin/authority to which the Alternative Service + being advertised applies. Must not be provided at the same time as + ``stream_id``. + :type origin: ``bytes`` or ``None`` + + :param stream_id: The ID of the stream which was sent to the authority + for which this Alternative Service advertisement applies. Must not + be provided at the same time as ``origin``. + :type stream_id: ``int`` or ``None`` + + :returns: Nothing. + """ + if not isinstance(field_value, bytes): + raise ValueError("Field must be bytestring.") + + if origin is not None and stream_id is not None: + raise ValueError("Must not provide both origin and stream_id") + + self.state_machine.process_input( + ConnectionInputs.SEND_ALTERNATIVE_SERVICE + ) + + if origin is not None: + # This ALTSVC is sent on stream zero. + f = AltSvcFrame(stream_id=0) + f.origin = origin + f.field = field_value + frames = [f] + else: + stream = self._get_stream_by_id(stream_id) + frames = stream.advertise_alternative_service(field_value) + + self._prepare_for_sending(frames) + + def prioritize(self, stream_id, weight=None, depends_on=None, + exclusive=None): + """ + Notify a server about the priority of a stream. + + Stream priorities are a form of guidance to a remote server: they + inform the server about how important a given response is, so that the + server may allocate its resources (e.g. bandwidth, CPU time, etc.) + accordingly. This exists to allow clients to ensure that the most + important data arrives earlier, while less important data does not + starve out the more important data. + + Stream priorities are explained in depth in `RFC 7540 Section 5.3 + `_. + + This method updates the priority information of a single stream. It may + be called well before a stream is actively in use, or well after a + stream is closed. + + .. warning:: RFC 7540 allows for servers to change the priority of + streams. However, h2 **does not** allow server + stacks to do this. This is because most clients do not + adequately know how to respond when provided conflicting + priority information, and relatively little utility is + provided by making that functionality available. + + .. note:: h2 **does not** maintain any information about the + RFC 7540 priority tree. That means that h2 does not + prevent incautious users from creating invalid priority + trees, particularly by creating priority loops. While some + basic error checking is provided by h2, users are + strongly recommended to understand their prioritisation + strategies before using the priority tools here. + + .. note:: Priority information is strictly advisory. Servers are + allowed to disregard it entirely. Avoid relying on the idea + that your priority signaling will definitely be obeyed. + + .. versionadded:: 2.4.0 + + :param stream_id: The ID of the stream to prioritize. + :type stream_id: ``int`` + + :param weight: The weight to give the stream. Defaults to ``16``, the + default weight of any stream. May be any value between ``1`` and + ``256`` inclusive. The relative weight of a stream indicates what + proportion of available resources will be allocated to that + stream. + :type weight: ``int`` + + :param depends_on: The ID of the stream on which this stream depends. + This stream will only be progressed if it is impossible to + progress the parent stream (the one on which this one depends). + Passing the value ``0`` means that this stream does not depend on + any other. Defaults to ``0``. + :type depends_on: ``int`` + + :param exclusive: Whether this stream is an exclusive dependency of its + "parent" stream (i.e. the stream given by ``depends_on``). If a + stream is an exclusive dependency of another, that means that all + previously-set children of the parent are moved to become children + of the new exclusively-dependent stream. Defaults to ``False``. + :type exclusive: ``bool`` + """ + if not self.config.client_side: + raise RFC1122Error("Servers SHOULD NOT prioritize streams.") + + self.state_machine.process_input( + ConnectionInputs.SEND_PRIORITY + ) + + frame = PriorityFrame(stream_id) + frame = _add_frame_priority(frame, weight, depends_on, exclusive) + + self._prepare_for_sending([frame]) + + def local_flow_control_window(self, stream_id): + """ + Returns the maximum amount of data that can be sent on stream + ``stream_id``. + + This value will never be larger than the total data that can be sent on + the connection: even if the given stream allows more data, the + connection window provides a logical maximum to the amount of data that + can be sent. + + The maximum data that can be sent in a single data frame on a stream + is either this value, or the maximum frame size, whichever is + *smaller*. + + :param stream_id: The ID of the stream whose flow control window is + being queried. + :type stream_id: ``int`` + :returns: The amount of data in bytes that can be sent on the stream + before the flow control window is exhausted. + :rtype: ``int`` + """ + stream = self._get_stream_by_id(stream_id) + return min( + self.outbound_flow_control_window, + stream.outbound_flow_control_window + ) + + def remote_flow_control_window(self, stream_id): + """ + Returns the maximum amount of data the remote peer can send on stream + ``stream_id``. + + This value will never be larger than the total data that can be sent on + the connection: even if the given stream allows more data, the + connection window provides a logical maximum to the amount of data that + can be sent. + + The maximum data that can be sent in a single data frame on a stream + is either this value, or the maximum frame size, whichever is + *smaller*. + + :param stream_id: The ID of the stream whose flow control window is + being queried. + :type stream_id: ``int`` + :returns: The amount of data in bytes that can be received on the + stream before the flow control window is exhausted. + :rtype: ``int`` + """ + stream = self._get_stream_by_id(stream_id) + return min( + self.inbound_flow_control_window, + stream.inbound_flow_control_window + ) + + def acknowledge_received_data(self, acknowledged_size, stream_id): + """ + Inform the :class:`H2Connection ` that a + certain number of flow-controlled bytes have been processed, and that + the space should be handed back to the remote peer at an opportune + time. + + .. versionadded:: 2.5.0 + + :param acknowledged_size: The total *flow-controlled size* of the data + that has been processed. Note that this must include the amount of + padding that was sent with that data. + :type acknowledged_size: ``int`` + :param stream_id: The ID of the stream on which this data was received. + :type stream_id: ``int`` + :returns: Nothing + :rtype: ``None`` + """ + self.config.logger.debug( + "Ack received data on stream ID %d with size %d", + stream_id, acknowledged_size + ) + if stream_id <= 0: + raise ValueError( + "Stream ID %d is not valid for acknowledge_received_data" % + stream_id + ) + if acknowledged_size < 0: + raise ValueError("Cannot acknowledge negative data") + + frames = [] + + conn_manager = self._inbound_flow_control_window_manager + conn_increment = conn_manager.process_bytes(acknowledged_size) + if conn_increment: + f = WindowUpdateFrame(0) + f.window_increment = conn_increment + frames.append(f) + + try: + stream = self._get_stream_by_id(stream_id) + except StreamClosedError: + # The stream is already gone. We're not worried about incrementing + # the window in this case. + pass + else: + # No point incrementing the windows of closed streams. + if stream.open: + frames.extend( + stream.acknowledge_received_data(acknowledged_size) + ) + + self._prepare_for_sending(frames) + + def data_to_send(self, amount=None): + """ + Returns some data for sending out of the internal data buffer. + + This method is analogous to ``read`` on a file-like object, but it + doesn't block. Instead, it returns as much data as the user asks for, + or less if that much data is not available. It does not perform any + I/O, and so uses a different name. + + :param amount: (optional) The maximum amount of data to return. If not + set, or set to ``None``, will return as much data as possible. + :type amount: ``int`` + :returns: A bytestring containing the data to send on the wire. + :rtype: ``bytes`` + """ + if amount is None: + data = bytes(self._data_to_send) + self._data_to_send = bytearray() + return data + else: + data = bytes(self._data_to_send[:amount]) + self._data_to_send = self._data_to_send[amount:] + return data + + def clear_outbound_data_buffer(self): + """ + Clears the outbound data buffer, such that if this call was immediately + followed by a call to + :meth:`data_to_send `, that + call would return no data. + + This method should not normally be used, but is made available to avoid + exposing implementation details. + """ + self._data_to_send = bytearray() + + def _acknowledge_settings(self): + """ + Acknowledge settings that have been received. + + .. versionchanged:: 2.0.0 + Removed from public API, removed useless ``event`` parameter, made + automatic. + + :returns: Nothing + """ + self.state_machine.process_input(ConnectionInputs.SEND_SETTINGS) + + changes = self.remote_settings.acknowledge() + + if SettingCodes.INITIAL_WINDOW_SIZE in changes: + setting = changes[SettingCodes.INITIAL_WINDOW_SIZE] + self._flow_control_change_from_settings( + setting.original_value, + setting.new_value, + ) + + # HEADER_TABLE_SIZE changes by the remote part affect our encoder: cf. + # RFC 7540 Section 6.5.2. + if SettingCodes.HEADER_TABLE_SIZE in changes: + setting = changes[SettingCodes.HEADER_TABLE_SIZE] + self.encoder.header_table_size = setting.new_value + + if SettingCodes.MAX_FRAME_SIZE in changes: + setting = changes[SettingCodes.MAX_FRAME_SIZE] + self.max_outbound_frame_size = setting.new_value + for stream in self.streams.values(): + stream.max_outbound_frame_size = setting.new_value + + f = SettingsFrame(0) + f.flags.add('ACK') + return [f] + + def _flow_control_change_from_settings(self, old_value, new_value): + """ + Update flow control windows in response to a change in the value of + SETTINGS_INITIAL_WINDOW_SIZE. + + When this setting is changed, it automatically updates all flow control + windows by the delta in the settings values. Note that it does not + increment the *connection* flow control window, per section 6.9.2 of + RFC 7540. + """ + delta = new_value - old_value + + for stream in self.streams.values(): + stream.outbound_flow_control_window = guard_increment_window( + stream.outbound_flow_control_window, + delta + ) + + def _inbound_flow_control_change_from_settings(self, old_value, new_value): + """ + Update remote flow control windows in response to a change in the value + of SETTINGS_INITIAL_WINDOW_SIZE. + + When this setting is changed, it automatically updates all remote flow + control windows by the delta in the settings values. + """ + delta = new_value - old_value + + for stream in self.streams.values(): + stream._inbound_flow_control_change_from_settings(delta) + + def receive_data(self, data): + """ + Pass some received HTTP/2 data to the connection for handling. + + :param data: The data received from the remote peer on the network. + :type data: ``bytes`` + :returns: A list of events that the remote peer triggered by sending + this data. + """ + self.config.logger.trace( + "Process received data on connection. Received data: %r", data + ) + + events = [] + self.incoming_buffer.add_data(data) + self.incoming_buffer.max_frame_size = self.max_inbound_frame_size + + try: + for frame in self.incoming_buffer: + events.extend(self._receive_frame(frame)) + except InvalidPaddingError: + self._terminate_connection(ErrorCodes.PROTOCOL_ERROR) + raise ProtocolError("Received frame with invalid padding.") + except ProtocolError as e: + # For whatever reason, receiving the frame caused a protocol error. + # We should prepare to emit a GoAway frame before throwing the + # exception up further. No need for an event: the exception will + # do fine. + self._terminate_connection(e.error_code) + raise + + return events + + def _receive_frame(self, frame): + """ + Handle a frame received on the connection. + + .. versionchanged:: 2.0.0 + Removed from the public API. + """ + self.config.logger.trace("Received frame: %s", repr(frame)) + try: + # I don't love using __class__ here, maybe reconsider it. + frames, events = self._frame_dispatch_table[frame.__class__](frame) + except StreamClosedError as e: + # If the stream was closed by RST_STREAM, we just send a RST_STREAM + # to the remote peer. Otherwise, this is a connection error, and so + # we will re-raise to trigger one. + if self._stream_is_closed_by_reset(e.stream_id): + f = RstStreamFrame(e.stream_id) + f.error_code = e.error_code + self._prepare_for_sending([f]) + events = e._events + else: + raise + except StreamIDTooLowError as e: + # The stream ID seems invalid. This may happen when the closed + # stream has been cleaned up, or when the remote peer has opened a + # new stream with a higher stream ID than this one, forcing it + # closed implicitly. + # + # Check how the stream was closed: depending on the mechanism, it + # is either a stream error or a connection error. + if self._stream_is_closed_by_reset(e.stream_id): + # Closed by RST_STREAM is a stream error. + f = RstStreamFrame(e.stream_id) + f.error_code = ErrorCodes.STREAM_CLOSED + self._prepare_for_sending([f]) + events = [] + elif self._stream_is_closed_by_end(e.stream_id): + # Closed by END_STREAM is a connection error. + raise StreamClosedError(e.stream_id) + else: + # Closed implicitly, also a connection error, but of type + # PROTOCOL_ERROR. + raise + else: + self._prepare_for_sending(frames) + + return events + + def _terminate_connection(self, error_code): + """ + Terminate the connection early. Used in error handling blocks to send + GOAWAY frames. + """ + f = GoAwayFrame(0) + f.last_stream_id = self.highest_inbound_stream_id + f.error_code = error_code + self.state_machine.process_input(ConnectionInputs.SEND_GOAWAY) + self._prepare_for_sending([f]) + + def _receive_headers_frame(self, frame): + """ + Receive a headers frame on the connection. + """ + # If necessary, check we can open the stream. Also validate that the + # stream ID is valid. + if frame.stream_id not in self.streams: + max_open_streams = self.local_settings.max_concurrent_streams + if (self.open_inbound_streams + 1) > max_open_streams: + raise TooManyStreamsError( + "Max outbound streams is %d, %d open" % + (max_open_streams, self.open_outbound_streams) + ) + + # Let's decode the headers. We handle headers as bytes internally up + # until we hang them off the event, at which point we may optionally + # convert them to unicode. + headers = _decode_headers(self.decoder, frame.data) + + events = self.state_machine.process_input( + ConnectionInputs.RECV_HEADERS + ) + stream = self._get_or_create_stream( + frame.stream_id, AllowedStreamIDs(not self.config.client_side) + ) + frames, stream_events = stream.receive_headers( + headers, + 'END_STREAM' in frame.flags, + self.config.header_encoding + ) + + if 'PRIORITY' in frame.flags: + p_frames, p_events = self._receive_priority_frame(frame) + stream_events[0].priority_updated = p_events[0] + stream_events.extend(p_events) + assert not p_frames + + return frames, events + stream_events + + def _receive_push_promise_frame(self, frame): + """ + Receive a push-promise frame on the connection. + """ + if not self.local_settings.enable_push: + raise ProtocolError("Received pushed stream") + + pushed_headers = _decode_headers(self.decoder, frame.data) + + events = self.state_machine.process_input( + ConnectionInputs.RECV_PUSH_PROMISE + ) + + try: + stream = self._get_stream_by_id(frame.stream_id) + except NoSuchStreamError: + # We need to check if the parent stream was reset by us. If it was + # then we presume that the PUSH_PROMISE was in flight when we reset + # the parent stream. Rather than accept the new stream, just reset + # it. + # + # If this was closed naturally, however, we should call this a + # PROTOCOL_ERROR: pushing a stream on a naturally closed stream is + # a real problem because it creates a brand new stream that the + # remote peer now believes exists. + if (self._stream_closed_by(frame.stream_id) == + StreamClosedBy.SEND_RST_STREAM): + f = RstStreamFrame(frame.promised_stream_id) + f.error_code = ErrorCodes.REFUSED_STREAM + return [f], events + + raise ProtocolError("Attempted to push on closed stream.") + + # We need to prevent peers pushing streams in response to streams that + # they themselves have already pushed: see #163 and RFC 7540 ยง 6.6. The + # easiest way to do that is to assert that the stream_id is not even: + # this shortcut works because only servers can push and the state + # machine will enforce this. + if (frame.stream_id % 2) == 0: + raise ProtocolError("Cannot recursively push streams.") + + try: + frames, stream_events = stream.receive_push_promise_in_band( + frame.promised_stream_id, + pushed_headers, + self.config.header_encoding, + ) + except StreamClosedError: + # The parent stream was reset by us, so we presume that + # PUSH_PROMISE was in flight when we reset the parent stream. + # So we just reset the new stream. + f = RstStreamFrame(frame.promised_stream_id) + f.error_code = ErrorCodes.REFUSED_STREAM + return [f], events + + new_stream = self._begin_new_stream( + frame.promised_stream_id, AllowedStreamIDs.EVEN + ) + self.streams[frame.promised_stream_id] = new_stream + new_stream.remotely_pushed(pushed_headers) + + return frames, events + stream_events + + def _handle_data_on_closed_stream(self, events, exc, frame): + # This stream is already closed - and yet we received a DATA frame. + # The received DATA frame counts towards the connection flow window. + # We need to manually to acknowledge the DATA frame to update the flow + # window of the connection. Otherwise the whole connection stalls due + # the inbound flow window being 0. + frames = [] + conn_manager = self._inbound_flow_control_window_manager + conn_increment = conn_manager.process_bytes( + frame.flow_controlled_length + ) + if conn_increment: + f = WindowUpdateFrame(0) + f.window_increment = conn_increment + frames.append(f) + self.config.logger.debug( + "Received DATA frame on closed stream %d - " + "auto-emitted a WINDOW_UPDATE by %d", + frame.stream_id, conn_increment + ) + f = RstStreamFrame(exc.stream_id) + f.error_code = exc.error_code + frames.append(f) + self.config.logger.debug( + "Stream %d already CLOSED or cleaned up - " + "auto-emitted a RST_FRAME" % frame.stream_id + ) + return frames, events + exc._events + + def _receive_data_frame(self, frame): + """ + Receive a data frame on the connection. + """ + flow_controlled_length = frame.flow_controlled_length + + events = self.state_machine.process_input( + ConnectionInputs.RECV_DATA + ) + self._inbound_flow_control_window_manager.window_consumed( + flow_controlled_length + ) + + try: + stream = self._get_stream_by_id(frame.stream_id) + frames, stream_events = stream.receive_data( + frame.data, + 'END_STREAM' in frame.flags, + flow_controlled_length + ) + except StreamClosedError as e: + # This stream is either marked as CLOSED or already gone from our + # internal state. + return self._handle_data_on_closed_stream(events, e, frame) + + return frames, events + stream_events + + def _receive_settings_frame(self, frame): + """ + Receive a SETTINGS frame on the connection. + """ + events = self.state_machine.process_input( + ConnectionInputs.RECV_SETTINGS + ) + + # This is an ack of the local settings. + if 'ACK' in frame.flags: + changed_settings = self._local_settings_acked() + ack_event = SettingsAcknowledged() + ack_event.changed_settings = changed_settings + events.append(ack_event) + return [], events + + # Add the new settings. + self.remote_settings.update(frame.settings) + events.append( + RemoteSettingsChanged.from_settings( + self.remote_settings, frame.settings + ) + ) + frames = self._acknowledge_settings() + + return frames, events + + def _receive_window_update_frame(self, frame): + """ + Receive a WINDOW_UPDATE frame on the connection. + """ + # hyperframe will take care of validating the window_increment. + # If we reach in here, we can assume a valid value. + + events = self.state_machine.process_input( + ConnectionInputs.RECV_WINDOW_UPDATE + ) + + if frame.stream_id: + try: + stream = self._get_stream_by_id(frame.stream_id) + frames, stream_events = stream.receive_window_update( + frame.window_increment + ) + except StreamClosedError: + return [], events + else: + # Increment our local flow control window. + self.outbound_flow_control_window = guard_increment_window( + self.outbound_flow_control_window, + frame.window_increment + ) + + # FIXME: Should we split this into one event per active stream? + window_updated_event = WindowUpdated() + window_updated_event.stream_id = 0 + window_updated_event.delta = frame.window_increment + stream_events = [window_updated_event] + frames = [] + + return frames, events + stream_events + + def _receive_ping_frame(self, frame): + """ + Receive a PING frame on the connection. + """ + events = self.state_machine.process_input( + ConnectionInputs.RECV_PING + ) + flags = [] + + if 'ACK' in frame.flags: + evt = PingAckReceived() + else: + evt = PingReceived() + + # automatically ACK the PING with the same 'opaque data' + f = PingFrame(0) + f.flags = {'ACK'} + f.opaque_data = frame.opaque_data + flags.append(f) + + evt.ping_data = frame.opaque_data + events.append(evt) + + return flags, events + + def _receive_rst_stream_frame(self, frame): + """ + Receive a RST_STREAM frame on the connection. + """ + events = self.state_machine.process_input( + ConnectionInputs.RECV_RST_STREAM + ) + try: + stream = self._get_stream_by_id(frame.stream_id) + except NoSuchStreamError: + # The stream is missing. That's ok, we just do nothing here. + stream_frames = [] + stream_events = [] + else: + stream_frames, stream_events = stream.stream_reset(frame) + + return stream_frames, events + stream_events + + def _receive_priority_frame(self, frame): + """ + Receive a PRIORITY frame on the connection. + """ + events = self.state_machine.process_input( + ConnectionInputs.RECV_PRIORITY + ) + + event = PriorityUpdated() + event.stream_id = frame.stream_id + event.depends_on = frame.depends_on + event.exclusive = frame.exclusive + + # Weight is an integer between 1 and 256, but the byte only allows + # 0 to 255: add one. + event.weight = frame.stream_weight + 1 + + # A stream may not depend on itself. + if event.depends_on == frame.stream_id: + raise ProtocolError( + "Stream %d may not depend on itself" % frame.stream_id + ) + events.append(event) + + return [], events + + def _receive_goaway_frame(self, frame): + """ + Receive a GOAWAY frame on the connection. + """ + events = self.state_machine.process_input( + ConnectionInputs.RECV_GOAWAY + ) + + # Clear the outbound data buffer: we cannot send further data now. + self.clear_outbound_data_buffer() + + # Fire an appropriate ConnectionTerminated event. + new_event = ConnectionTerminated() + new_event.error_code = _error_code_from_int(frame.error_code) + new_event.last_stream_id = frame.last_stream_id + new_event.additional_data = (frame.additional_data + if frame.additional_data else None) + events.append(new_event) + + return [], events + + def _receive_naked_continuation(self, frame): + """ + A naked CONTINUATION frame has been received. This is always an error, + but the type of error it is depends on the state of the stream and must + transition the state of the stream, so we need to pass it to the + appropriate stream. + """ + stream = self._get_stream_by_id(frame.stream_id) + stream.receive_continuation() + assert False, "Should not be reachable" + + def _receive_alt_svc_frame(self, frame): + """ + An ALTSVC frame has been received. This frame, specified in RFC 7838, + is used to advertise alternative places where the same service can be + reached. + + This frame can optionally be received either on a stream or on stream + 0, and its semantics are different in each case. + """ + events = self.state_machine.process_input( + ConnectionInputs.RECV_ALTERNATIVE_SERVICE + ) + frames = [] + + if frame.stream_id: + # Given that it makes no sense to receive ALTSVC on a stream + # before that stream has been opened with a HEADERS frame, the + # ALTSVC frame cannot create a stream. If the stream is not + # present, we simply ignore the frame. + try: + stream = self._get_stream_by_id(frame.stream_id) + except (NoSuchStreamError, StreamClosedError): + pass + else: + stream_frames, stream_events = stream.receive_alt_svc(frame) + frames.extend(stream_frames) + events.extend(stream_events) + else: + # This frame is sent on stream 0. The origin field on the frame + # must be present, though if it isn't it's not a ProtocolError + # (annoyingly), we just need to ignore it. + if not frame.origin: + return frames, events + + # If we're a server, we want to ignore this (RFC 7838 says so). + if not self.config.client_side: + return frames, events + + event = AlternativeServiceAvailable() + event.origin = frame.origin + event.field_value = frame.field + events.append(event) + + return frames, events + + def _receive_unknown_frame(self, frame): + """ + We have received a frame that we do not understand. This is almost + certainly an extension frame, though it's impossible to be entirely + sure. + + RFC 7540 ยง 5.5 says that we MUST ignore unknown frame types: so we + do. We do notify the user that we received one, however. + """ + # All we do here is log. + self.config.logger.debug( + "Received unknown extension frame (ID %d)", frame.stream_id + ) + event = UnknownFrameReceived() + event.frame = frame + return [], [event] + + def _local_settings_acked(self): + """ + Handle the local settings being ACKed, update internal state. + """ + changes = self.local_settings.acknowledge() + + if SettingCodes.INITIAL_WINDOW_SIZE in changes: + setting = changes[SettingCodes.INITIAL_WINDOW_SIZE] + self._inbound_flow_control_change_from_settings( + setting.original_value, + setting.new_value, + ) + + if SettingCodes.MAX_HEADER_LIST_SIZE in changes: + setting = changes[SettingCodes.MAX_HEADER_LIST_SIZE] + self.decoder.max_header_list_size = setting.new_value + + if SettingCodes.MAX_FRAME_SIZE in changes: + setting = changes[SettingCodes.MAX_FRAME_SIZE] + self.max_inbound_frame_size = setting.new_value + + if SettingCodes.HEADER_TABLE_SIZE in changes: + setting = changes[SettingCodes.HEADER_TABLE_SIZE] + # This is safe across all hpack versions: some versions just won't + # respect it. + self.decoder.max_allowed_table_size = setting.new_value + + return changes + + def _stream_id_is_outbound(self, stream_id): + """ + Returns ``True`` if the stream ID corresponds to an outbound stream + (one initiated by this peer), returns ``False`` otherwise. + """ + return (stream_id % 2 == int(self.config.client_side)) + + def _stream_closed_by(self, stream_id): + """ + Returns how the stream was closed. + + The return value will be either a member of + ``h2.stream.StreamClosedBy`` or ``None``. If ``None``, the stream was + closed implicitly by the peer opening a stream with a higher stream ID + before opening this one. + """ + if stream_id in self.streams: + return self.streams[stream_id].closed_by + if stream_id in self._closed_streams: + return self._closed_streams[stream_id] + return None + + def _stream_is_closed_by_reset(self, stream_id): + """ + Returns ``True`` if the stream was closed by sending or receiving a + RST_STREAM frame. Returns ``False`` otherwise. + """ + return self._stream_closed_by(stream_id) in ( + StreamClosedBy.RECV_RST_STREAM, StreamClosedBy.SEND_RST_STREAM + ) + + def _stream_is_closed_by_end(self, stream_id): + """ + Returns ``True`` if the stream was closed by sending or receiving an + END_STREAM flag in a HEADERS or DATA frame. Returns ``False`` + otherwise. + """ + return self._stream_closed_by(stream_id) in ( + StreamClosedBy.RECV_END_STREAM, StreamClosedBy.SEND_END_STREAM + ) + + +def _add_frame_priority(frame, weight=None, depends_on=None, exclusive=None): + """ + Adds priority data to a given frame. Does not change any flags set on that + frame: if the caller is adding priority information to a HEADERS frame they + must set that themselves. + + This method also deliberately sets defaults for anything missing. + + This method validates the input values. + """ + # A stream may not depend on itself. + if depends_on == frame.stream_id: + raise ProtocolError( + "Stream %d may not depend on itself" % frame.stream_id + ) + + # Weight must be between 1 and 256. + if weight is not None: + if weight > 256 or weight < 1: + raise ProtocolError( + "Weight must be between 1 and 256, not %d" % weight + ) + else: + # Weight is an integer between 1 and 256, but the byte only allows + # 0 to 255: subtract one. + weight -= 1 + + # Set defaults for anything not provided. + weight = weight if weight is not None else 15 + depends_on = depends_on if depends_on is not None else 0 + exclusive = exclusive if exclusive is not None else False + + frame.stream_weight = weight + frame.depends_on = depends_on + frame.exclusive = exclusive + + return frame + + +def _decode_headers(decoder, encoded_header_block): + """ + Decode a HPACK-encoded header block, translating HPACK exceptions into + sensible h2 errors. + + This only ever returns bytestring headers: h2 may emit them as + unicode later, but internally it processes them as bytestrings only. + """ + try: + return decoder.decode(encoded_header_block, raw=True) + except OversizedHeaderListError as e: + # This is a symptom of a HPACK bomb attack: the user has + # disregarded our requirements on how large a header block we'll + # accept. + raise DenialOfServiceError("Oversized header block: %s" % e) + except (HPACKError, IndexError, TypeError, UnicodeDecodeError) as e: + # We should only need HPACKError here, but versions of HPACK older + # than 2.1.0 throw all three others as well. For maximum + # compatibility, catch all of them. + raise ProtocolError("Error decoding header block: %s" % e) diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/h2/src/h2/errors.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/h2/src/h2/errors.py new file mode 100644 index 0000000000..303df59769 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/h2/src/h2/errors.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- +""" +h2/errors +~~~~~~~~~ + +Global error code registry containing the established HTTP/2 error codes. + +The current registry is available at: +https://tools.ietf.org/html/rfc7540#section-11.4 +""" +import enum + + +class ErrorCodes(enum.IntEnum): + """ + All known HTTP/2 error codes. + + .. versionadded:: 2.5.0 + """ + #: Graceful shutdown. + NO_ERROR = 0x0 + + #: Protocol error detected. + PROTOCOL_ERROR = 0x1 + + #: Implementation fault. + INTERNAL_ERROR = 0x2 + + #: Flow-control limits exceeded. + FLOW_CONTROL_ERROR = 0x3 + + #: Settings not acknowledged. + SETTINGS_TIMEOUT = 0x4 + + #: Frame received for closed stream. + STREAM_CLOSED = 0x5 + + #: Frame size incorrect. + FRAME_SIZE_ERROR = 0x6 + + #: Stream not processed. + REFUSED_STREAM = 0x7 + + #: Stream cancelled. + CANCEL = 0x8 + + #: Compression state not updated. + COMPRESSION_ERROR = 0x9 + + #: TCP connection error for CONNECT method. + CONNECT_ERROR = 0xa + + #: Processing capacity exceeded. + ENHANCE_YOUR_CALM = 0xb + + #: Negotiated TLS parameters not acceptable. + INADEQUATE_SECURITY = 0xc + + #: Use HTTP/1.1 for the request. + HTTP_1_1_REQUIRED = 0xd + + +def _error_code_from_int(code): + """ + Given an integer error code, returns either one of :class:`ErrorCodes + ` or, if not present in the known set of codes, + returns the integer directly. + """ + try: + return ErrorCodes(code) + except ValueError: + return code + + +__all__ = ['ErrorCodes'] diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/h2/src/h2/events.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/h2/src/h2/events.py new file mode 100644 index 0000000000..66c3cff4ab --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/h2/src/h2/events.py @@ -0,0 +1,634 @@ +# -*- coding: utf-8 -*- +""" +h2/events +~~~~~~~~~ + +Defines Event types for HTTP/2. + +Events are returned by the H2 state machine to allow implementations to keep +track of events triggered by receiving data. Each time data is provided to the +H2 state machine it processes the data and returns a list of Event objects. +""" +import binascii + +from .settings import ChangedSetting, _setting_code_from_int + + +class Event: + """ + Base class for h2 events. + """ + pass + + +class RequestReceived(Event): + """ + The RequestReceived event is fired whenever request headers are received. + This event carries the HTTP headers for the given request and the stream ID + of the new stream. + + .. versionchanged:: 2.3.0 + Changed the type of ``headers`` to :class:`HeaderTuple + `. This has no effect on current users. + + .. versionchanged:: 2.4.0 + Added ``stream_ended`` and ``priority_updated`` properties. + """ + def __init__(self): + #: The Stream ID for the stream this request was made on. + self.stream_id = None + + #: The request headers. + self.headers = None + + #: If this request also ended the stream, the associated + #: :class:`StreamEnded ` event will be available + #: here. + #: + #: .. versionadded:: 2.4.0 + self.stream_ended = None + + #: If this request also had associated priority information, the + #: associated :class:`PriorityUpdated ` + #: event will be available here. + #: + #: .. versionadded:: 2.4.0 + self.priority_updated = None + + def __repr__(self): + return "" % ( + self.stream_id, self.headers + ) + + +class ResponseReceived(Event): + """ + The ResponseReceived event is fired whenever response headers are received. + This event carries the HTTP headers for the given response and the stream + ID of the new stream. + + .. versionchanged:: 2.3.0 + Changed the type of ``headers`` to :class:`HeaderTuple + `. This has no effect on current users. + + .. versionchanged:: 2.4.0 + Added ``stream_ended`` and ``priority_updated`` properties. + """ + def __init__(self): + #: The Stream ID for the stream this response was made on. + self.stream_id = None + + #: The response headers. + self.headers = None + + #: If this response also ended the stream, the associated + #: :class:`StreamEnded ` event will be available + #: here. + #: + #: .. versionadded:: 2.4.0 + self.stream_ended = None + + #: If this response also had associated priority information, the + #: associated :class:`PriorityUpdated ` + #: event will be available here. + #: + #: .. versionadded:: 2.4.0 + self.priority_updated = None + + def __repr__(self): + return "" % ( + self.stream_id, self.headers + ) + + +class TrailersReceived(Event): + """ + The TrailersReceived event is fired whenever trailers are received on a + stream. Trailers are a set of headers sent after the body of the + request/response, and are used to provide information that wasn't known + ahead of time (e.g. content-length). This event carries the HTTP header + fields that form the trailers and the stream ID of the stream on which they + were received. + + .. versionchanged:: 2.3.0 + Changed the type of ``headers`` to :class:`HeaderTuple + `. This has no effect on current users. + + .. versionchanged:: 2.4.0 + Added ``stream_ended`` and ``priority_updated`` properties. + """ + def __init__(self): + #: The Stream ID for the stream on which these trailers were received. + self.stream_id = None + + #: The trailers themselves. + self.headers = None + + #: Trailers always end streams. This property has the associated + #: :class:`StreamEnded ` in it. + #: + #: .. versionadded:: 2.4.0 + self.stream_ended = None + + #: If the trailers also set associated priority information, the + #: associated :class:`PriorityUpdated ` + #: event will be available here. + #: + #: .. versionadded:: 2.4.0 + self.priority_updated = None + + def __repr__(self): + return "" % ( + self.stream_id, self.headers + ) + + +class _HeadersSent(Event): + """ + The _HeadersSent event is fired whenever headers are sent. + + This is an internal event, used to determine validation steps on + outgoing header blocks. + """ + pass + + +class _ResponseSent(_HeadersSent): + """ + The _ResponseSent event is fired whenever response headers are sent + on a stream. + + This is an internal event, used to determine validation steps on + outgoing header blocks. + """ + pass + + +class _RequestSent(_HeadersSent): + """ + The _RequestSent event is fired whenever request headers are sent + on a stream. + + This is an internal event, used to determine validation steps on + outgoing header blocks. + """ + pass + + +class _TrailersSent(_HeadersSent): + """ + The _TrailersSent event is fired whenever trailers are sent on a + stream. Trailers are a set of headers sent after the body of the + request/response, and are used to provide information that wasn't known + ahead of time (e.g. content-length). + + This is an internal event, used to determine validation steps on + outgoing header blocks. + """ + pass + + +class _PushedRequestSent(_HeadersSent): + """ + The _PushedRequestSent event is fired whenever pushed request headers are + sent. + + This is an internal event, used to determine validation steps on outgoing + header blocks. + """ + pass + + +class InformationalResponseReceived(Event): + """ + The InformationalResponseReceived event is fired when an informational + response (that is, one whose status code is a 1XX code) is received from + the remote peer. + + The remote peer may send any number of these, from zero upwards. These + responses are most commonly sent in response to requests that have the + ``expect: 100-continue`` header field present. Most users can safely + ignore this event unless you are intending to use the + ``expect: 100-continue`` flow, or are for any reason expecting a different + 1XX status code. + + .. versionadded:: 2.2.0 + + .. versionchanged:: 2.3.0 + Changed the type of ``headers`` to :class:`HeaderTuple + `. This has no effect on current users. + + .. versionchanged:: 2.4.0 + Added ``priority_updated`` property. + """ + def __init__(self): + #: The Stream ID for the stream this informational response was made + #: on. + self.stream_id = None + + #: The headers for this informational response. + self.headers = None + + #: If this response also had associated priority information, the + #: associated :class:`PriorityUpdated ` + #: event will be available here. + #: + #: .. versionadded:: 2.4.0 + self.priority_updated = None + + def __repr__(self): + return "" % ( + self.stream_id, self.headers + ) + + +class DataReceived(Event): + """ + The DataReceived event is fired whenever data is received on a stream from + the remote peer. The event carries the data itself, and the stream ID on + which the data was received. + + .. versionchanged:: 2.4.0 + Added ``stream_ended`` property. + """ + def __init__(self): + #: The Stream ID for the stream this data was received on. + self.stream_id = None + + #: The data itself. + self.data = None + + #: The amount of data received that counts against the flow control + #: window. Note that padding counts against the flow control window, so + #: when adjusting flow control you should always use this field rather + #: than ``len(data)``. + self.flow_controlled_length = None + + #: If this data chunk also completed the stream, the associated + #: :class:`StreamEnded ` event will be available + #: here. + #: + #: .. versionadded:: 2.4.0 + self.stream_ended = None + + def __repr__(self): + return ( + "" % ( + self.stream_id, + self.flow_controlled_length, + _bytes_representation(self.data[:20]), + ) + ) + + +class WindowUpdated(Event): + """ + The WindowUpdated event is fired whenever a flow control window changes + size. HTTP/2 defines flow control windows for connections and streams: this + event fires for both connections and streams. The event carries the ID of + the stream to which it applies (set to zero if the window update applies to + the connection), and the delta in the window size. + """ + def __init__(self): + #: The Stream ID of the stream whose flow control window was changed. + #: May be ``0`` if the connection window was changed. + self.stream_id = None + + #: The window delta. + self.delta = None + + def __repr__(self): + return "" % ( + self.stream_id, self.delta + ) + + +class RemoteSettingsChanged(Event): + """ + The RemoteSettingsChanged event is fired whenever the remote peer changes + its settings. It contains a complete inventory of changed settings, + including their previous values. + + In HTTP/2, settings changes need to be acknowledged. h2 automatically + acknowledges settings changes for efficiency. However, it is possible that + the caller may not be happy with the changed setting. + + When this event is received, the caller should confirm that the new + settings are acceptable. If they are not acceptable, the user should close + the connection with the error code :data:`PROTOCOL_ERROR + `. + + .. versionchanged:: 2.0.0 + Prior to this version the user needed to acknowledge settings changes. + This is no longer the case: h2 now automatically acknowledges + them. + """ + def __init__(self): + #: A dictionary of setting byte to + #: :class:`ChangedSetting `, representing + #: the changed settings. + self.changed_settings = {} + + @classmethod + def from_settings(cls, old_settings, new_settings): + """ + Build a RemoteSettingsChanged event from a set of changed settings. + + :param old_settings: A complete collection of old settings, in the form + of a dictionary of ``{setting: value}``. + :param new_settings: All the changed settings and their new values, in + the form of a dictionary of ``{setting: value}``. + """ + e = cls() + for setting, new_value in new_settings.items(): + setting = _setting_code_from_int(setting) + original_value = old_settings.get(setting) + change = ChangedSetting(setting, original_value, new_value) + e.changed_settings[setting] = change + + return e + + def __repr__(self): + return "" % ( + ", ".join(repr(cs) for cs in self.changed_settings.values()), + ) + + +class PingReceived(Event): + """ + The PingReceived event is fired whenever a PING is received. It contains + the 'opaque data' of the PING frame. A ping acknowledgment with the same + 'opaque data' is automatically emitted after receiving a ping. + + .. versionadded:: 3.1.0 + """ + def __init__(self): + #: The data included on the ping. + self.ping_data = None + + def __repr__(self): + return "" % ( + _bytes_representation(self.ping_data), + ) + + +class PingAckReceived(Event): + """ + The PingAckReceived event is fired whenever a PING acknowledgment is + received. It contains the 'opaque data' of the PING+ACK frame, allowing the + user to correlate PINGs and calculate RTT. + + .. versionadded:: 3.1.0 + + .. versionchanged:: 4.0.0 + Removed deprecated but equivalent ``PingAcknowledged``. + """ + def __init__(self): + #: The data included on the ping. + self.ping_data = None + + def __repr__(self): + return "" % ( + _bytes_representation(self.ping_data), + ) + + +class StreamEnded(Event): + """ + The StreamEnded event is fired whenever a stream is ended by a remote + party. The stream may not be fully closed if it has not been closed + locally, but no further data or headers should be expected on that stream. + """ + def __init__(self): + #: The Stream ID of the stream that was closed. + self.stream_id = None + + def __repr__(self): + return "" % self.stream_id + + +class StreamReset(Event): + """ + The StreamReset event is fired in two situations. The first is when the + remote party forcefully resets the stream. The second is when the remote + party has made a protocol error which only affects a single stream. In this + case, h2 will terminate the stream early and return this event. + + .. versionchanged:: 2.0.0 + This event is now fired when h2 automatically resets a stream. + """ + def __init__(self): + #: The Stream ID of the stream that was reset. + self.stream_id = None + + #: The error code given. Either one of :class:`ErrorCodes + #: ` or ``int`` + self.error_code = None + + #: Whether the remote peer sent a RST_STREAM or we did. + self.remote_reset = True + + def __repr__(self): + return "" % ( + self.stream_id, self.error_code, self.remote_reset + ) + + +class PushedStreamReceived(Event): + """ + The PushedStreamReceived event is fired whenever a pushed stream has been + received from a remote peer. The event carries on it the new stream ID, the + ID of the parent stream, and the request headers pushed by the remote peer. + """ + def __init__(self): + #: The Stream ID of the stream created by the push. + self.pushed_stream_id = None + + #: The Stream ID of the stream that the push is related to. + self.parent_stream_id = None + + #: The request headers, sent by the remote party in the push. + self.headers = None + + def __repr__(self): + return ( + "" % ( + self.pushed_stream_id, + self.parent_stream_id, + self.headers, + ) + ) + + +class SettingsAcknowledged(Event): + """ + The SettingsAcknowledged event is fired whenever a settings ACK is received + from the remote peer. The event carries on it the settings that were + acknowedged, in the same format as + :class:`h2.events.RemoteSettingsChanged`. + """ + def __init__(self): + #: A dictionary of setting byte to + #: :class:`ChangedSetting `, representing + #: the changed settings. + self.changed_settings = {} + + def __repr__(self): + return "" % ( + ", ".join(repr(cs) for cs in self.changed_settings.values()), + ) + + +class PriorityUpdated(Event): + """ + The PriorityUpdated event is fired whenever a stream sends updated priority + information. This can occur when the stream is opened, or at any time + during the stream lifetime. + + This event is purely advisory, and does not need to be acted on. + + .. versionadded:: 2.0.0 + """ + def __init__(self): + #: The ID of the stream whose priority information is being updated. + self.stream_id = None + + #: The new stream weight. May be the same as the original stream + #: weight. An integer between 1 and 256. + self.weight = None + + #: The stream ID this stream now depends on. May be ``0``. + self.depends_on = None + + #: Whether the stream *exclusively* depends on the parent stream. If it + #: does, this stream should inherit the current children of its new + #: parent. + self.exclusive = None + + def __repr__(self): + return ( + "" % ( + self.stream_id, + self.weight, + self.depends_on, + self.exclusive + ) + ) + + +class ConnectionTerminated(Event): + """ + The ConnectionTerminated event is fired when a connection is torn down by + the remote peer using a GOAWAY frame. Once received, no further action may + be taken on the connection: a new connection must be established. + """ + def __init__(self): + #: The error code cited when tearing down the connection. Should be + #: one of :class:`ErrorCodes `, but may not be if + #: unknown HTTP/2 extensions are being used. + self.error_code = None + + #: The stream ID of the last stream the remote peer saw. This can + #: provide an indication of what data, if any, never reached the remote + #: peer and so can safely be resent. + self.last_stream_id = None + + #: Additional debug data that can be appended to GOAWAY frame. + self.additional_data = None + + def __repr__(self): + return ( + "" % ( + self.error_code, + self.last_stream_id, + _bytes_representation( + self.additional_data[:20] + if self.additional_data else None) + ) + ) + + +class AlternativeServiceAvailable(Event): + """ + The AlternativeServiceAvailable event is fired when the remote peer + advertises an `RFC 7838 `_ Alternative + Service using an ALTSVC frame. + + This event always carries the origin to which the ALTSVC information + applies. That origin is either supplied by the server directly, or inferred + by h2 from the ``:authority`` pseudo-header field that was sent by + the user when initiating a given stream. + + This event also carries what RFC 7838 calls the "Alternative Service Field + Value", which is formatted like a HTTP header field and contains the + relevant alternative service information. h2 does not parse or in any + way modify that information: the user is required to do that. + + This event can only be fired on the client end of a connection. + + .. versionadded:: 2.3.0 + """ + def __init__(self): + #: The origin to which the alternative service field value applies. + #: This field is either supplied by the server directly, or inferred by + #: h2 from the ``:authority`` pseudo-header field that was sent + #: by the user when initiating the stream on which the frame was + #: received. + self.origin = None + + #: The ALTSVC field value. This contains information about the HTTP + #: alternative service being advertised by the server. h2 does + #: not parse this field: it is left exactly as sent by the server. The + #: structure of the data in this field is given by `RFC 7838 Section 3 + #: `_. + self.field_value = None + + def __repr__(self): + return ( + "" % ( + self.origin.decode('utf-8', 'ignore'), + self.field_value.decode('utf-8', 'ignore'), + ) + ) + + +class UnknownFrameReceived(Event): + """ + The UnknownFrameReceived event is fired when the remote peer sends a frame + that h2 does not understand. This occurs primarily when the remote + peer is employing HTTP/2 extensions that h2 doesn't know anything + about. + + RFC 7540 requires that HTTP/2 implementations ignore these frames. h2 + does so. However, this event is fired to allow implementations to perform + special processing on those frames if needed (e.g. if the implementation + is capable of handling the frame itself). + + .. versionadded:: 2.7.0 + """ + def __init__(self): + #: The hyperframe Frame object that encapsulates the received frame. + self.frame = None + + def __repr__(self): + return "" + + +def _bytes_representation(data): + """ + Converts a bytestring into something that is safe to print on all Python + platforms. + + This function is relatively expensive, so it should not be called on the + mainline of the code. It's safe to use in things like object repr methods + though. + """ + if data is None: + return None + + return binascii.hexlify(data).decode('ascii') diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/h2/src/h2/exceptions.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/h2/src/h2/exceptions.py new file mode 100644 index 0000000000..e22bebc0b5 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/h2/src/h2/exceptions.py @@ -0,0 +1,187 @@ +# -*- coding: utf-8 -*- +""" +h2/exceptions +~~~~~~~~~~~~~ + +Exceptions for the HTTP/2 module. +""" +import h2.errors + + +class H2Error(Exception): + """ + The base class for all exceptions for the HTTP/2 module. + """ + + +class ProtocolError(H2Error): + """ + An action was attempted in violation of the HTTP/2 protocol. + """ + #: The error code corresponds to this kind of Protocol Error. + error_code = h2.errors.ErrorCodes.PROTOCOL_ERROR + + +class FrameTooLargeError(ProtocolError): + """ + The frame that we tried to send or that we received was too large. + """ + #: The error code corresponds to this kind of Protocol Error. + error_code = h2.errors.ErrorCodes.FRAME_SIZE_ERROR + + +class FrameDataMissingError(ProtocolError): + """ + The frame that we received is missing some data. + + .. versionadded:: 2.0.0 + """ + #: The error code corresponds to this kind of Protocol Error. + error_code = h2.errors.ErrorCodes.FRAME_SIZE_ERROR + + +class TooManyStreamsError(ProtocolError): + """ + An attempt was made to open a stream that would lead to too many concurrent + streams. + """ + pass + + +class FlowControlError(ProtocolError): + """ + An attempted action violates flow control constraints. + """ + #: The error code corresponds to this kind of Protocol Error. + error_code = h2.errors.ErrorCodes.FLOW_CONTROL_ERROR + + +class StreamIDTooLowError(ProtocolError): + """ + An attempt was made to open a stream that had an ID that is lower than the + highest ID we have seen on this connection. + """ + def __init__(self, stream_id, max_stream_id): + #: The ID of the stream that we attempted to open. + self.stream_id = stream_id + + #: The current highest-seen stream ID. + self.max_stream_id = max_stream_id + + def __str__(self): + return "StreamIDTooLowError: %d is lower than %d" % ( + self.stream_id, self.max_stream_id + ) + + +class NoAvailableStreamIDError(ProtocolError): + """ + There are no available stream IDs left to the connection. All stream IDs + have been exhausted. + + .. versionadded:: 2.0.0 + """ + pass + + +class NoSuchStreamError(ProtocolError): + """ + A stream-specific action referenced a stream that does not exist. + + .. versionchanged:: 2.0.0 + Became a subclass of :class:`ProtocolError + ` + """ + def __init__(self, stream_id): + #: The stream ID corresponds to the non-existent stream. + self.stream_id = stream_id + + +class StreamClosedError(NoSuchStreamError): + """ + A more specific form of + :class:`NoSuchStreamError `. Indicates + that the stream has since been closed, and that all state relating to that + stream has been removed. + """ + def __init__(self, stream_id): + #: The stream ID corresponds to the nonexistent stream. + self.stream_id = stream_id + + #: The relevant HTTP/2 error code. + self.error_code = h2.errors.ErrorCodes.STREAM_CLOSED + + # Any events that internal code may need to fire. Not relevant to + # external users that may receive a StreamClosedError. + self._events = [] + + +class InvalidSettingsValueError(ProtocolError, ValueError): + """ + An attempt was made to set an invalid Settings value. + + .. versionadded:: 2.0.0 + """ + def __init__(self, msg, error_code): + super(InvalidSettingsValueError, self).__init__(msg) + self.error_code = error_code + + +class InvalidBodyLengthError(ProtocolError): + """ + The remote peer sent more or less data that the Content-Length header + indicated. + + .. versionadded:: 2.0.0 + """ + def __init__(self, expected, actual): + self.expected_length = expected + self.actual_length = actual + + def __str__(self): + return "InvalidBodyLengthError: Expected %d bytes, received %d" % ( + self.expected_length, self.actual_length + ) + + +class UnsupportedFrameError(ProtocolError): + """ + The remote peer sent a frame that is unsupported in this context. + + .. versionadded:: 2.1.0 + + .. versionchanged:: 4.0.0 + Removed deprecated KeyError parent class. + """ + pass + + +class RFC1122Error(H2Error): + """ + Emitted when users attempt to do something that is literally allowed by the + relevant RFC, but is sufficiently ill-defined that it's unwise to allow + users to actually do it. + + While there is some disagreement about whether or not we should be liberal + in what accept, it is a truth universally acknowledged that we should be + conservative in what emit. + + .. versionadded:: 2.4.0 + """ + # shazow says I'm going to regret naming the exception this way. If that + # turns out to be true, TELL HIM NOTHING. + pass + + +class DenialOfServiceError(ProtocolError): + """ + Emitted when the remote peer exhibits a behaviour that is likely to be an + attempt to perform a Denial of Service attack on the implementation. This + is a form of ProtocolError that carries a different error code, and allows + more easy detection of this kind of behaviour. + + .. versionadded:: 2.5.0 + """ + #: The error code corresponds to this kind of + #: :class:`ProtocolError ` + error_code = h2.errors.ErrorCodes.ENHANCE_YOUR_CALM diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/h2/src/h2/frame_buffer.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/h2/src/h2/frame_buffer.py new file mode 100644 index 0000000000..785775eb56 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/h2/src/h2/frame_buffer.py @@ -0,0 +1,160 @@ +# -*- coding: utf-8 -*- +""" +h2/frame_buffer +~~~~~~~~~~~~~~~ + +A data structure that provides a way to iterate over a byte buffer in terms of +frames. +""" +from hyperframe.exceptions import InvalidFrameError, InvalidDataError +from hyperframe.frame import ( + Frame, HeadersFrame, ContinuationFrame, PushPromiseFrame +) + +from .exceptions import ( + ProtocolError, FrameTooLargeError, FrameDataMissingError +) + +# To avoid a DOS attack based on sending loads of continuation frames, we limit +# the maximum number we're perpared to receive. In this case, we'll set the +# limit to 64, which means the largest encoded header block we can receive by +# default is 262144 bytes long, and the largest possible *at all* is 1073741760 +# bytes long. +# +# This value seems reasonable for now, but in future we may want to evaluate +# making it configurable. +CONTINUATION_BACKLOG = 64 + + +class FrameBuffer: + """ + This is a data structure that expects to act as a buffer for HTTP/2 data + that allows iteraton in terms of H2 frames. + """ + def __init__(self, server=False): + self.data = b'' + self.max_frame_size = 0 + self._preamble = b'PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n' if server else b'' + self._preamble_len = len(self._preamble) + self._headers_buffer = [] + + def add_data(self, data): + """ + Add more data to the frame buffer. + + :param data: A bytestring containing the byte buffer. + """ + if self._preamble_len: + data_len = len(data) + of_which_preamble = min(self._preamble_len, data_len) + + if self._preamble[:of_which_preamble] != data[:of_which_preamble]: + raise ProtocolError("Invalid HTTP/2 preamble.") + + data = data[of_which_preamble:] + self._preamble_len -= of_which_preamble + self._preamble = self._preamble[of_which_preamble:] + + self.data += data + + def _validate_frame_length(self, length): + """ + Confirm that the frame is an appropriate length. + """ + if length > self.max_frame_size: + raise FrameTooLargeError( + "Received overlong frame: length %d, max %d" % + (length, self.max_frame_size) + ) + + def _update_header_buffer(self, f): + """ + Updates the internal header buffer. Returns a frame that should replace + the current one. May throw exceptions if this frame is invalid. + """ + # Check if we're in the middle of a headers block. If we are, this + # frame *must* be a CONTINUATION frame with the same stream ID as the + # leading HEADERS or PUSH_PROMISE frame. Anything else is a + # ProtocolError. If the frame *is* valid, append it to the header + # buffer. + if self._headers_buffer: + stream_id = self._headers_buffer[0].stream_id + valid_frame = ( + f is not None and + isinstance(f, ContinuationFrame) and + f.stream_id == stream_id + ) + if not valid_frame: + raise ProtocolError("Invalid frame during header block.") + + # Append the frame to the buffer. + self._headers_buffer.append(f) + if len(self._headers_buffer) > CONTINUATION_BACKLOG: + raise ProtocolError("Too many continuation frames received.") + + # If this is the end of the header block, then we want to build a + # mutant HEADERS frame that's massive. Use the original one we got, + # then set END_HEADERS and set its data appopriately. If it's not + # the end of the block, lose the current frame: we can't yield it. + if 'END_HEADERS' in f.flags: + f = self._headers_buffer[0] + f.flags.add('END_HEADERS') + f.data = b''.join(x.data for x in self._headers_buffer) + self._headers_buffer = [] + else: + f = None + elif (isinstance(f, (HeadersFrame, PushPromiseFrame)) and + 'END_HEADERS' not in f.flags): + # This is the start of a headers block! Save the frame off and then + # act like we didn't receive one. + self._headers_buffer.append(f) + f = None + + return f + + # The methods below support the iterator protocol. + def __iter__(self): + return self + + def __next__(self): + # First, check that we have enough data to successfully parse the + # next frame header. If not, bail. Otherwise, parse it. + if len(self.data) < 9: + raise StopIteration() + + try: + f, length = Frame.parse_frame_header(self.data[:9]) + except (InvalidDataError, InvalidFrameError) as e: # pragma: no cover + raise ProtocolError( + "Received frame with invalid header: %s" % str(e) + ) + + # Next, check that we have enough length to parse the frame body. If + # not, bail, leaving the frame header data in the buffer for next time. + if len(self.data) < length + 9: + raise StopIteration() + + # Confirm the frame has an appropriate length. + self._validate_frame_length(length) + + # Try to parse the frame body + try: + f.parse_body(memoryview(self.data[9:9+length])) + except InvalidDataError: + raise ProtocolError("Received frame with non-compliant data") + except InvalidFrameError: + raise FrameDataMissingError("Frame data missing or invalid") + + # At this point, as we know we'll use or discard the entire frame, we + # can update the data. + self.data = self.data[9+length:] + + # Pass the frame through the header buffer. + f = self._update_header_buffer(f) + + # If we got a frame we didn't understand or shouldn't yield, rather + # than return None it'd be better if we just tried to get the next + # frame in the sequence instead. Recurse back into ourselves to do + # that. This is safe because the amount of work we have to do here is + # strictly bounded by the length of the buffer. + return f if f is not None else self.__next__() diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/h2/src/h2/settings.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/h2/src/h2/settings.py new file mode 100644 index 0000000000..969a162eb6 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/h2/src/h2/settings.py @@ -0,0 +1,334 @@ +# -*- coding: utf-8 -*- +""" +h2/settings +~~~~~~~~~~~ + +This module contains a HTTP/2 settings object. This object provides a simple +API for manipulating HTTP/2 settings, keeping track of both the current active +state of the settings and the unacknowledged future values of the settings. +""" +import collections +from collections.abc import MutableMapping +import enum + +from hyperframe.frame import SettingsFrame + +from h2.errors import ErrorCodes +from h2.exceptions import InvalidSettingsValueError + + +class SettingCodes(enum.IntEnum): + """ + All known HTTP/2 setting codes. + + .. versionadded:: 2.6.0 + """ + + #: Allows the sender to inform the remote endpoint of the maximum size of + #: the header compression table used to decode header blocks, in octets. + HEADER_TABLE_SIZE = SettingsFrame.HEADER_TABLE_SIZE + + #: This setting can be used to disable server push. To disable server push + #: on a client, set this to 0. + ENABLE_PUSH = SettingsFrame.ENABLE_PUSH + + #: Indicates the maximum number of concurrent streams that the sender will + #: allow. + MAX_CONCURRENT_STREAMS = SettingsFrame.MAX_CONCURRENT_STREAMS + + #: Indicates the sender's initial window size (in octets) for stream-level + #: flow control. + INITIAL_WINDOW_SIZE = SettingsFrame.INITIAL_WINDOW_SIZE + + #: Indicates the size of the largest frame payload that the sender is + #: willing to receive, in octets. + MAX_FRAME_SIZE = SettingsFrame.MAX_FRAME_SIZE + + #: This advisory setting informs a peer of the maximum size of header list + #: that the sender is prepared to accept, in octets. The value is based on + #: the uncompressed size of header fields, including the length of the name + #: and value in octets plus an overhead of 32 octets for each header field. + MAX_HEADER_LIST_SIZE = SettingsFrame.MAX_HEADER_LIST_SIZE + + #: This setting can be used to enable the connect protocol. To enable on a + #: client set this to 1. + ENABLE_CONNECT_PROTOCOL = SettingsFrame.ENABLE_CONNECT_PROTOCOL + + +def _setting_code_from_int(code): + """ + Given an integer setting code, returns either one of :class:`SettingCodes + ` or, if not present in the known set of codes, + returns the integer directly. + """ + try: + return SettingCodes(code) + except ValueError: + return code + + +class ChangedSetting: + + def __init__(self, setting, original_value, new_value): + #: The setting code given. Either one of :class:`SettingCodes + #: ` or ``int`` + #: + #: .. versionchanged:: 2.6.0 + self.setting = setting + + #: The original value before being changed. + self.original_value = original_value + + #: The new value after being changed. + self.new_value = new_value + + def __repr__(self): + return ( + "ChangedSetting(setting=%s, original_value=%s, " + "new_value=%s)" + ) % ( + self.setting, + self.original_value, + self.new_value + ) + + +class Settings(MutableMapping): + """ + An object that encapsulates HTTP/2 settings state. + + HTTP/2 Settings are a complex beast. Each party, remote and local, has its + own settings and a view of the other party's settings. When a settings + frame is emitted by a peer it cannot assume that the new settings values + are in place until the remote peer acknowledges the setting. In principle, + multiple settings changes can be "in flight" at the same time, all with + different values. + + This object encapsulates this mess. It provides a dict-like interface to + settings, which return the *current* values of the settings in question. + Additionally, it keeps track of the stack of proposed values: each time an + acknowledgement is sent/received, it updates the current values with the + stack of proposed values. On top of all that, it validates the values to + make sure they're allowed, and raises :class:`InvalidSettingsValueError + ` if they are not. + + Finally, this object understands what the default values of the HTTP/2 + settings are, and sets those defaults appropriately. + + .. versionchanged:: 2.2.0 + Added the ``initial_values`` parameter. + + .. versionchanged:: 2.5.0 + Added the ``max_header_list_size`` property. + + :param client: (optional) Whether these settings should be defaulted for a + client implementation or a server implementation. Defaults to ``True``. + :type client: ``bool`` + :param initial_values: (optional) Any initial values the user would like + set, rather than RFC 7540's defaults. + :type initial_vales: ``MutableMapping`` + """ + def __init__(self, client=True, initial_values=None): + # Backing object for the settings. This is a dictionary of + # (setting: [list of values]), where the first value in the list is the + # current value of the setting. Strictly this doesn't use lists but + # instead uses collections.deque to avoid repeated memory allocations. + # + # This contains the default values for HTTP/2. + self._settings = { + SettingCodes.HEADER_TABLE_SIZE: collections.deque([4096]), + SettingCodes.ENABLE_PUSH: collections.deque([int(client)]), + SettingCodes.INITIAL_WINDOW_SIZE: collections.deque([65535]), + SettingCodes.MAX_FRAME_SIZE: collections.deque([16384]), + SettingCodes.ENABLE_CONNECT_PROTOCOL: collections.deque([0]), + } + if initial_values is not None: + for key, value in initial_values.items(): + invalid = _validate_setting(key, value) + if invalid: + raise InvalidSettingsValueError( + "Setting %d has invalid value %d" % (key, value), + error_code=invalid + ) + self._settings[key] = collections.deque([value]) + + def acknowledge(self): + """ + The settings have been acknowledged, either by the user (remote + settings) or by the remote peer (local settings). + + :returns: A dict of {setting: ChangedSetting} that were applied. + """ + changed_settings = {} + + # If there is more than one setting in the list, we have a setting + # value outstanding. Update them. + for k, v in self._settings.items(): + if len(v) > 1: + old_setting = v.popleft() + new_setting = v[0] + changed_settings[k] = ChangedSetting( + k, old_setting, new_setting + ) + + return changed_settings + + # Provide easy-access to well known settings. + @property + def header_table_size(self): + """ + The current value of the :data:`HEADER_TABLE_SIZE + ` setting. + """ + return self[SettingCodes.HEADER_TABLE_SIZE] + + @header_table_size.setter + def header_table_size(self, value): + self[SettingCodes.HEADER_TABLE_SIZE] = value + + @property + def enable_push(self): + """ + The current value of the :data:`ENABLE_PUSH + ` setting. + """ + return self[SettingCodes.ENABLE_PUSH] + + @enable_push.setter + def enable_push(self, value): + self[SettingCodes.ENABLE_PUSH] = value + + @property + def initial_window_size(self): + """ + The current value of the :data:`INITIAL_WINDOW_SIZE + ` setting. + """ + return self[SettingCodes.INITIAL_WINDOW_SIZE] + + @initial_window_size.setter + def initial_window_size(self, value): + self[SettingCodes.INITIAL_WINDOW_SIZE] = value + + @property + def max_frame_size(self): + """ + The current value of the :data:`MAX_FRAME_SIZE + ` setting. + """ + return self[SettingCodes.MAX_FRAME_SIZE] + + @max_frame_size.setter + def max_frame_size(self, value): + self[SettingCodes.MAX_FRAME_SIZE] = value + + @property + def max_concurrent_streams(self): + """ + The current value of the :data:`MAX_CONCURRENT_STREAMS + ` setting. + """ + return self.get(SettingCodes.MAX_CONCURRENT_STREAMS, 2**32+1) + + @max_concurrent_streams.setter + def max_concurrent_streams(self, value): + self[SettingCodes.MAX_CONCURRENT_STREAMS] = value + + @property + def max_header_list_size(self): + """ + The current value of the :data:`MAX_HEADER_LIST_SIZE + ` setting. If not set, + returns ``None``, which means unlimited. + + .. versionadded:: 2.5.0 + """ + return self.get(SettingCodes.MAX_HEADER_LIST_SIZE, None) + + @max_header_list_size.setter + def max_header_list_size(self, value): + self[SettingCodes.MAX_HEADER_LIST_SIZE] = value + + @property + def enable_connect_protocol(self): + """ + The current value of the :data:`ENABLE_CONNECT_PROTOCOL + ` setting. + """ + return self[SettingCodes.ENABLE_CONNECT_PROTOCOL] + + @enable_connect_protocol.setter + def enable_connect_protocol(self, value): + self[SettingCodes.ENABLE_CONNECT_PROTOCOL] = value + + # Implement the MutableMapping API. + def __getitem__(self, key): + val = self._settings[key][0] + + # Things that were created when a setting was received should stay + # KeyError'd. + if val is None: + raise KeyError + + return val + + def __setitem__(self, key, value): + invalid = _validate_setting(key, value) + if invalid: + raise InvalidSettingsValueError( + "Setting %d has invalid value %d" % (key, value), + error_code=invalid + ) + + try: + items = self._settings[key] + except KeyError: + items = collections.deque([None]) + self._settings[key] = items + + items.append(value) + + def __delitem__(self, key): + del self._settings[key] + + def __iter__(self): + return self._settings.__iter__() + + def __len__(self): + return len(self._settings) + + def __eq__(self, other): + if isinstance(other, Settings): + return self._settings == other._settings + else: + return NotImplemented + + def __ne__(self, other): + if isinstance(other, Settings): + return not self == other + else: + return NotImplemented + + +def _validate_setting(setting, value): # noqa: C901 + """ + Confirms that a specific setting has a well-formed value. If the setting is + invalid, returns an error code. Otherwise, returns 0 (NO_ERROR). + """ + if setting == SettingCodes.ENABLE_PUSH: + if value not in (0, 1): + return ErrorCodes.PROTOCOL_ERROR + elif setting == SettingCodes.INITIAL_WINDOW_SIZE: + if not 0 <= value <= 2147483647: # 2^31 - 1 + return ErrorCodes.FLOW_CONTROL_ERROR + elif setting == SettingCodes.MAX_FRAME_SIZE: + if not 16384 <= value <= 16777215: # 2^14 and 2^24 - 1 + return ErrorCodes.PROTOCOL_ERROR + elif setting == SettingCodes.MAX_HEADER_LIST_SIZE: + if value < 0: + return ErrorCodes.PROTOCOL_ERROR + elif setting == SettingCodes.ENABLE_CONNECT_PROTOCOL: + if value not in (0, 1): + return ErrorCodes.PROTOCOL_ERROR + + return 0 diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/h2/src/h2/stream.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/h2/src/h2/stream.py new file mode 100644 index 0000000000..817636f8ed --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/h2/src/h2/stream.py @@ -0,0 +1,1371 @@ +# -*- coding: utf-8 -*- +""" +h2/stream +~~~~~~~~~ + +An implementation of a HTTP/2 stream. +""" +from enum import Enum, IntEnum +from hpack import HeaderTuple +from hyperframe.frame import ( + HeadersFrame, ContinuationFrame, DataFrame, WindowUpdateFrame, + RstStreamFrame, PushPromiseFrame, AltSvcFrame +) + +from .errors import ErrorCodes, _error_code_from_int +from .events import ( + RequestReceived, ResponseReceived, DataReceived, WindowUpdated, + StreamEnded, PushedStreamReceived, StreamReset, TrailersReceived, + InformationalResponseReceived, AlternativeServiceAvailable, + _ResponseSent, _RequestSent, _TrailersSent, _PushedRequestSent +) +from .exceptions import ( + ProtocolError, StreamClosedError, InvalidBodyLengthError, FlowControlError +) +from .utilities import ( + guard_increment_window, is_informational_response, authority_from_headers, + validate_headers, validate_outbound_headers, normalize_outbound_headers, + HeaderValidationFlags, extract_method_header, normalize_inbound_headers +) +from .windows import WindowManager + + +class StreamState(IntEnum): + IDLE = 0 + RESERVED_REMOTE = 1 + RESERVED_LOCAL = 2 + OPEN = 3 + HALF_CLOSED_REMOTE = 4 + HALF_CLOSED_LOCAL = 5 + CLOSED = 6 + + +class StreamInputs(Enum): + SEND_HEADERS = 0 + SEND_PUSH_PROMISE = 1 + SEND_RST_STREAM = 2 + SEND_DATA = 3 + SEND_WINDOW_UPDATE = 4 + SEND_END_STREAM = 5 + RECV_HEADERS = 6 + RECV_PUSH_PROMISE = 7 + RECV_RST_STREAM = 8 + RECV_DATA = 9 + RECV_WINDOW_UPDATE = 10 + RECV_END_STREAM = 11 + RECV_CONTINUATION = 12 # Added in 2.0.0 + SEND_INFORMATIONAL_HEADERS = 13 # Added in 2.2.0 + RECV_INFORMATIONAL_HEADERS = 14 # Added in 2.2.0 + SEND_ALTERNATIVE_SERVICE = 15 # Added in 2.3.0 + RECV_ALTERNATIVE_SERVICE = 16 # Added in 2.3.0 + UPGRADE_CLIENT = 17 # Added 2.3.0 + UPGRADE_SERVER = 18 # Added 2.3.0 + + +class StreamClosedBy(Enum): + SEND_END_STREAM = 0 + RECV_END_STREAM = 1 + SEND_RST_STREAM = 2 + RECV_RST_STREAM = 3 + + +# This array is initialized once, and is indexed by the stream states above. +# It indicates whether a stream in the given state is open. The reason we do +# this is that we potentially check whether a stream in a given state is open +# quite frequently: given that we check so often, we should do so in the +# fastest and most performant way possible. +STREAM_OPEN = [False for _ in range(0, len(StreamState))] +STREAM_OPEN[StreamState.OPEN] = True +STREAM_OPEN[StreamState.HALF_CLOSED_LOCAL] = True +STREAM_OPEN[StreamState.HALF_CLOSED_REMOTE] = True + + +class H2StreamStateMachine: + """ + A single HTTP/2 stream state machine. + + This stream object implements basically the state machine described in + RFC 7540 section 5.1. + + :param stream_id: The stream ID of this stream. This is stored primarily + for logging purposes. + """ + def __init__(self, stream_id): + self.state = StreamState.IDLE + self.stream_id = stream_id + + #: Whether this peer is the client side of this stream. + self.client = None + + # Whether trailers have been sent/received on this stream or not. + self.headers_sent = None + self.trailers_sent = None + self.headers_received = None + self.trailers_received = None + + # How the stream was closed. One of StreamClosedBy. + self.stream_closed_by = None + + def process_input(self, input_): + """ + Process a specific input in the state machine. + """ + if not isinstance(input_, StreamInputs): + raise ValueError("Input must be an instance of StreamInputs") + + try: + func, target_state = _transitions[(self.state, input_)] + except KeyError: + old_state = self.state + self.state = StreamState.CLOSED + raise ProtocolError( + "Invalid input %s in state %s" % (input_, old_state) + ) + else: + previous_state = self.state + self.state = target_state + if func is not None: + try: + return func(self, previous_state) + except ProtocolError: + self.state = StreamState.CLOSED + raise + except AssertionError as e: # pragma: no cover + self.state = StreamState.CLOSED + raise ProtocolError(e) + + return [] + + def request_sent(self, previous_state): + """ + Fires when a request is sent. + """ + self.client = True + self.headers_sent = True + event = _RequestSent() + + return [event] + + def response_sent(self, previous_state): + """ + Fires when something that should be a response is sent. This 'response' + may actually be trailers. + """ + if not self.headers_sent: + if self.client is True or self.client is None: + raise ProtocolError("Client cannot send responses.") + self.headers_sent = True + event = _ResponseSent() + else: + assert not self.trailers_sent + self.trailers_sent = True + event = _TrailersSent() + + return [event] + + def request_received(self, previous_state): + """ + Fires when a request is received. + """ + assert not self.headers_received + assert not self.trailers_received + + self.client = False + self.headers_received = True + event = RequestReceived() + + event.stream_id = self.stream_id + return [event] + + def response_received(self, previous_state): + """ + Fires when a response is received. Also disambiguates between responses + and trailers. + """ + if not self.headers_received: + assert self.client is True + self.headers_received = True + event = ResponseReceived() + else: + assert not self.trailers_received + self.trailers_received = True + event = TrailersReceived() + + event.stream_id = self.stream_id + return [event] + + def data_received(self, previous_state): + """ + Fires when data is received. + """ + if not self.headers_received: + raise ProtocolError("cannot receive data before headers") + event = DataReceived() + event.stream_id = self.stream_id + return [event] + + def window_updated(self, previous_state): + """ + Fires when a window update frame is received. + """ + event = WindowUpdated() + event.stream_id = self.stream_id + return [event] + + def stream_half_closed(self, previous_state): + """ + Fires when an END_STREAM flag is received in the OPEN state, + transitioning this stream to a HALF_CLOSED_REMOTE state. + """ + event = StreamEnded() + event.stream_id = self.stream_id + return [event] + + def stream_ended(self, previous_state): + """ + Fires when a stream is cleanly ended. + """ + self.stream_closed_by = StreamClosedBy.RECV_END_STREAM + event = StreamEnded() + event.stream_id = self.stream_id + return [event] + + def stream_reset(self, previous_state): + """ + Fired when a stream is forcefully reset. + """ + self.stream_closed_by = StreamClosedBy.RECV_RST_STREAM + event = StreamReset() + event.stream_id = self.stream_id + return [event] + + def send_new_pushed_stream(self, previous_state): + """ + Fires on the newly pushed stream, when pushed by the local peer. + + No event here, but definitionally this peer must be a server. + """ + assert self.client is None + self.client = False + self.headers_received = True + return [] + + def recv_new_pushed_stream(self, previous_state): + """ + Fires on the newly pushed stream, when pushed by the remote peer. + + No event here, but definitionally this peer must be a client. + """ + assert self.client is None + self.client = True + self.headers_sent = True + return [] + + def send_push_promise(self, previous_state): + """ + Fires on the already-existing stream when a PUSH_PROMISE frame is sent. + We may only send PUSH_PROMISE frames if we're a server. + """ + if self.client is True: + raise ProtocolError("Cannot push streams from client peers.") + + event = _PushedRequestSent() + return [event] + + def recv_push_promise(self, previous_state): + """ + Fires on the already-existing stream when a PUSH_PROMISE frame is + received. We may only receive PUSH_PROMISE frames if we're a client. + + Fires a PushedStreamReceived event. + """ + if not self.client: + if self.client is None: # pragma: no cover + msg = "Idle streams cannot receive pushes" + else: # pragma: no cover + msg = "Cannot receive pushed streams as a server" + raise ProtocolError(msg) + + event = PushedStreamReceived() + event.parent_stream_id = self.stream_id + return [event] + + def send_end_stream(self, previous_state): + """ + Called when an attempt is made to send END_STREAM in the + HALF_CLOSED_REMOTE state. + """ + self.stream_closed_by = StreamClosedBy.SEND_END_STREAM + + def send_reset_stream(self, previous_state): + """ + Called when an attempt is made to send RST_STREAM in a non-closed + stream state. + """ + self.stream_closed_by = StreamClosedBy.SEND_RST_STREAM + + def reset_stream_on_error(self, previous_state): + """ + Called when we need to forcefully emit another RST_STREAM frame on + behalf of the state machine. + + If this is the first time we've done this, we should also hang an event + off the StreamClosedError so that the user can be informed. We know + it's the first time we've done this if the stream is currently in a + state other than CLOSED. + """ + self.stream_closed_by = StreamClosedBy.SEND_RST_STREAM + + error = StreamClosedError(self.stream_id) + + event = StreamReset() + event.stream_id = self.stream_id + event.error_code = ErrorCodes.STREAM_CLOSED + event.remote_reset = False + error._events = [event] + raise error + + def recv_on_closed_stream(self, previous_state): + """ + Called when an unexpected frame is received on an already-closed + stream. + + An endpoint that receives an unexpected frame should treat it as + a stream error or connection error with type STREAM_CLOSED, depending + on the specific frame. The error handling is done at a higher level: + this just raises the appropriate error. + """ + raise StreamClosedError(self.stream_id) + + def send_on_closed_stream(self, previous_state): + """ + Called when an attempt is made to send data on an already-closed + stream. + + This essentially overrides the standard logic by throwing a + more-specific error: StreamClosedError. This is a ProtocolError, so it + matches the standard API of the state machine, but provides more detail + to the user. + """ + raise StreamClosedError(self.stream_id) + + def recv_push_on_closed_stream(self, previous_state): + """ + Called when a PUSH_PROMISE frame is received on a full stop + stream. + + If the stream was closed by us sending a RST_STREAM frame, then we + presume that the PUSH_PROMISE was in flight when we reset the parent + stream. Rathen than accept the new stream, we just reset it. + Otherwise, we should call this a PROTOCOL_ERROR: pushing a stream on a + naturally closed stream is a real problem because it creates a brand + new stream that the remote peer now believes exists. + """ + assert self.stream_closed_by is not None + + if self.stream_closed_by == StreamClosedBy.SEND_RST_STREAM: + raise StreamClosedError(self.stream_id) + else: + raise ProtocolError("Attempted to push on closed stream.") + + def send_push_on_closed_stream(self, previous_state): + """ + Called when an attempt is made to push on an already-closed stream. + + This essentially overrides the standard logic by providing a more + useful error message. It's necessary because simply indicating that the + stream is closed is not enough: there is now a new stream that is not + allowed to be there. The only recourse is to tear the whole connection + down. + """ + raise ProtocolError("Attempted to push on closed stream.") + + def send_informational_response(self, previous_state): + """ + Called when an informational header block is sent (that is, a block + where the :status header has a 1XX value). + + Only enforces that these are sent *before* final headers are sent. + """ + if self.headers_sent: + raise ProtocolError("Information response after final response") + + event = _ResponseSent() + return [event] + + def recv_informational_response(self, previous_state): + """ + Called when an informational header block is received (that is, a block + where the :status header has a 1XX value). + """ + if self.headers_received: + raise ProtocolError("Informational response after final response") + + event = InformationalResponseReceived() + event.stream_id = self.stream_id + return [event] + + def recv_alt_svc(self, previous_state): + """ + Called when receiving an ALTSVC frame. + + RFC 7838 allows us to receive ALTSVC frames at any stream state, which + is really absurdly overzealous. For that reason, we want to limit the + states in which we can actually receive it. It's really only sensible + to receive it after we've sent our own headers and before the server + has sent its header block: the server can't guarantee that we have any + state around after it completes its header block, and the server + doesn't know what origin we're talking about before we've sent ours. + + For that reason, this function applies a few extra checks on both state + and some of the little state variables we keep around. If those suggest + an unreasonable situation for the ALTSVC frame to have been sent in, + we quietly ignore it (as RFC 7838 suggests). + + This function is also *not* always called by the state machine. In some + states (IDLE, RESERVED_LOCAL, CLOSED) we don't bother to call it, + because we know the frame cannot be valid in that state (IDLE because + the server cannot know what origin the stream applies to, CLOSED + because the server cannot assume we still have state around, + RESERVED_LOCAL because by definition if we're in the RESERVED_LOCAL + state then *we* are the server). + """ + # Servers can't receive ALTSVC frames, but RFC 7838 tells us to ignore + # them. + if self.client is False: + return [] + + # If we've received the response headers from the server they can't + # guarantee we still have any state around. Other implementations + # (like nghttp2) ignore ALTSVC in this state, so we will too. + if self.headers_received: + return [] + + # Otherwise, this is a sensible enough frame to have received. Return + # the event and let it get populated. + return [AlternativeServiceAvailable()] + + def send_alt_svc(self, previous_state): + """ + Called when sending an ALTSVC frame on this stream. + + For consistency with the restrictions we apply on receiving ALTSVC + frames in ``recv_alt_svc``, we want to restrict when users can send + ALTSVC frames to the situations when we ourselves would accept them. + + That means: when we are a server, when we have received the request + headers, and when we have not yet sent our own response headers. + """ + # We should not send ALTSVC after we've sent response headers, as the + # client may have disposed of its state. + if self.headers_sent: + raise ProtocolError( + "Cannot send ALTSVC after sending response headers." + ) + + return + + +# STATE MACHINE +# +# The stream state machine is defined here to avoid the need to allocate it +# repeatedly for each stream. It cannot be defined in the stream class because +# it needs to be able to reference the callbacks defined on the class, but +# because Python's scoping rules are weird the class object is not actually in +# scope during the body of the class object. +# +# For the sake of clarity, we reproduce the RFC 7540 state machine here: +# +# +--------+ +# send PP | | recv PP +# ,--------| idle |--------. +# / | | \ +# v +--------+ v +# +----------+ | +----------+ +# | | | send H / | | +# ,------| reserved | | recv H | reserved |------. +# | | (local) | | | (remote) | | +# | +----------+ v +----------+ | +# | | +--------+ | | +# | | recv ES | | send ES | | +# | send H | ,-------| open |-------. | recv H | +# | | / | | \ | | +# | v v +--------+ v v | +# | +----------+ | +----------+ | +# | | half | | | half | | +# | | closed | | send R / | closed | | +# | | (remote) | | recv R | (local) | | +# | +----------+ | +----------+ | +# | | | | | +# | | send ES / | recv ES / | | +# | | send R / v send R / | | +# | | recv R +--------+ recv R | | +# | send R / `----------->| |<-----------' send R / | +# | recv R | closed | recv R | +# `----------------------->| |<----------------------' +# +--------+ +# +# send: endpoint sends this frame +# recv: endpoint receives this frame +# +# H: HEADERS frame (with implied CONTINUATIONs) +# PP: PUSH_PROMISE frame (with implied CONTINUATIONs) +# ES: END_STREAM flag +# R: RST_STREAM frame +# +# For the purposes of this state machine we treat HEADERS and their +# associated CONTINUATION frames as a single jumbo frame. The protocol +# allows/requires this by preventing other frames from being interleved in +# between HEADERS/CONTINUATION frames. However, if a CONTINUATION frame is +# received without a prior HEADERS frame, it *will* be passed to this state +# machine. The state machine should always reject that frame, either as an +# invalid transition or because the stream is closed. +# +# There is a confusing relationship around PUSH_PROMISE frames. The state +# machine above considers them to be frames belonging to the new stream, +# which is *somewhat* true. However, they are sent with the stream ID of +# their related stream, and are only sendable in some cases. +# For this reason, our state machine implementation below allows for +# PUSH_PROMISE frames both in the IDLE state (as in the diagram), but also +# in the OPEN, HALF_CLOSED_LOCAL, and HALF_CLOSED_REMOTE states. +# Essentially, for h2, PUSH_PROMISE frames are effectively sent on +# two streams. +# +# The _transitions dictionary contains a mapping of tuples of +# (state, input) to tuples of (side_effect_function, end_state). This +# map contains all allowed transitions: anything not in this map is +# invalid and immediately causes a transition to ``closed``. +_transitions = { + # State: idle + (StreamState.IDLE, StreamInputs.SEND_HEADERS): + (H2StreamStateMachine.request_sent, StreamState.OPEN), + (StreamState.IDLE, StreamInputs.RECV_HEADERS): + (H2StreamStateMachine.request_received, StreamState.OPEN), + (StreamState.IDLE, StreamInputs.RECV_DATA): + (H2StreamStateMachine.reset_stream_on_error, StreamState.CLOSED), + (StreamState.IDLE, StreamInputs.SEND_PUSH_PROMISE): + (H2StreamStateMachine.send_new_pushed_stream, + StreamState.RESERVED_LOCAL), + (StreamState.IDLE, StreamInputs.RECV_PUSH_PROMISE): + (H2StreamStateMachine.recv_new_pushed_stream, + StreamState.RESERVED_REMOTE), + (StreamState.IDLE, StreamInputs.RECV_ALTERNATIVE_SERVICE): + (None, StreamState.IDLE), + (StreamState.IDLE, StreamInputs.UPGRADE_CLIENT): + (H2StreamStateMachine.request_sent, StreamState.HALF_CLOSED_LOCAL), + (StreamState.IDLE, StreamInputs.UPGRADE_SERVER): + (H2StreamStateMachine.request_received, + StreamState.HALF_CLOSED_REMOTE), + + # State: reserved local + (StreamState.RESERVED_LOCAL, StreamInputs.SEND_HEADERS): + (H2StreamStateMachine.response_sent, StreamState.HALF_CLOSED_REMOTE), + (StreamState.RESERVED_LOCAL, StreamInputs.RECV_DATA): + (H2StreamStateMachine.reset_stream_on_error, StreamState.CLOSED), + (StreamState.RESERVED_LOCAL, StreamInputs.SEND_WINDOW_UPDATE): + (None, StreamState.RESERVED_LOCAL), + (StreamState.RESERVED_LOCAL, StreamInputs.RECV_WINDOW_UPDATE): + (H2StreamStateMachine.window_updated, StreamState.RESERVED_LOCAL), + (StreamState.RESERVED_LOCAL, StreamInputs.SEND_RST_STREAM): + (H2StreamStateMachine.send_reset_stream, StreamState.CLOSED), + (StreamState.RESERVED_LOCAL, StreamInputs.RECV_RST_STREAM): + (H2StreamStateMachine.stream_reset, StreamState.CLOSED), + (StreamState.RESERVED_LOCAL, StreamInputs.SEND_ALTERNATIVE_SERVICE): + (H2StreamStateMachine.send_alt_svc, StreamState.RESERVED_LOCAL), + (StreamState.RESERVED_LOCAL, StreamInputs.RECV_ALTERNATIVE_SERVICE): + (None, StreamState.RESERVED_LOCAL), + + # State: reserved remote + (StreamState.RESERVED_REMOTE, StreamInputs.RECV_HEADERS): + (H2StreamStateMachine.response_received, + StreamState.HALF_CLOSED_LOCAL), + (StreamState.RESERVED_REMOTE, StreamInputs.RECV_DATA): + (H2StreamStateMachine.reset_stream_on_error, StreamState.CLOSED), + (StreamState.RESERVED_REMOTE, StreamInputs.SEND_WINDOW_UPDATE): + (None, StreamState.RESERVED_REMOTE), + (StreamState.RESERVED_REMOTE, StreamInputs.RECV_WINDOW_UPDATE): + (H2StreamStateMachine.window_updated, StreamState.RESERVED_REMOTE), + (StreamState.RESERVED_REMOTE, StreamInputs.SEND_RST_STREAM): + (H2StreamStateMachine.send_reset_stream, StreamState.CLOSED), + (StreamState.RESERVED_REMOTE, StreamInputs.RECV_RST_STREAM): + (H2StreamStateMachine.stream_reset, StreamState.CLOSED), + (StreamState.RESERVED_REMOTE, StreamInputs.RECV_ALTERNATIVE_SERVICE): + (H2StreamStateMachine.recv_alt_svc, StreamState.RESERVED_REMOTE), + + # State: open + (StreamState.OPEN, StreamInputs.SEND_HEADERS): + (H2StreamStateMachine.response_sent, StreamState.OPEN), + (StreamState.OPEN, StreamInputs.RECV_HEADERS): + (H2StreamStateMachine.response_received, StreamState.OPEN), + (StreamState.OPEN, StreamInputs.SEND_DATA): + (None, StreamState.OPEN), + (StreamState.OPEN, StreamInputs.RECV_DATA): + (H2StreamStateMachine.data_received, StreamState.OPEN), + (StreamState.OPEN, StreamInputs.SEND_END_STREAM): + (None, StreamState.HALF_CLOSED_LOCAL), + (StreamState.OPEN, StreamInputs.RECV_END_STREAM): + (H2StreamStateMachine.stream_half_closed, + StreamState.HALF_CLOSED_REMOTE), + (StreamState.OPEN, StreamInputs.SEND_WINDOW_UPDATE): + (None, StreamState.OPEN), + (StreamState.OPEN, StreamInputs.RECV_WINDOW_UPDATE): + (H2StreamStateMachine.window_updated, StreamState.OPEN), + (StreamState.OPEN, StreamInputs.SEND_RST_STREAM): + (H2StreamStateMachine.send_reset_stream, StreamState.CLOSED), + (StreamState.OPEN, StreamInputs.RECV_RST_STREAM): + (H2StreamStateMachine.stream_reset, StreamState.CLOSED), + (StreamState.OPEN, StreamInputs.SEND_PUSH_PROMISE): + (H2StreamStateMachine.send_push_promise, StreamState.OPEN), + (StreamState.OPEN, StreamInputs.RECV_PUSH_PROMISE): + (H2StreamStateMachine.recv_push_promise, StreamState.OPEN), + (StreamState.OPEN, StreamInputs.SEND_INFORMATIONAL_HEADERS): + (H2StreamStateMachine.send_informational_response, StreamState.OPEN), + (StreamState.OPEN, StreamInputs.RECV_INFORMATIONAL_HEADERS): + (H2StreamStateMachine.recv_informational_response, StreamState.OPEN), + (StreamState.OPEN, StreamInputs.SEND_ALTERNATIVE_SERVICE): + (H2StreamStateMachine.send_alt_svc, StreamState.OPEN), + (StreamState.OPEN, StreamInputs.RECV_ALTERNATIVE_SERVICE): + (H2StreamStateMachine.recv_alt_svc, StreamState.OPEN), + + # State: half-closed remote + (StreamState.HALF_CLOSED_REMOTE, StreamInputs.SEND_HEADERS): + (H2StreamStateMachine.response_sent, StreamState.HALF_CLOSED_REMOTE), + (StreamState.HALF_CLOSED_REMOTE, StreamInputs.RECV_HEADERS): + (H2StreamStateMachine.reset_stream_on_error, StreamState.CLOSED), + (StreamState.HALF_CLOSED_REMOTE, StreamInputs.SEND_DATA): + (None, StreamState.HALF_CLOSED_REMOTE), + (StreamState.HALF_CLOSED_REMOTE, StreamInputs.RECV_DATA): + (H2StreamStateMachine.reset_stream_on_error, StreamState.CLOSED), + (StreamState.HALF_CLOSED_REMOTE, StreamInputs.SEND_END_STREAM): + (H2StreamStateMachine.send_end_stream, StreamState.CLOSED), + (StreamState.HALF_CLOSED_REMOTE, StreamInputs.SEND_WINDOW_UPDATE): + (None, StreamState.HALF_CLOSED_REMOTE), + (StreamState.HALF_CLOSED_REMOTE, StreamInputs.RECV_WINDOW_UPDATE): + (H2StreamStateMachine.window_updated, StreamState.HALF_CLOSED_REMOTE), + (StreamState.HALF_CLOSED_REMOTE, StreamInputs.SEND_RST_STREAM): + (H2StreamStateMachine.send_reset_stream, StreamState.CLOSED), + (StreamState.HALF_CLOSED_REMOTE, StreamInputs.RECV_RST_STREAM): + (H2StreamStateMachine.stream_reset, StreamState.CLOSED), + (StreamState.HALF_CLOSED_REMOTE, StreamInputs.SEND_PUSH_PROMISE): + (H2StreamStateMachine.send_push_promise, + StreamState.HALF_CLOSED_REMOTE), + (StreamState.HALF_CLOSED_REMOTE, StreamInputs.RECV_PUSH_PROMISE): + (H2StreamStateMachine.reset_stream_on_error, StreamState.CLOSED), + (StreamState.HALF_CLOSED_REMOTE, StreamInputs.SEND_INFORMATIONAL_HEADERS): + (H2StreamStateMachine.send_informational_response, + StreamState.HALF_CLOSED_REMOTE), + (StreamState.HALF_CLOSED_REMOTE, StreamInputs.SEND_ALTERNATIVE_SERVICE): + (H2StreamStateMachine.send_alt_svc, StreamState.HALF_CLOSED_REMOTE), + (StreamState.HALF_CLOSED_REMOTE, StreamInputs.RECV_ALTERNATIVE_SERVICE): + (H2StreamStateMachine.recv_alt_svc, StreamState.HALF_CLOSED_REMOTE), + + # State: half-closed local + (StreamState.HALF_CLOSED_LOCAL, StreamInputs.RECV_HEADERS): + (H2StreamStateMachine.response_received, + StreamState.HALF_CLOSED_LOCAL), + (StreamState.HALF_CLOSED_LOCAL, StreamInputs.RECV_DATA): + (H2StreamStateMachine.data_received, StreamState.HALF_CLOSED_LOCAL), + (StreamState.HALF_CLOSED_LOCAL, StreamInputs.RECV_END_STREAM): + (H2StreamStateMachine.stream_ended, StreamState.CLOSED), + (StreamState.HALF_CLOSED_LOCAL, StreamInputs.SEND_WINDOW_UPDATE): + (None, StreamState.HALF_CLOSED_LOCAL), + (StreamState.HALF_CLOSED_LOCAL, StreamInputs.RECV_WINDOW_UPDATE): + (H2StreamStateMachine.window_updated, StreamState.HALF_CLOSED_LOCAL), + (StreamState.HALF_CLOSED_LOCAL, StreamInputs.SEND_RST_STREAM): + (H2StreamStateMachine.send_reset_stream, StreamState.CLOSED), + (StreamState.HALF_CLOSED_LOCAL, StreamInputs.RECV_RST_STREAM): + (H2StreamStateMachine.stream_reset, StreamState.CLOSED), + (StreamState.HALF_CLOSED_LOCAL, StreamInputs.RECV_PUSH_PROMISE): + (H2StreamStateMachine.recv_push_promise, + StreamState.HALF_CLOSED_LOCAL), + (StreamState.HALF_CLOSED_LOCAL, StreamInputs.RECV_INFORMATIONAL_HEADERS): + (H2StreamStateMachine.recv_informational_response, + StreamState.HALF_CLOSED_LOCAL), + (StreamState.HALF_CLOSED_LOCAL, StreamInputs.SEND_ALTERNATIVE_SERVICE): + (H2StreamStateMachine.send_alt_svc, StreamState.HALF_CLOSED_LOCAL), + (StreamState.HALF_CLOSED_LOCAL, StreamInputs.RECV_ALTERNATIVE_SERVICE): + (H2StreamStateMachine.recv_alt_svc, StreamState.HALF_CLOSED_LOCAL), + + # State: closed + (StreamState.CLOSED, StreamInputs.RECV_END_STREAM): + (None, StreamState.CLOSED), + (StreamState.CLOSED, StreamInputs.RECV_ALTERNATIVE_SERVICE): + (None, StreamState.CLOSED), + + # RFC 7540 Section 5.1 defines how the end point should react when + # receiving a frame on a closed stream with the following statements: + # + # > An endpoint that receives any frame other than PRIORITY after receiving + # > a RST_STREAM MUST treat that as a stream error of type STREAM_CLOSED. + # > An endpoint that receives any frames after receiving a frame with the + # > END_STREAM flag set MUST treat that as a connection error of type + # > STREAM_CLOSED. + (StreamState.CLOSED, StreamInputs.RECV_HEADERS): + (H2StreamStateMachine.recv_on_closed_stream, StreamState.CLOSED), + (StreamState.CLOSED, StreamInputs.RECV_DATA): + (H2StreamStateMachine.recv_on_closed_stream, StreamState.CLOSED), + + # > WINDOW_UPDATE or RST_STREAM frames can be received in this state + # > for a short period after a DATA or HEADERS frame containing a + # > END_STREAM flag is sent, as instructed in RFC 7540 Section 5.1. But we + # > don't have access to a clock so we just always allow it. + (StreamState.CLOSED, StreamInputs.RECV_WINDOW_UPDATE): + (None, StreamState.CLOSED), + (StreamState.CLOSED, StreamInputs.RECV_RST_STREAM): + (None, StreamState.CLOSED), + + # > A receiver MUST treat the receipt of a PUSH_PROMISE on a stream that is + # > neither "open" nor "half-closed (local)" as a connection error of type + # > PROTOCOL_ERROR. + (StreamState.CLOSED, StreamInputs.RECV_PUSH_PROMISE): + (H2StreamStateMachine.recv_push_on_closed_stream, StreamState.CLOSED), + + # Also, users should be forbidden from sending on closed streams. + (StreamState.CLOSED, StreamInputs.SEND_HEADERS): + (H2StreamStateMachine.send_on_closed_stream, StreamState.CLOSED), + (StreamState.CLOSED, StreamInputs.SEND_PUSH_PROMISE): + (H2StreamStateMachine.send_push_on_closed_stream, StreamState.CLOSED), + (StreamState.CLOSED, StreamInputs.SEND_RST_STREAM): + (H2StreamStateMachine.send_on_closed_stream, StreamState.CLOSED), + (StreamState.CLOSED, StreamInputs.SEND_DATA): + (H2StreamStateMachine.send_on_closed_stream, StreamState.CLOSED), + (StreamState.CLOSED, StreamInputs.SEND_WINDOW_UPDATE): + (H2StreamStateMachine.send_on_closed_stream, StreamState.CLOSED), + (StreamState.CLOSED, StreamInputs.SEND_END_STREAM): + (H2StreamStateMachine.send_on_closed_stream, StreamState.CLOSED), +} + + +class H2Stream: + """ + A low-level HTTP/2 stream object. This handles building and receiving + frames and maintains per-stream state. + + This wraps a HTTP/2 Stream state machine implementation, ensuring that + frames can only be sent/received when the stream is in a valid state. + Attempts to create frames that cannot be sent will raise a + ``ProtocolError``. + """ + def __init__(self, + stream_id, + config, + inbound_window_size, + outbound_window_size): + self.state_machine = H2StreamStateMachine(stream_id) + self.stream_id = stream_id + self.max_outbound_frame_size = None + self.request_method = None + + # The current value of the outbound stream flow control window + self.outbound_flow_control_window = outbound_window_size + + # The flow control manager. + self._inbound_window_manager = WindowManager(inbound_window_size) + + # The expected content length, if any. + self._expected_content_length = None + + # The actual received content length. Always tracked. + self._actual_content_length = 0 + + # The authority we believe this stream belongs to. + self._authority = None + + # The configuration for this stream. + self.config = config + + def __repr__(self): + return "<%s id:%d state:%r>" % ( + type(self).__name__, + self.stream_id, + self.state_machine.state + ) + + @property + def inbound_flow_control_window(self): + """ + The size of the inbound flow control window for the stream. This is + rarely publicly useful: instead, use :meth:`remote_flow_control_window + `. This shortcut is + largely present to provide a shortcut to this data. + """ + return self._inbound_window_manager.current_window_size + + @property + def open(self): + """ + Whether the stream is 'open' in any sense: that is, whether it counts + against the number of concurrent streams. + """ + # RFC 7540 Section 5.1.2 defines 'open' for this purpose to mean either + # the OPEN state or either of the HALF_CLOSED states. Perplexingly, + # this excludes the reserved states. + # For more detail on why we're doing this in this slightly weird way, + # see the comment on ``STREAM_OPEN`` at the top of the file. + return STREAM_OPEN[self.state_machine.state] + + @property + def closed(self): + """ + Whether the stream is closed. + """ + return self.state_machine.state == StreamState.CLOSED + + @property + def closed_by(self): + """ + Returns how the stream was closed, as one of StreamClosedBy. + """ + return self.state_machine.stream_closed_by + + def upgrade(self, client_side): + """ + Called by the connection to indicate that this stream is the initial + request/response of an upgraded connection. Places the stream into an + appropriate state. + """ + self.config.logger.debug("Upgrading %r", self) + + assert self.stream_id == 1 + input_ = ( + StreamInputs.UPGRADE_CLIENT if client_side + else StreamInputs.UPGRADE_SERVER + ) + + # This may return events, we deliberately don't want them. + self.state_machine.process_input(input_) + return + + def send_headers(self, headers, encoder, end_stream=False): + """ + Returns a list of HEADERS/CONTINUATION frames to emit as either headers + or trailers. + """ + self.config.logger.debug("Send headers %s on %r", headers, self) + + # Because encoding headers makes an irreversible change to the header + # compression context, we make the state transition before we encode + # them. + + # First, check if we're a client. If we are, no problem: if we aren't, + # we need to scan the header block to see if this is an informational + # response. + input_ = StreamInputs.SEND_HEADERS + if ((not self.state_machine.client) and + is_informational_response(headers)): + if end_stream: + raise ProtocolError( + "Cannot set END_STREAM on informational responses." + ) + + input_ = StreamInputs.SEND_INFORMATIONAL_HEADERS + + events = self.state_machine.process_input(input_) + + hf = HeadersFrame(self.stream_id) + hdr_validation_flags = self._build_hdr_validation_flags(events) + frames = self._build_headers_frames( + headers, encoder, hf, hdr_validation_flags + ) + + if end_stream: + # Not a bug: the END_STREAM flag is valid on the initial HEADERS + # frame, not the CONTINUATION frames that follow. + self.state_machine.process_input(StreamInputs.SEND_END_STREAM) + frames[0].flags.add('END_STREAM') + + if self.state_machine.trailers_sent and not end_stream: + raise ProtocolError("Trailers must have END_STREAM set.") + + if self.state_machine.client and self._authority is None: + self._authority = authority_from_headers(headers) + + # store request method for _initialize_content_length + self.request_method = extract_method_header(headers) + + return frames + + def push_stream_in_band(self, related_stream_id, headers, encoder): + """ + Returns a list of PUSH_PROMISE/CONTINUATION frames to emit as a pushed + stream header. Called on the stream that has the PUSH_PROMISE frame + sent on it. + """ + self.config.logger.debug("Push stream %r", self) + + # Because encoding headers makes an irreversible change to the header + # compression context, we make the state transition *first*. + + events = self.state_machine.process_input( + StreamInputs.SEND_PUSH_PROMISE + ) + + ppf = PushPromiseFrame(self.stream_id) + ppf.promised_stream_id = related_stream_id + hdr_validation_flags = self._build_hdr_validation_flags(events) + frames = self._build_headers_frames( + headers, encoder, ppf, hdr_validation_flags + ) + + return frames + + def locally_pushed(self): + """ + Mark this stream as one that was pushed by this peer. Must be called + immediately after initialization. Sends no frames, simply updates the + state machine. + """ + # This does not trigger any events. + events = self.state_machine.process_input( + StreamInputs.SEND_PUSH_PROMISE + ) + assert not events + return [] + + def send_data(self, data, end_stream=False, pad_length=None): + """ + Prepare some data frames. Optionally end the stream. + + .. warning:: Does not perform flow control checks. + """ + self.config.logger.debug( + "Send data on %r with end stream set to %s", self, end_stream + ) + + self.state_machine.process_input(StreamInputs.SEND_DATA) + + df = DataFrame(self.stream_id) + df.data = data + if end_stream: + self.state_machine.process_input(StreamInputs.SEND_END_STREAM) + df.flags.add('END_STREAM') + if pad_length is not None: + df.flags.add('PADDED') + df.pad_length = pad_length + + # Subtract flow_controlled_length to account for possible padding + self.outbound_flow_control_window -= df.flow_controlled_length + assert self.outbound_flow_control_window >= 0 + + return [df] + + def end_stream(self): + """ + End a stream without sending data. + """ + self.config.logger.debug("End stream %r", self) + + self.state_machine.process_input(StreamInputs.SEND_END_STREAM) + df = DataFrame(self.stream_id) + df.flags.add('END_STREAM') + return [df] + + def advertise_alternative_service(self, field_value): + """ + Advertise an RFC 7838 alternative service. The semantics of this are + better documented in the ``H2Connection`` class. + """ + self.config.logger.debug( + "Advertise alternative service of %r for %r", field_value, self + ) + self.state_machine.process_input(StreamInputs.SEND_ALTERNATIVE_SERVICE) + asf = AltSvcFrame(self.stream_id) + asf.field = field_value + return [asf] + + def increase_flow_control_window(self, increment): + """ + Increase the size of the flow control window for the remote side. + """ + self.config.logger.debug( + "Increase flow control window for %r by %d", + self, increment + ) + self.state_machine.process_input(StreamInputs.SEND_WINDOW_UPDATE) + self._inbound_window_manager.window_opened(increment) + + wuf = WindowUpdateFrame(self.stream_id) + wuf.window_increment = increment + return [wuf] + + def receive_push_promise_in_band(self, + promised_stream_id, + headers, + header_encoding): + """ + Receives a push promise frame sent on this stream, pushing a remote + stream. This is called on the stream that has the PUSH_PROMISE sent + on it. + """ + self.config.logger.debug( + "Receive Push Promise on %r for remote stream %d", + self, promised_stream_id + ) + events = self.state_machine.process_input( + StreamInputs.RECV_PUSH_PROMISE + ) + events[0].pushed_stream_id = promised_stream_id + + hdr_validation_flags = self._build_hdr_validation_flags(events) + events[0].headers = self._process_received_headers( + headers, hdr_validation_flags, header_encoding + ) + return [], events + + def remotely_pushed(self, pushed_headers): + """ + Mark this stream as one that was pushed by the remote peer. Must be + called immediately after initialization. Sends no frames, simply + updates the state machine. + """ + self.config.logger.debug("%r pushed by remote peer", self) + events = self.state_machine.process_input( + StreamInputs.RECV_PUSH_PROMISE + ) + self._authority = authority_from_headers(pushed_headers) + return [], events + + def receive_headers(self, headers, end_stream, header_encoding): + """ + Receive a set of headers (or trailers). + """ + if is_informational_response(headers): + if end_stream: + raise ProtocolError( + "Cannot set END_STREAM on informational responses" + ) + input_ = StreamInputs.RECV_INFORMATIONAL_HEADERS + else: + input_ = StreamInputs.RECV_HEADERS + + events = self.state_machine.process_input(input_) + + if end_stream: + es_events = self.state_machine.process_input( + StreamInputs.RECV_END_STREAM + ) + events[0].stream_ended = es_events[0] + events += es_events + + self._initialize_content_length(headers) + + if isinstance(events[0], TrailersReceived): + if not end_stream: + raise ProtocolError("Trailers must have END_STREAM set") + + hdr_validation_flags = self._build_hdr_validation_flags(events) + events[0].headers = self._process_received_headers( + headers, hdr_validation_flags, header_encoding + ) + return [], events + + def receive_data(self, data, end_stream, flow_control_len): + """ + Receive some data. + """ + self.config.logger.debug( + "Receive data on %r with end stream %s and flow control length " + "set to %d", self, end_stream, flow_control_len + ) + events = self.state_machine.process_input(StreamInputs.RECV_DATA) + self._inbound_window_manager.window_consumed(flow_control_len) + self._track_content_length(len(data), end_stream) + + if end_stream: + es_events = self.state_machine.process_input( + StreamInputs.RECV_END_STREAM + ) + events[0].stream_ended = es_events[0] + events.extend(es_events) + + events[0].data = data + events[0].flow_controlled_length = flow_control_len + return [], events + + def receive_window_update(self, increment): + """ + Handle a WINDOW_UPDATE increment. + """ + self.config.logger.debug( + "Receive Window Update on %r for increment of %d", + self, increment + ) + events = self.state_machine.process_input( + StreamInputs.RECV_WINDOW_UPDATE + ) + frames = [] + + # If we encounter a problem with incrementing the flow control window, + # this should be treated as a *stream* error, not a *connection* error. + # That means we need to catch the error and forcibly close the stream. + if events: + events[0].delta = increment + try: + self.outbound_flow_control_window = guard_increment_window( + self.outbound_flow_control_window, + increment + ) + except FlowControlError: + # Ok, this is bad. We're going to need to perform a local + # reset. + event = StreamReset() + event.stream_id = self.stream_id + event.error_code = ErrorCodes.FLOW_CONTROL_ERROR + event.remote_reset = False + + events = [event] + frames = self.reset_stream(event.error_code) + + return frames, events + + def receive_continuation(self): + """ + A naked CONTINUATION frame has been received. This is always an error, + but the type of error it is depends on the state of the stream and must + transition the state of the stream, so we need to handle it. + """ + self.config.logger.debug("Receive Continuation frame on %r", self) + self.state_machine.process_input( + StreamInputs.RECV_CONTINUATION + ) + assert False, "Should not be reachable" + + def receive_alt_svc(self, frame): + """ + An Alternative Service frame was received on the stream. This frame + inherits the origin associated with this stream. + """ + self.config.logger.debug( + "Receive Alternative Service frame on stream %r", self + ) + + # If the origin is present, RFC 7838 says we have to ignore it. + if frame.origin: + return [], [] + + events = self.state_machine.process_input( + StreamInputs.RECV_ALTERNATIVE_SERVICE + ) + + # There are lots of situations where we want to ignore the ALTSVC + # frame. If we need to pay attention, we'll have an event and should + # fill it out. + if events: + assert isinstance(events[0], AlternativeServiceAvailable) + events[0].origin = self._authority + events[0].field_value = frame.field + + return [], events + + def reset_stream(self, error_code=0): + """ + Close the stream locally. Reset the stream with an error code. + """ + self.config.logger.debug( + "Local reset %r with error code: %d", self, error_code + ) + self.state_machine.process_input(StreamInputs.SEND_RST_STREAM) + + rsf = RstStreamFrame(self.stream_id) + rsf.error_code = error_code + return [rsf] + + def stream_reset(self, frame): + """ + Handle a stream being reset remotely. + """ + self.config.logger.debug( + "Remote reset %r with error code: %d", self, frame.error_code + ) + events = self.state_machine.process_input(StreamInputs.RECV_RST_STREAM) + + if events: + # We don't fire an event if this stream is already closed. + events[0].error_code = _error_code_from_int(frame.error_code) + + return [], events + + def acknowledge_received_data(self, acknowledged_size): + """ + The user has informed us that they've processed some amount of data + that was received on this stream. Pass that to the window manager and + potentially return some WindowUpdate frames. + """ + self.config.logger.debug( + "Acknowledge received data with size %d on %r", + acknowledged_size, self + ) + increment = self._inbound_window_manager.process_bytes( + acknowledged_size + ) + if increment: + f = WindowUpdateFrame(self.stream_id) + f.window_increment = increment + return [f] + + return [] + + def _build_hdr_validation_flags(self, events): + """ + Constructs a set of header validation flags for use when normalizing + and validating header blocks. + """ + is_trailer = isinstance( + events[0], (_TrailersSent, TrailersReceived) + ) + is_response_header = isinstance( + events[0], + ( + _ResponseSent, + ResponseReceived, + InformationalResponseReceived + ) + ) + is_push_promise = isinstance( + events[0], (PushedStreamReceived, _PushedRequestSent) + ) + + return HeaderValidationFlags( + is_client=self.state_machine.client, + is_trailer=is_trailer, + is_response_header=is_response_header, + is_push_promise=is_push_promise, + ) + + def _build_headers_frames(self, + headers, + encoder, + first_frame, + hdr_validation_flags): + """ + Helper method to build headers or push promise frames. + """ + # We need to lowercase the header names, and to ensure that secure + # header fields are kept out of compression contexts. + if self.config.normalize_outbound_headers: + headers = normalize_outbound_headers( + headers, hdr_validation_flags + ) + if self.config.validate_outbound_headers: + headers = validate_outbound_headers( + headers, hdr_validation_flags + ) + + encoded_headers = encoder.encode(headers) + + # Slice into blocks of max_outbound_frame_size. Be careful with this: + # it only works right because we never send padded frames or priority + # information on the frames. Revisit this if we do. + header_blocks = [ + encoded_headers[i:i+self.max_outbound_frame_size] + for i in range( + 0, len(encoded_headers), self.max_outbound_frame_size + ) + ] + + frames = [] + first_frame.data = header_blocks[0] + frames.append(first_frame) + + for block in header_blocks[1:]: + cf = ContinuationFrame(self.stream_id) + cf.data = block + frames.append(cf) + + frames[-1].flags.add('END_HEADERS') + return frames + + def _process_received_headers(self, + headers, + header_validation_flags, + header_encoding): + """ + When headers have been received from the remote peer, run a processing + pipeline on them to transform them into the appropriate form for + attaching to an event. + """ + if self.config.normalize_inbound_headers: + headers = normalize_inbound_headers( + headers, header_validation_flags + ) + + if self.config.validate_inbound_headers: + headers = validate_headers(headers, header_validation_flags) + + if header_encoding: + headers = _decode_headers(headers, header_encoding) + + # The above steps are all generators, so we need to concretize the + # headers now. + return list(headers) + + def _initialize_content_length(self, headers): + """ + Checks the headers for a content-length header and initializes the + _expected_content_length field from it. It's not an error for no + Content-Length header to be present. + """ + if self.request_method == b'HEAD': + self._expected_content_length = 0 + return + + for n, v in headers: + if n == b'content-length': + try: + self._expected_content_length = int(v, 10) + except ValueError: + raise ProtocolError( + "Invalid content-length header: %s" % v + ) + + return + + def _track_content_length(self, length, end_stream): + """ + Update the expected content length in response to data being received. + Validates that the appropriate amount of data is sent. Always updates + the received data, but only validates the length against the + content-length header if one was sent. + + :param length: The length of the body chunk received. + :param end_stream: If this is the last body chunk received. + """ + self._actual_content_length += length + actual = self._actual_content_length + expected = self._expected_content_length + + if expected is not None: + if expected < actual: + raise InvalidBodyLengthError(expected, actual) + + if end_stream and expected != actual: + raise InvalidBodyLengthError(expected, actual) + + def _inbound_flow_control_change_from_settings(self, delta): + """ + We changed SETTINGS_INITIAL_WINDOW_SIZE, which means we need to + update the target window size for flow control. For our flow control + strategy, this means we need to do two things: we need to adjust the + current window size, but we also need to set the target maximum window + size to the new value. + """ + new_max_size = self._inbound_window_manager.max_window_size + delta + self._inbound_window_manager.window_opened(delta) + self._inbound_window_manager.max_window_size = new_max_size + + +def _decode_headers(headers, encoding): + """ + Given an iterable of header two-tuples and an encoding, decodes those + headers using that encoding while preserving the type of the header tuple. + This ensures that the use of ``HeaderTuple`` is preserved. + """ + for header in headers: + # This function expects to work on decoded headers, which are always + # HeaderTuple objects. + assert isinstance(header, HeaderTuple) + + name, value = header + name = name.decode(encoding) + value = value.decode(encoding) + yield header.__class__(name, value) diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/h2/src/h2/utilities.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/h2/src/h2/utilities.py new file mode 100644 index 0000000000..c7a620d8d4 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/h2/src/h2/utilities.py @@ -0,0 +1,672 @@ +# -*- coding: utf-8 -*- +""" +h2/utilities +~~~~~~~~~~~~ + +Utility functions that do not belong in a separate module. +""" +import collections +import re +from string import whitespace + +from hpack import HeaderTuple, NeverIndexedHeaderTuple + +from .exceptions import ProtocolError, FlowControlError + +UPPER_RE = re.compile(b"[A-Z]") + +# A set of headers that are hop-by-hop or connection-specific and thus +# forbidden in HTTP/2. This list comes from RFC 7540 ยง 8.1.2.2. +CONNECTION_HEADERS = frozenset([ + b'connection', u'connection', + b'proxy-connection', u'proxy-connection', + b'keep-alive', u'keep-alive', + b'transfer-encoding', u'transfer-encoding', + b'upgrade', u'upgrade', +]) + + +_ALLOWED_PSEUDO_HEADER_FIELDS = frozenset([ + b':method', u':method', + b':scheme', u':scheme', + b':authority', u':authority', + b':path', u':path', + b':status', u':status', + b':protocol', u':protocol', +]) + + +_SECURE_HEADERS = frozenset([ + # May have basic credentials which are vulnerable to dictionary attacks. + b'authorization', u'authorization', + b'proxy-authorization', u'proxy-authorization', +]) + + +_REQUEST_ONLY_HEADERS = frozenset([ + b':scheme', u':scheme', + b':path', u':path', + b':authority', u':authority', + b':method', u':method', + b':protocol', u':protocol', +]) + + +_RESPONSE_ONLY_HEADERS = frozenset([b':status', u':status']) + + +# A Set of pseudo headers that are only valid if the method is +# CONNECT, see RFC 8441 ยง 5 +_CONNECT_REQUEST_ONLY_HEADERS = frozenset([b':protocol', u':protocol']) + + +_WHITESPACE = frozenset(map(ord, whitespace)) + + +def _secure_headers(headers, hdr_validation_flags): + """ + Certain headers are at risk of being attacked during the header compression + phase, and so need to be kept out of header compression contexts. This + function automatically transforms certain specific headers into HPACK + never-indexed fields to ensure they don't get added to header compression + contexts. + + This function currently implements two rules: + + - 'authorization' and 'proxy-authorization' fields are automatically made + never-indexed. + - Any 'cookie' header field shorter than 20 bytes long is made + never-indexed. + + These fields are the most at-risk. These rules are inspired by Firefox + and nghttp2. + """ + for header in headers: + if header[0] in _SECURE_HEADERS: + yield NeverIndexedHeaderTuple(*header) + elif header[0] in (b'cookie', u'cookie') and len(header[1]) < 20: + yield NeverIndexedHeaderTuple(*header) + else: + yield header + + +def extract_method_header(headers): + """ + Extracts the request method from the headers list. + """ + for k, v in headers: + if k in (b':method', u':method'): + if not isinstance(v, bytes): + return v.encode('utf-8') + else: + return v + + +def is_informational_response(headers): + """ + Searches a header block for a :status header to confirm that a given + collection of headers are an informational response. Assumes the header + block is well formed: that is, that the HTTP/2 special headers are first + in the block, and so that it can stop looking when it finds the first + header field whose name does not begin with a colon. + + :param headers: The HTTP/2 header block. + :returns: A boolean indicating if this is an informational response. + """ + for n, v in headers: + if isinstance(n, bytes): + sigil = b':' + status = b':status' + informational_start = b'1' + else: + sigil = u':' + status = u':status' + informational_start = u'1' + + # If we find a non-special header, we're done here: stop looping. + if not n.startswith(sigil): + return False + + # This isn't the status header, bail. + if n != status: + continue + + # If the first digit is a 1, we've got informational headers. + return v.startswith(informational_start) + + +def guard_increment_window(current, increment): + """ + Increments a flow control window, guarding against that window becoming too + large. + + :param current: The current value of the flow control window. + :param increment: The increment to apply to that window. + :returns: The new value of the window. + :raises: ``FlowControlError`` + """ + # The largest value the flow control window may take. + LARGEST_FLOW_CONTROL_WINDOW = 2**31 - 1 + + new_size = current + increment + + if new_size > LARGEST_FLOW_CONTROL_WINDOW: + raise FlowControlError( + "May not increment flow control window past %d" % + LARGEST_FLOW_CONTROL_WINDOW + ) + + return new_size + + +def authority_from_headers(headers): + """ + Given a header set, searches for the authority header and returns the + value. + + Note that this doesn't terminate early, so should only be called if the + headers are for a client request. Otherwise, will loop over the entire + header set, which is potentially unwise. + + :param headers: The HTTP header set. + :returns: The value of the authority header, or ``None``. + :rtype: ``bytes`` or ``None``. + """ + for n, v in headers: + # This gets run against headers that come both from HPACK and from the + # user, so we may have unicode floating around in here. We only want + # bytes. + if n in (b':authority', u':authority'): + return v.encode('utf-8') if not isinstance(v, bytes) else v + + return None + + +# Flags used by the validate_headers pipeline to determine which checks +# should be applied to a given set of headers. +HeaderValidationFlags = collections.namedtuple( + 'HeaderValidationFlags', + ['is_client', 'is_trailer', 'is_response_header', 'is_push_promise'] +) + + +def validate_headers(headers, hdr_validation_flags): + """ + Validates a header sequence against a set of constraints from RFC 7540. + + :param headers: The HTTP header set. + :param hdr_validation_flags: An instance of HeaderValidationFlags. + """ + # This validation logic is built on a sequence of generators that are + # iterated over to provide the final header list. This reduces some of the + # overhead of doing this checking. However, it's worth noting that this + # checking remains somewhat expensive, and attempts should be made wherever + # possible to reduce the time spent doing them. + # + # For example, we avoid tuple unpacking in loops because it represents a + # fixed cost that we don't want to spend, instead indexing into the header + # tuples. + headers = _reject_empty_header_names( + headers, hdr_validation_flags + ) + headers = _reject_uppercase_header_fields( + headers, hdr_validation_flags + ) + headers = _reject_surrounding_whitespace( + headers, hdr_validation_flags + ) + headers = _reject_te( + headers, hdr_validation_flags + ) + headers = _reject_connection_header( + headers, hdr_validation_flags + ) + headers = _reject_pseudo_header_fields( + headers, hdr_validation_flags + ) + headers = _check_host_authority_header( + headers, hdr_validation_flags + ) + headers = _check_path_header(headers, hdr_validation_flags) + + return headers + + +def _reject_empty_header_names(headers, hdr_validation_flags): + """ + Raises a ProtocolError if any header names are empty (length 0). + While hpack decodes such headers without errors, they are semantically + forbidden in HTTP, see RFC 7230, stating that they must be at least one + character long. + """ + for header in headers: + if len(header[0]) == 0: + raise ProtocolError("Received header name with zero length.") + yield header + + +def _reject_uppercase_header_fields(headers, hdr_validation_flags): + """ + Raises a ProtocolError if any uppercase character is found in a header + block. + """ + for header in headers: + if UPPER_RE.search(header[0]): + raise ProtocolError( + "Received uppercase header name %s." % header[0]) + yield header + + +def _reject_surrounding_whitespace(headers, hdr_validation_flags): + """ + Raises a ProtocolError if any header name or value is surrounded by + whitespace characters. + """ + # For compatibility with RFC 7230 header fields, we need to allow the field + # value to be an empty string. This is ludicrous, but technically allowed. + # The field name may not be empty, though, so we can safely assume that it + # must have at least one character in it and throw exceptions if it + # doesn't. + for header in headers: + if header[0][0] in _WHITESPACE or header[0][-1] in _WHITESPACE: + raise ProtocolError( + "Received header name surrounded by whitespace %r" % header[0]) + if header[1] and ((header[1][0] in _WHITESPACE) or + (header[1][-1] in _WHITESPACE)): + raise ProtocolError( + "Received header value surrounded by whitespace %r" % header[1] + ) + yield header + + +def _reject_te(headers, hdr_validation_flags): + """ + Raises a ProtocolError if the TE header is present in a header block and + its value is anything other than "trailers". + """ + for header in headers: + if header[0] in (b'te', u'te'): + if header[1].lower() not in (b'trailers', u'trailers'): + raise ProtocolError( + "Invalid value for TE header: %s" % + header[1] + ) + + yield header + + +def _reject_connection_header(headers, hdr_validation_flags): + """ + Raises a ProtocolError if the Connection header is present in a header + block. + """ + for header in headers: + if header[0] in CONNECTION_HEADERS: + raise ProtocolError( + "Connection-specific header field present: %s." % header[0] + ) + + yield header + + +def _custom_startswith(test_string, bytes_prefix, unicode_prefix): + """ + Given a string that might be a bytestring or a Unicode string, + return True if it starts with the appropriate prefix. + """ + if isinstance(test_string, bytes): + return test_string.startswith(bytes_prefix) + else: + return test_string.startswith(unicode_prefix) + + +def _assert_header_in_set(string_header, bytes_header, header_set): + """ + Given a set of header names, checks whether the string or byte version of + the header name is present. Raises a Protocol error with the appropriate + error if it's missing. + """ + if not (string_header in header_set or bytes_header in header_set): + raise ProtocolError( + "Header block missing mandatory %s header" % string_header + ) + + +def _reject_pseudo_header_fields(headers, hdr_validation_flags): + """ + Raises a ProtocolError if duplicate pseudo-header fields are found in a + header block or if a pseudo-header field appears in a block after an + ordinary header field. + + Raises a ProtocolError if pseudo-header fields are found in trailers. + """ + seen_pseudo_header_fields = set() + seen_regular_header = False + method = None + + for header in headers: + if _custom_startswith(header[0], b':', u':'): + if header[0] in seen_pseudo_header_fields: + raise ProtocolError( + "Received duplicate pseudo-header field %s" % header[0] + ) + + seen_pseudo_header_fields.add(header[0]) + + if seen_regular_header: + raise ProtocolError( + "Received pseudo-header field out of sequence: %s" % + header[0] + ) + + if header[0] not in _ALLOWED_PSEUDO_HEADER_FIELDS: + raise ProtocolError( + "Received custom pseudo-header field %s" % header[0] + ) + + if header[0] in (b':method', u':method'): + if not isinstance(header[1], bytes): + method = header[1].encode('utf-8') + else: + method = header[1] + + else: + seen_regular_header = True + + yield header + + # Check the pseudo-headers we got to confirm they're acceptable. + _check_pseudo_header_field_acceptability( + seen_pseudo_header_fields, method, hdr_validation_flags + ) + + +def _check_pseudo_header_field_acceptability(pseudo_headers, + method, + hdr_validation_flags): + """ + Given the set of pseudo-headers present in a header block and the + validation flags, confirms that RFC 7540 allows them. + """ + # Pseudo-header fields MUST NOT appear in trailers - RFC 7540 ยง 8.1.2.1 + if hdr_validation_flags.is_trailer and pseudo_headers: + raise ProtocolError( + "Received pseudo-header in trailer %s" % pseudo_headers + ) + + # If ':status' pseudo-header is not there in a response header, reject it. + # Similarly, if ':path', ':method', or ':scheme' are not there in a request + # header, reject it. Additionally, if a response contains any request-only + # headers or vice-versa, reject it. + # Relevant RFC section: RFC 7540 ยง 8.1.2.4 + # https://tools.ietf.org/html/rfc7540#section-8.1.2.4 + if hdr_validation_flags.is_response_header: + _assert_header_in_set(u':status', b':status', pseudo_headers) + invalid_response_headers = pseudo_headers & _REQUEST_ONLY_HEADERS + if invalid_response_headers: + raise ProtocolError( + "Encountered request-only headers %s" % + invalid_response_headers + ) + elif (not hdr_validation_flags.is_response_header and + not hdr_validation_flags.is_trailer): + # This is a request, so we need to have seen :path, :method, and + # :scheme. + _assert_header_in_set(u':path', b':path', pseudo_headers) + _assert_header_in_set(u':method', b':method', pseudo_headers) + _assert_header_in_set(u':scheme', b':scheme', pseudo_headers) + invalid_request_headers = pseudo_headers & _RESPONSE_ONLY_HEADERS + if invalid_request_headers: + raise ProtocolError( + "Encountered response-only headers %s" % + invalid_request_headers + ) + if method != b'CONNECT': + invalid_headers = pseudo_headers & _CONNECT_REQUEST_ONLY_HEADERS + if invalid_headers: + raise ProtocolError( + "Encountered connect-request-only headers %s" % + invalid_headers + ) + + +def _validate_host_authority_header(headers): + """ + Given the :authority and Host headers from a request block that isn't + a trailer, check that: + 1. At least one of these headers is set. + 2. If both headers are set, they match. + + :param headers: The HTTP header set. + :raises: ``ProtocolError`` + """ + # We use None as a sentinel value. Iterate over the list of headers, + # and record the value of these headers (if present). We don't need + # to worry about receiving duplicate :authority headers, as this is + # enforced by the _reject_pseudo_header_fields() pipeline. + # + # TODO: We should also guard against receiving duplicate Host headers, + # and against sending duplicate headers. + authority_header_val = None + host_header_val = None + + for header in headers: + if header[0] in (b':authority', u':authority'): + authority_header_val = header[1] + elif header[0] in (b'host', u'host'): + host_header_val = header[1] + + yield header + + # If we have not-None values for these variables, then we know we saw + # the corresponding header. + authority_present = (authority_header_val is not None) + host_present = (host_header_val is not None) + + # It is an error for a request header block to contain neither + # an :authority header nor a Host header. + if not authority_present and not host_present: + raise ProtocolError( + "Request header block does not have an :authority or Host header." + ) + + # If we receive both headers, they should definitely match. + if authority_present and host_present: + if authority_header_val != host_header_val: + raise ProtocolError( + "Request header block has mismatched :authority and " + "Host headers: %r / %r" + % (authority_header_val, host_header_val) + ) + + +def _check_host_authority_header(headers, hdr_validation_flags): + """ + Raises a ProtocolError if a header block arrives that does not contain an + :authority or a Host header, or if a header block contains both fields, + but their values do not match. + """ + # We only expect to see :authority and Host headers on request header + # blocks that aren't trailers, so skip this validation if this is a + # response header or we're looking at trailer blocks. + skip_validation = ( + hdr_validation_flags.is_response_header or + hdr_validation_flags.is_trailer + ) + if skip_validation: + return headers + + return _validate_host_authority_header(headers) + + +def _check_path_header(headers, hdr_validation_flags): + """ + Raise a ProtocolError if a header block arrives or is sent that contains an + empty :path header. + """ + def inner(): + for header in headers: + if header[0] in (b':path', u':path'): + if not header[1]: + raise ProtocolError("An empty :path header is forbidden") + + yield header + + # We only expect to see :authority and Host headers on request header + # blocks that aren't trailers, so skip this validation if this is a + # response header or we're looking at trailer blocks. + skip_validation = ( + hdr_validation_flags.is_response_header or + hdr_validation_flags.is_trailer + ) + if skip_validation: + return headers + else: + return inner() + + +def _lowercase_header_names(headers, hdr_validation_flags): + """ + Given an iterable of header two-tuples, rebuilds that iterable with the + header names lowercased. This generator produces tuples that preserve the + original type of the header tuple for tuple and any ``HeaderTuple``. + """ + for header in headers: + if isinstance(header, HeaderTuple): + yield header.__class__(header[0].lower(), header[1]) + else: + yield (header[0].lower(), header[1]) + + +def _strip_surrounding_whitespace(headers, hdr_validation_flags): + """ + Given an iterable of header two-tuples, strip both leading and trailing + whitespace from both header names and header values. This generator + produces tuples that preserve the original type of the header tuple for + tuple and any ``HeaderTuple``. + """ + for header in headers: + if isinstance(header, HeaderTuple): + yield header.__class__(header[0].strip(), header[1].strip()) + else: + yield (header[0].strip(), header[1].strip()) + + +def _strip_connection_headers(headers, hdr_validation_flags): + """ + Strip any connection headers as per RFC7540 ยง 8.1.2.2. + """ + for header in headers: + if header[0] not in CONNECTION_HEADERS: + yield header + + +def _check_sent_host_authority_header(headers, hdr_validation_flags): + """ + Raises an InvalidHeaderBlockError if we try to send a header block + that does not contain an :authority or a Host header, or if + the header block contains both fields, but their values do not match. + """ + # We only expect to see :authority and Host headers on request header + # blocks that aren't trailers, so skip this validation if this is a + # response header or we're looking at trailer blocks. + skip_validation = ( + hdr_validation_flags.is_response_header or + hdr_validation_flags.is_trailer + ) + if skip_validation: + return headers + + return _validate_host_authority_header(headers) + + +def _combine_cookie_fields(headers, hdr_validation_flags): + """ + RFC 7540 ยง 8.1.2.5 allows HTTP/2 clients to split the Cookie header field, + which must normally appear only once, into multiple fields for better + compression. However, they MUST be joined back up again when received. + This normalization step applies that transform. The side-effect is that + all cookie fields now appear *last* in the header block. + """ + # There is a problem here about header indexing. Specifically, it's + # possible that all these cookies are sent with different header indexing + # values. At this point it shouldn't matter too much, so we apply our own + # logic and make them never-indexed. + cookies = [] + for header in headers: + if header[0] == b'cookie': + cookies.append(header[1]) + else: + yield header + if cookies: + cookie_val = b'; '.join(cookies) + yield NeverIndexedHeaderTuple(b'cookie', cookie_val) + + +def normalize_outbound_headers(headers, hdr_validation_flags): + """ + Normalizes a header sequence that we are about to send. + + :param headers: The HTTP header set. + :param hdr_validation_flags: An instance of HeaderValidationFlags. + """ + headers = _lowercase_header_names(headers, hdr_validation_flags) + headers = _strip_surrounding_whitespace(headers, hdr_validation_flags) + headers = _strip_connection_headers(headers, hdr_validation_flags) + headers = _secure_headers(headers, hdr_validation_flags) + + return headers + + +def normalize_inbound_headers(headers, hdr_validation_flags): + """ + Normalizes a header sequence that we have received. + + :param headers: The HTTP header set. + :param hdr_validation_flags: An instance of HeaderValidationFlags + """ + headers = _combine_cookie_fields(headers, hdr_validation_flags) + return headers + + +def validate_outbound_headers(headers, hdr_validation_flags): + """ + Validates and normalizes a header sequence that we are about to send. + + :param headers: The HTTP header set. + :param hdr_validation_flags: An instance of HeaderValidationFlags. + """ + headers = _reject_te( + headers, hdr_validation_flags + ) + headers = _reject_connection_header( + headers, hdr_validation_flags + ) + headers = _reject_pseudo_header_fields( + headers, hdr_validation_flags + ) + headers = _check_sent_host_authority_header( + headers, hdr_validation_flags + ) + headers = _check_path_header(headers, hdr_validation_flags) + + return headers + + +class SizeLimitDict(collections.OrderedDict): + + def __init__(self, *args, **kwargs): + self._size_limit = kwargs.pop("size_limit", None) + super(SizeLimitDict, self).__init__(*args, **kwargs) + + self._check_size_limit() + + def __setitem__(self, key, value): + super(SizeLimitDict, self).__setitem__(key, value) + + self._check_size_limit() + + def _check_size_limit(self): + if self._size_limit is not None: + while len(self) > self._size_limit: + self.popitem(last=False) diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/h2/src/h2/windows.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/h2/src/h2/windows.py new file mode 100644 index 0000000000..be4eb438b1 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/h2/src/h2/windows.py @@ -0,0 +1,139 @@ +# -*- coding: utf-8 -*- +""" +h2/windows +~~~~~~~~~~ + +Defines tools for managing HTTP/2 flow control windows. + +The objects defined in this module are used to automatically manage HTTP/2 +flow control windows. Specifically, they keep track of what the size of the +window is, how much data has been consumed from that window, and how much data +the user has already used. It then implements a basic algorithm that attempts +to manage the flow control window without user input, trying to ensure that it +does not emit too many WINDOW_UPDATE frames. +""" +from __future__ import division + +from .exceptions import FlowControlError + + +# The largest acceptable value for a HTTP/2 flow control window. +LARGEST_FLOW_CONTROL_WINDOW = 2**31 - 1 + + +class WindowManager: + """ + A basic HTTP/2 window manager. + + :param max_window_size: The maximum size of the flow control window. + :type max_window_size: ``int`` + """ + def __init__(self, max_window_size): + assert max_window_size <= LARGEST_FLOW_CONTROL_WINDOW + self.max_window_size = max_window_size + self.current_window_size = max_window_size + self._bytes_processed = 0 + + def window_consumed(self, size): + """ + We have received a certain number of bytes from the remote peer. This + necessarily shrinks the flow control window! + + :param size: The number of flow controlled bytes we received from the + remote peer. + :type size: ``int`` + :returns: Nothing. + :rtype: ``None`` + """ + self.current_window_size -= size + if self.current_window_size < 0: + raise FlowControlError("Flow control window shrunk below 0") + + def window_opened(self, size): + """ + The flow control window has been incremented, either because of manual + flow control management or because of the user changing the flow + control settings. This can have the effect of increasing what we + consider to be the "maximum" flow control window size. + + This does not increase our view of how many bytes have been processed, + only of how much space is in the window. + + :param size: The increment to the flow control window we received. + :type size: ``int`` + :returns: Nothing + :rtype: ``None`` + """ + self.current_window_size += size + + if self.current_window_size > LARGEST_FLOW_CONTROL_WINDOW: + raise FlowControlError( + "Flow control window mustn't exceed %d" % + LARGEST_FLOW_CONTROL_WINDOW + ) + + if self.current_window_size > self.max_window_size: + self.max_window_size = self.current_window_size + + def process_bytes(self, size): + """ + The application has informed us that it has processed a certain number + of bytes. This may cause us to want to emit a window update frame. If + we do want to emit a window update frame, this method will return the + number of bytes that we should increment the window by. + + :param size: The number of flow controlled bytes that the application + has processed. + :type size: ``int`` + :returns: The number of bytes to increment the flow control window by, + or ``None``. + :rtype: ``int`` or ``None`` + """ + self._bytes_processed += size + return self._maybe_update_window() + + def _maybe_update_window(self): + """ + Run the algorithm. + + Our current algorithm can be described like this. + + 1. If no bytes have been processed, we immediately return 0. There is + no meaningful way for us to hand space in the window back to the + remote peer, so let's not even try. + 2. If there is no space in the flow control window, and we have + processed at least 1024 bytes (or 1/4 of the window, if the window + is smaller), we will emit a window update frame. This is to avoid + the risk of blocking a stream altogether. + 3. If there is space in the flow control window, and we have processed + at least 1/2 of the window worth of bytes, we will emit a window + update frame. This is to minimise the number of window update frames + we have to emit. + + In a healthy system with large flow control windows, this will + irregularly emit WINDOW_UPDATE frames. This prevents us starving the + connection by emitting eleventy bajillion WINDOW_UPDATE frames, + especially in situations where the remote peer is sending a lot of very + small DATA frames. + """ + # TODO: Can the window be smaller than 1024 bytes? If not, we can + # streamline this algorithm. + if not self._bytes_processed: + return None + + max_increment = (self.max_window_size - self.current_window_size) + increment = 0 + + # Note that, even though we may increment less than _bytes_processed, + # we still want to set it to zero whenever we emit an increment. This + # is because we'll always increment up to the maximum we can. + if (self.current_window_size == 0) and ( + self._bytes_processed > min(1024, self.max_window_size // 4)): + increment = min(self._bytes_processed, max_increment) + self._bytes_processed = 0 + elif self._bytes_processed >= (self.max_window_size // 2): + increment = min(self._bytes_processed, max_increment) + self._bytes_processed = 0 + + self.current_window_size += increment + return increment diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/hpack/src/hpack/__init__.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/hpack/src/hpack/__init__.py new file mode 100644 index 0000000000..fc26ac5a56 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/hpack/src/hpack/__init__.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +""" +hpack +~~~~~ + +HTTP/2 header encoding for Python. +""" +from .hpack import Encoder, Decoder +from .struct import HeaderTuple, NeverIndexedHeaderTuple +from .exceptions import ( + HPACKError, + HPACKDecodingError, + InvalidTableIndex, + OversizedHeaderListError, + InvalidTableSizeError +) + +__all__ = [ + 'Encoder', + 'Decoder', + 'HeaderTuple', + 'NeverIndexedHeaderTuple', + 'HPACKError', + 'HPACKDecodingError', + 'InvalidTableIndex', + 'OversizedHeaderListError', + 'InvalidTableSizeError', +] + +__version__ = '4.0.0' diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/hpack/src/hpack/exceptions.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/hpack/src/hpack/exceptions.py new file mode 100644 index 0000000000..571ba98f2c --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/hpack/src/hpack/exceptions.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- +""" +hyper/http20/exceptions +~~~~~~~~~~~~~~~~~~~~~~~ + +This defines exceptions used in the HTTP/2 portion of hyper. +""" + + +class HPACKError(Exception): + """ + The base class for all ``hpack`` exceptions. + """ + pass + + +class HPACKDecodingError(HPACKError): + """ + An error has been encountered while performing HPACK decoding. + """ + pass + + +class InvalidTableIndex(HPACKDecodingError): + """ + An invalid table index was received. + """ + pass + + +class OversizedHeaderListError(HPACKDecodingError): + """ + A header list that was larger than we allow has been received. This may be + a DoS attack. + + .. versionadded:: 2.3.0 + """ + pass + + +class InvalidTableSizeError(HPACKDecodingError): + """ + An attempt was made to change the decoder table size to a value larger than + allowed, or the list was shrunk and the remote peer didn't shrink their + table size. + + .. versionadded:: 3.0.0 + """ + pass diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/hpack/src/hpack/hpack.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/hpack/src/hpack/hpack.py new file mode 100644 index 0000000000..cc39bfd6fc --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/hpack/src/hpack/hpack.py @@ -0,0 +1,633 @@ +# -*- coding: utf-8 -*- +""" +hpack/hpack +~~~~~~~~~~~ + +Implements the HPACK header compression algorithm as detailed by the IETF. +""" +import logging + +from .table import HeaderTable, table_entry_size +from .exceptions import ( + HPACKDecodingError, OversizedHeaderListError, InvalidTableSizeError +) +from .huffman import HuffmanEncoder +from .huffman_constants import ( + REQUEST_CODES, REQUEST_CODES_LENGTH +) +from .huffman_table import decode_huffman +from .struct import HeaderTuple, NeverIndexedHeaderTuple + +log = logging.getLogger(__name__) + +INDEX_NONE = b'\x00' +INDEX_NEVER = b'\x10' +INDEX_INCREMENTAL = b'\x40' + +# Precompute 2^i for 1-8 for use in prefix calcs. +# Zero index is not used but there to save a subtraction +# as prefix numbers are not zero indexed. +_PREFIX_BIT_MAX_NUMBERS = [(2 ** i) - 1 for i in range(9)] + +try: # pragma: no cover + basestring = basestring +except NameError: # pragma: no cover + basestring = (str, bytes) + + +# We default the maximum header list we're willing to accept to 64kB. That's a +# lot of headers, but if applications want to raise it they can do. +DEFAULT_MAX_HEADER_LIST_SIZE = 2 ** 16 + + +def _unicode_if_needed(header, raw): + """ + Provides a header as a unicode string if raw is False, otherwise returns + it as a bytestring. + """ + name = bytes(header[0]) + value = bytes(header[1]) + if not raw: + name = name.decode('utf-8') + value = value.decode('utf-8') + return header.__class__(name, value) + + +def encode_integer(integer, prefix_bits): + """ + This encodes an integer according to the wacky integer encoding rules + defined in the HPACK spec. + """ + log.debug("Encoding %d with %d bits", integer, prefix_bits) + + if integer < 0: + raise ValueError( + "Can only encode positive integers, got %s" % integer + ) + + if prefix_bits < 1 or prefix_bits > 8: + raise ValueError( + "Prefix bits must be between 1 and 8, got %s" % prefix_bits + ) + + max_number = _PREFIX_BIT_MAX_NUMBERS[prefix_bits] + + if integer < max_number: + return bytearray([integer]) # Seriously? + else: + elements = [max_number] + integer -= max_number + + while integer >= 128: + elements.append((integer & 127) + 128) + integer >>= 7 + + elements.append(integer) + + return bytearray(elements) + + +def decode_integer(data, prefix_bits): + """ + This decodes an integer according to the wacky integer encoding rules + defined in the HPACK spec. Returns a tuple of the decoded integer and the + number of bytes that were consumed from ``data`` in order to get that + integer. + """ + if prefix_bits < 1 or prefix_bits > 8: + raise ValueError( + "Prefix bits must be between 1 and 8, got %s" % prefix_bits + ) + + max_number = _PREFIX_BIT_MAX_NUMBERS[prefix_bits] + index = 1 + shift = 0 + mask = (0xFF >> (8 - prefix_bits)) + + try: + number = data[0] & mask + if number == max_number: + while True: + next_byte = data[index] + index += 1 + + if next_byte >= 128: + number += (next_byte - 128) << shift + else: + number += next_byte << shift + break + shift += 7 + + except IndexError: + raise HPACKDecodingError( + "Unable to decode HPACK integer representation from %r" % data + ) + + log.debug("Decoded %d, consumed %d bytes", number, index) + + return number, index + + +def _dict_to_iterable(header_dict): + """ + This converts a dictionary to an iterable of two-tuples. This is a + HPACK-specific function because it pulls "special-headers" out first and + then emits them. + """ + assert isinstance(header_dict, dict) + keys = sorted( + header_dict.keys(), + key=lambda k: not _to_bytes(k).startswith(b':') + ) + for key in keys: + yield key, header_dict[key] + + +def _to_bytes(string): + """ + Convert string to bytes. + """ + if not isinstance(string, basestring): # pragma: no cover + string = str(string) + + return string if isinstance(string, bytes) else string.encode('utf-8') + + +class Encoder: + """ + An HPACK encoder object. This object takes HTTP headers and emits encoded + HTTP/2 header blocks. + """ + + def __init__(self): + self.header_table = HeaderTable() + self.huffman_coder = HuffmanEncoder( + REQUEST_CODES, REQUEST_CODES_LENGTH + ) + self.table_size_changes = [] + + @property + def header_table_size(self): + """ + Controls the size of the HPACK header table. + """ + return self.header_table.maxsize + + @header_table_size.setter + def header_table_size(self, value): + self.header_table.maxsize = value + if self.header_table.resized: + self.table_size_changes.append(value) + + def encode(self, headers, huffman=True): + """ + Takes a set of headers and encodes them into a HPACK-encoded header + block. + + :param headers: The headers to encode. Must be either an iterable of + tuples, an iterable of :class:`HeaderTuple + `, or a ``dict``. + + If an iterable of tuples, the tuples may be either + two-tuples or three-tuples. If they are two-tuples, the + tuples must be of the format ``(name, value)``. If they + are three-tuples, they must be of the format + ``(name, value, sensitive)``, where ``sensitive`` is a + boolean value indicating whether the header should be + added to header tables anywhere. If not present, + ``sensitive`` defaults to ``False``. + + If an iterable of :class:`HeaderTuple + `, the tuples must always be + two-tuples. Instead of using ``sensitive`` as a third + tuple entry, use :class:`NeverIndexedHeaderTuple + ` to request that + the field never be indexed. + + .. warning:: HTTP/2 requires that all special headers + (headers whose names begin with ``:`` characters) + appear at the *start* of the header block. While + this method will ensure that happens for ``dict`` + subclasses, callers using any other iterable of + tuples **must** ensure they place their special + headers at the start of the iterable. + + For efficiency reasons users should prefer to use + iterables of two-tuples: fixing the ordering of + dictionary headers is an expensive operation that + should be avoided if possible. + + :param huffman: (optional) Whether to Huffman-encode any header sent as + a literal value. Except for use when debugging, it is + recommended that this be left enabled. + + :returns: A bytestring containing the HPACK-encoded header block. + """ + # Transforming the headers into a header block is a procedure that can + # be modeled as a chain or pipe. First, the headers are encoded. This + # encoding can be done a number of ways. If the header name-value pair + # are already in the header table we can represent them using the + # indexed representation: the same is true if they are in the static + # table. Otherwise, a literal representation will be used. + header_block = [] + + # Turn the headers into a list of tuples if possible. This is the + # natural way to interact with them in HPACK. Because dictionaries are + # un-ordered, we need to make sure we grab the "special" headers first. + if isinstance(headers, dict): + headers = _dict_to_iterable(headers) + + # Before we begin, if the header table size has been changed we need + # to signal all changes since last emission appropriately. + if self.header_table.resized: + header_block.append(self._encode_table_size_change()) + self.header_table.resized = False + + # Add each header to the header block + for header in headers: + sensitive = False + if isinstance(header, HeaderTuple): + sensitive = not header.indexable + elif len(header) > 2: + sensitive = header[2] + + header = (_to_bytes(header[0]), _to_bytes(header[1])) + header_block.append(self.add(header, sensitive, huffman)) + + header_block = b''.join(header_block) + + log.debug("Encoded header block to %s", header_block) + + return header_block + + def add(self, to_add, sensitive, huffman=False): + """ + This function takes a header key-value tuple and serializes it. + """ + log.debug( + "Adding %s to the header table, sensitive:%s, huffman:%s", + to_add, + sensitive, + huffman + ) + + name, value = to_add + + # Set our indexing mode + indexbit = INDEX_INCREMENTAL if not sensitive else INDEX_NEVER + + # Search for a matching header in the header table. + match = self.header_table.search(name, value) + + if match is None: + # Not in the header table. Encode using the literal syntax, + # and add it to the header table. + encoded = self._encode_literal(name, value, indexbit, huffman) + if not sensitive: + self.header_table.add(name, value) + return encoded + + # The header is in the table, break out the values. If we matched + # perfectly, we can use the indexed representation: otherwise we + # can use the indexed literal. + index, name, perfect = match + + if perfect: + # Indexed representation. + encoded = self._encode_indexed(index) + else: + # Indexed literal. We are going to add header to the + # header table unconditionally. It is a future todo to + # filter out headers which are known to be ineffective for + # indexing since they just take space in the table and + # pushed out other valuable headers. + encoded = self._encode_indexed_literal( + index, value, indexbit, huffman + ) + if not sensitive: + self.header_table.add(name, value) + + return encoded + + def _encode_indexed(self, index): + """ + Encodes a header using the indexed representation. + """ + field = encode_integer(index, 7) + field[0] |= 0x80 # we set the top bit + return bytes(field) + + def _encode_literal(self, name, value, indexbit, huffman=False): + """ + Encodes a header with a literal name and literal value. If ``indexing`` + is True, the header will be added to the header table: otherwise it + will not. + """ + if huffman: + name = self.huffman_coder.encode(name) + value = self.huffman_coder.encode(value) + + name_len = encode_integer(len(name), 7) + value_len = encode_integer(len(value), 7) + + if huffman: + name_len[0] |= 0x80 + value_len[0] |= 0x80 + + return b''.join( + [indexbit, bytes(name_len), name, bytes(value_len), value] + ) + + def _encode_indexed_literal(self, index, value, indexbit, huffman=False): + """ + Encodes a header with an indexed name and a literal value and performs + incremental indexing. + """ + if indexbit != INDEX_INCREMENTAL: + prefix = encode_integer(index, 4) + else: + prefix = encode_integer(index, 6) + + prefix[0] |= ord(indexbit) + + if huffman: + value = self.huffman_coder.encode(value) + + value_len = encode_integer(len(value), 7) + + if huffman: + value_len[0] |= 0x80 + + return b''.join([bytes(prefix), bytes(value_len), value]) + + def _encode_table_size_change(self): + """ + Produces the encoded form of all header table size change context + updates. + """ + block = b'' + for size_bytes in self.table_size_changes: + size_bytes = encode_integer(size_bytes, 5) + size_bytes[0] |= 0x20 + block += bytes(size_bytes) + self.table_size_changes = [] + return block + + +class Decoder: + """ + An HPACK decoder object. + + .. versionchanged:: 2.3.0 + Added ``max_header_list_size`` argument. + + :param max_header_list_size: The maximum decompressed size we will allow + for any single header block. This is a protection against DoS attacks + that attempt to force the application to expand a relatively small + amount of data into a really large header list, allowing enormous + amounts of memory to be allocated. + + If this amount of data is exceeded, a `OversizedHeaderListError + ` exception will be raised. At this + point the connection should be shut down, as the HPACK state will no + longer be usable. + + Defaults to 64kB. + :type max_header_list_size: ``int`` + """ + def __init__(self, max_header_list_size=DEFAULT_MAX_HEADER_LIST_SIZE): + self.header_table = HeaderTable() + + #: The maximum decompressed size we will allow for any single header + #: block. This is a protection against DoS attacks that attempt to + #: force the application to expand a relatively small amount of data + #: into a really large header list, allowing enormous amounts of memory + #: to be allocated. + #: + #: If this amount of data is exceeded, a `OversizedHeaderListError + #: ` exception will be raised. At this + #: point the connection should be shut down, as the HPACK state will no + #: longer be usable. + #: + #: Defaults to 64kB. + #: + #: .. versionadded:: 2.3.0 + self.max_header_list_size = max_header_list_size + + #: Maximum allowed header table size. + #: + #: A HTTP/2 implementation should set this to the most recent value of + #: SETTINGS_HEADER_TABLE_SIZE that it sent *and has received an ACK + #: for*. Once this setting is set, the actual header table size will be + #: checked at the end of each decoding run and whenever it is changed, + #: to confirm that it fits in this size. + self.max_allowed_table_size = self.header_table.maxsize + + @property + def header_table_size(self): + """ + Controls the size of the HPACK header table. + """ + return self.header_table.maxsize + + @header_table_size.setter + def header_table_size(self, value): + self.header_table.maxsize = value + + def decode(self, data, raw=False): + """ + Takes an HPACK-encoded header block and decodes it into a header set. + + :param data: A bytestring representing a complete HPACK-encoded header + block. + :param raw: (optional) Whether to return the headers as tuples of raw + byte strings or to decode them as UTF-8 before returning + them. The default value is False, which returns tuples of + Unicode strings + :returns: A list of two-tuples of ``(name, value)`` representing the + HPACK-encoded headers, in the order they were decoded. + :raises HPACKDecodingError: If an error is encountered while decoding + the header block. + """ + log.debug("Decoding %s", data) + + data_mem = memoryview(data) + headers = [] + data_len = len(data) + inflated_size = 0 + current_index = 0 + + while current_index < data_len: + # Work out what kind of header we're decoding. + # If the high bit is 1, it's an indexed field. + current = data[current_index] + indexed = True if current & 0x80 else False + + # Otherwise, if the second-highest bit is 1 it's a field that does + # alter the header table. + literal_index = True if current & 0x40 else False + + # Otherwise, if the third-highest bit is 1 it's an encoding context + # update. + encoding_update = True if current & 0x20 else False + + if indexed: + header, consumed = self._decode_indexed( + data_mem[current_index:] + ) + elif literal_index: + # It's a literal header that does affect the header table. + header, consumed = self._decode_literal_index( + data_mem[current_index:] + ) + elif encoding_update: + # It's an update to the encoding context. These are forbidden + # in a header block after any actual header. + if headers: + raise HPACKDecodingError( + "Table size update not at the start of the block" + ) + consumed = self._update_encoding_context( + data_mem[current_index:] + ) + header = None + else: + # It's a literal header that does not affect the header table. + header, consumed = self._decode_literal_no_index( + data_mem[current_index:] + ) + + if header: + headers.append(header) + inflated_size += table_entry_size(*header) + + if inflated_size > self.max_header_list_size: + raise OversizedHeaderListError( + "A header list larger than %d has been received" % + self.max_header_list_size + ) + + current_index += consumed + + # Confirm that the table size is lower than the maximum. We do this + # here to ensure that we catch when the max has been *shrunk* and the + # remote peer hasn't actually done that. + self._assert_valid_table_size() + + try: + return [_unicode_if_needed(h, raw) for h in headers] + except UnicodeDecodeError: + raise HPACKDecodingError("Unable to decode headers as UTF-8.") + + def _assert_valid_table_size(self): + """ + Check that the table size set by the encoder is lower than the maximum + we expect to have. + """ + if self.header_table_size > self.max_allowed_table_size: + raise InvalidTableSizeError( + "Encoder did not shrink table size to within the max" + ) + + def _update_encoding_context(self, data): + """ + Handles a byte that updates the encoding context. + """ + # We've been asked to resize the header table. + new_size, consumed = decode_integer(data, 5) + if new_size > self.max_allowed_table_size: + raise InvalidTableSizeError( + "Encoder exceeded max allowable table size" + ) + self.header_table_size = new_size + return consumed + + def _decode_indexed(self, data): + """ + Decodes a header represented using the indexed representation. + """ + index, consumed = decode_integer(data, 7) + header = HeaderTuple(*self.header_table.get_by_index(index)) + log.debug("Decoded %s, consumed %d", header, consumed) + return header, consumed + + def _decode_literal_no_index(self, data): + return self._decode_literal(data, False) + + def _decode_literal_index(self, data): + return self._decode_literal(data, True) + + def _decode_literal(self, data, should_index): + """ + Decodes a header represented with a literal. + """ + total_consumed = 0 + + # When should_index is true, if the low six bits of the first byte are + # nonzero, the header name is indexed. + # When should_index is false, if the low four bits of the first byte + # are nonzero the header name is indexed. + if should_index: + indexed_name = data[0] & 0x3F + name_len = 6 + not_indexable = False + else: + high_byte = data[0] + indexed_name = high_byte & 0x0F + name_len = 4 + not_indexable = high_byte & 0x10 + + if indexed_name: + # Indexed header name. + index, consumed = decode_integer(data, name_len) + name = self.header_table.get_by_index(index)[0] + + total_consumed = consumed + length = 0 + else: + # Literal header name. The first byte was consumed, so we need to + # move forward. + data = data[1:] + + length, consumed = decode_integer(data, 7) + name = data[consumed:consumed + length] + if len(name) != length: + raise HPACKDecodingError("Truncated header block") + + if data[0] & 0x80: + name = decode_huffman(name) + total_consumed = consumed + length + 1 # Since we moved forward 1. + + data = data[consumed + length:] + + # The header value is definitely length-based. + length, consumed = decode_integer(data, 7) + value = data[consumed:consumed + length] + if len(value) != length: + raise HPACKDecodingError("Truncated header block") + + if data[0] & 0x80: + value = decode_huffman(value) + + # Updated the total consumed length. + total_consumed += length + consumed + + # If we have been told never to index the header field, encode that in + # the tuple we use. + if not_indexable: + header = NeverIndexedHeaderTuple(name, value) + else: + header = HeaderTuple(name, value) + + # If we've been asked to index this, add it to the header table. + if should_index: + self.header_table.add(name, value) + + log.debug( + "Decoded %s, total consumed %d bytes, indexed %s", + header, + total_consumed, + should_index + ) + + return header, total_consumed diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/hpack/src/hpack/huffman.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/hpack/src/hpack/huffman.py new file mode 100644 index 0000000000..595d69b6e1 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/hpack/src/hpack/huffman.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +""" +hpack/huffman_decoder +~~~~~~~~~~~~~~~~~~~~~ + +An implementation of a bitwise prefix tree specially built for decoding +Huffman-coded content where we already know the Huffman table. +""" + + +class HuffmanEncoder: + """ + Encodes a string according to the Huffman encoding table defined in the + HPACK specification. + """ + def __init__(self, huffman_code_list, huffman_code_list_lengths): + self.huffman_code_list = huffman_code_list + self.huffman_code_list_lengths = huffman_code_list_lengths + + def encode(self, bytes_to_encode): + """ + Given a string of bytes, encodes them according to the HPACK Huffman + specification. + """ + # If handed the empty string, just immediately return. + if not bytes_to_encode: + return b'' + + final_num = 0 + final_int_len = 0 + + # Turn each byte into its huffman code. These codes aren't necessarily + # octet aligned, so keep track of how far through an octet we are. To + # handle this cleanly, just use a single giant integer. + for byte in bytes_to_encode: + bin_int_len = self.huffman_code_list_lengths[byte] + bin_int = self.huffman_code_list[byte] & ( + 2 ** (bin_int_len + 1) - 1 + ) + final_num <<= bin_int_len + final_num |= bin_int + final_int_len += bin_int_len + + # Pad out to an octet with ones. + bits_to_be_padded = (8 - (final_int_len % 8)) % 8 + final_num <<= bits_to_be_padded + final_num |= (1 << bits_to_be_padded) - 1 + + # Convert the number to hex and strip off the leading '0x' and the + # trailing 'L', if present. + final_num = hex(final_num)[2:].rstrip('L') + + # If this is odd, prepend a zero. + final_num = '0' + final_num if len(final_num) % 2 != 0 else final_num + + # This number should have twice as many digits as bytes. If not, we're + # missing some leading zeroes. Work out how many bytes we want and how + # many digits we have, then add the missing zero digits to the front. + total_bytes = (final_int_len + bits_to_be_padded) // 8 + expected_digits = total_bytes * 2 + + if len(final_num) != expected_digits: + missing_digits = expected_digits - len(final_num) + final_num = ('0' * missing_digits) + final_num + + return bytes.fromhex(final_num) diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/hpack/src/hpack/huffman_constants.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/hpack/src/hpack/huffman_constants.py new file mode 100644 index 0000000000..4caf012d4f --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/hpack/src/hpack/huffman_constants.py @@ -0,0 +1,289 @@ +# -*- coding: utf-8 -*- +""" +hpack/huffman_constants +~~~~~~~~~~~~~~~~~~~~~~~ + +Defines the constant Huffman table. This takes up an upsetting amount of space, +but c'est la vie. +""" +# flake8: noqa + +REQUEST_CODES = [ + 0x1ff8, + 0x7fffd8, + 0xfffffe2, + 0xfffffe3, + 0xfffffe4, + 0xfffffe5, + 0xfffffe6, + 0xfffffe7, + 0xfffffe8, + 0xffffea, + 0x3ffffffc, + 0xfffffe9, + 0xfffffea, + 0x3ffffffd, + 0xfffffeb, + 0xfffffec, + 0xfffffed, + 0xfffffee, + 0xfffffef, + 0xffffff0, + 0xffffff1, + 0xffffff2, + 0x3ffffffe, + 0xffffff3, + 0xffffff4, + 0xffffff5, + 0xffffff6, + 0xffffff7, + 0xffffff8, + 0xffffff9, + 0xffffffa, + 0xffffffb, + 0x14, + 0x3f8, + 0x3f9, + 0xffa, + 0x1ff9, + 0x15, + 0xf8, + 0x7fa, + 0x3fa, + 0x3fb, + 0xf9, + 0x7fb, + 0xfa, + 0x16, + 0x17, + 0x18, + 0x0, + 0x1, + 0x2, + 0x19, + 0x1a, + 0x1b, + 0x1c, + 0x1d, + 0x1e, + 0x1f, + 0x5c, + 0xfb, + 0x7ffc, + 0x20, + 0xffb, + 0x3fc, + 0x1ffa, + 0x21, + 0x5d, + 0x5e, + 0x5f, + 0x60, + 0x61, + 0x62, + 0x63, + 0x64, + 0x65, + 0x66, + 0x67, + 0x68, + 0x69, + 0x6a, + 0x6b, + 0x6c, + 0x6d, + 0x6e, + 0x6f, + 0x70, + 0x71, + 0x72, + 0xfc, + 0x73, + 0xfd, + 0x1ffb, + 0x7fff0, + 0x1ffc, + 0x3ffc, + 0x22, + 0x7ffd, + 0x3, + 0x23, + 0x4, + 0x24, + 0x5, + 0x25, + 0x26, + 0x27, + 0x6, + 0x74, + 0x75, + 0x28, + 0x29, + 0x2a, + 0x7, + 0x2b, + 0x76, + 0x2c, + 0x8, + 0x9, + 0x2d, + 0x77, + 0x78, + 0x79, + 0x7a, + 0x7b, + 0x7ffe, + 0x7fc, + 0x3ffd, + 0x1ffd, + 0xffffffc, + 0xfffe6, + 0x3fffd2, + 0xfffe7, + 0xfffe8, + 0x3fffd3, + 0x3fffd4, + 0x3fffd5, + 0x7fffd9, + 0x3fffd6, + 0x7fffda, + 0x7fffdb, + 0x7fffdc, + 0x7fffdd, + 0x7fffde, + 0xffffeb, + 0x7fffdf, + 0xffffec, + 0xffffed, + 0x3fffd7, + 0x7fffe0, + 0xffffee, + 0x7fffe1, + 0x7fffe2, + 0x7fffe3, + 0x7fffe4, + 0x1fffdc, + 0x3fffd8, + 0x7fffe5, + 0x3fffd9, + 0x7fffe6, + 0x7fffe7, + 0xffffef, + 0x3fffda, + 0x1fffdd, + 0xfffe9, + 0x3fffdb, + 0x3fffdc, + 0x7fffe8, + 0x7fffe9, + 0x1fffde, + 0x7fffea, + 0x3fffdd, + 0x3fffde, + 0xfffff0, + 0x1fffdf, + 0x3fffdf, + 0x7fffeb, + 0x7fffec, + 0x1fffe0, + 0x1fffe1, + 0x3fffe0, + 0x1fffe2, + 0x7fffed, + 0x3fffe1, + 0x7fffee, + 0x7fffef, + 0xfffea, + 0x3fffe2, + 0x3fffe3, + 0x3fffe4, + 0x7ffff0, + 0x3fffe5, + 0x3fffe6, + 0x7ffff1, + 0x3ffffe0, + 0x3ffffe1, + 0xfffeb, + 0x7fff1, + 0x3fffe7, + 0x7ffff2, + 0x3fffe8, + 0x1ffffec, + 0x3ffffe2, + 0x3ffffe3, + 0x3ffffe4, + 0x7ffffde, + 0x7ffffdf, + 0x3ffffe5, + 0xfffff1, + 0x1ffffed, + 0x7fff2, + 0x1fffe3, + 0x3ffffe6, + 0x7ffffe0, + 0x7ffffe1, + 0x3ffffe7, + 0x7ffffe2, + 0xfffff2, + 0x1fffe4, + 0x1fffe5, + 0x3ffffe8, + 0x3ffffe9, + 0xffffffd, + 0x7ffffe3, + 0x7ffffe4, + 0x7ffffe5, + 0xfffec, + 0xfffff3, + 0xfffed, + 0x1fffe6, + 0x3fffe9, + 0x1fffe7, + 0x1fffe8, + 0x7ffff3, + 0x3fffea, + 0x3fffeb, + 0x1ffffee, + 0x1ffffef, + 0xfffff4, + 0xfffff5, + 0x3ffffea, + 0x7ffff4, + 0x3ffffeb, + 0x7ffffe6, + 0x3ffffec, + 0x3ffffed, + 0x7ffffe7, + 0x7ffffe8, + 0x7ffffe9, + 0x7ffffea, + 0x7ffffeb, + 0xffffffe, + 0x7ffffec, + 0x7ffffed, + 0x7ffffee, + 0x7ffffef, + 0x7fffff0, + 0x3ffffee, + 0x3fffffff, +] + +REQUEST_CODES_LENGTH = [ + 13, 23, 28, 28, 28, 28, 28, 28, 28, 24, 30, 28, 28, 30, 28, 28, + 28, 28, 28, 28, 28, 28, 30, 28, 28, 28, 28, 28, 28, 28, 28, 28, + 6, 10, 10, 12, 13, 6, 8, 11, 10, 10, 8, 11, 8, 6, 6, 6, + 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 7, 8, 15, 6, 12, 10, + 13, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 8, 7, 8, 13, 19, 13, 14, 6, + 15, 5, 6, 5, 6, 5, 6, 6, 6, 5, 7, 7, 6, 6, 6, 5, + 6, 7, 6, 5, 5, 6, 7, 7, 7, 7, 7, 15, 11, 14, 13, 28, + 20, 22, 20, 20, 22, 22, 22, 23, 22, 23, 23, 23, 23, 23, 24, 23, + 24, 24, 22, 23, 24, 23, 23, 23, 23, 21, 22, 23, 22, 23, 23, 24, + 22, 21, 20, 22, 22, 23, 23, 21, 23, 22, 22, 24, 21, 22, 23, 23, + 21, 21, 22, 21, 23, 22, 23, 23, 20, 22, 22, 22, 23, 22, 22, 23, + 26, 26, 20, 19, 22, 23, 22, 25, 26, 26, 26, 27, 27, 26, 24, 25, + 19, 21, 26, 27, 27, 26, 27, 24, 21, 21, 26, 26, 28, 27, 27, 27, + 20, 24, 20, 21, 22, 21, 21, 23, 22, 22, 25, 25, 24, 24, 26, 23, + 26, 27, 26, 26, 27, 27, 27, 27, 27, 28, 27, 27, 27, 27, 27, 26, + 30, +] diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/hpack/src/hpack/huffman_table.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/hpack/src/hpack/huffman_table.py new file mode 100644 index 0000000000..c199ef5a3f --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/hpack/src/hpack/huffman_table.py @@ -0,0 +1,4739 @@ +# -*- coding: utf-8 -*- +""" +hpack/huffman_table +~~~~~~~~~~~~~~~~~~~ + +This implementation of a Huffman decoding table for HTTP/2 is essentially a +Python port of the work originally done for nghttp2's Huffman decoding. For +this reason, while this file is made available under the MIT license as is the +rest of this module, this file is undoubtedly a derivative work of the nghttp2 +file ``nghttp2_hd_huffman_data.c``, obtained from +https://github.com/tatsuhiro-t/nghttp2/ at commit +d2b55ad1a245e1d1964579fa3fac36ebf3939e72. That work is made available under +the Apache 2.0 license under the following terms: + + Copyright (c) 2013 Tatsuhiro Tsujikawa + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +The essence of this approach is that it builds a finite state machine out of +4-bit nibbles of Huffman coded data. The input function passes 4 bits worth of +data to the state machine each time, which uses those 4 bits of data along with +the current accumulated state data to process the data given. + +For the sake of efficiency, the in-memory representation of the states, +transitions, and result values of the state machine are represented as a long +list containing three-tuples. This list is enormously long, and viewing it as +an in-memory representation is not very clear, but it is laid out here in a way +that is intended to be *somewhat* more clear. + +Essentially, the list is structured as 256 collections of 16 entries (one for +each nibble) of three-tuples. Each collection is called a "node", and the +zeroth collection is called the "root node". The state machine tracks one +value: the "state" byte. + +For each nibble passed to the state machine, it first multiplies the "state" +byte by 16 and adds the numerical value of the nibble. This number is the index +into the large flat list. + +The three-tuple that is found by looking up that index consists of three +values: + +- a new state value, used for subsequent decoding +- a collection of flags, used to determine whether data is emitted or whether + the state machine is complete. +- the byte value to emit, assuming that emitting a byte is required. + +The flags are consulted, if necessary a byte is emitted, and then the next +nibble is used. This continues until the state machine believes it has +completely Huffman-decoded the data. + +This approach has relatively little indirection, and therefore performs +relatively well, particularly on implementations like PyPy where the cost of +loops at the Python-level is not too expensive. The total number of loop +iterations is 4x the number of bytes passed to the decoder. +""" +from .exceptions import HPACKDecodingError + + +# This defines the state machine "class" at the top of the file. The reason we +# do this is to keep the terrifing monster state table at the *bottom* of the +# file so you don't have to actually *look* at the damn thing. +def decode_huffman(huffman_string): + """ + Given a bytestring of Huffman-encoded data for HPACK, returns a bytestring + of the decompressed data. + """ + if not huffman_string: + return b'' + + state = 0 + flags = 0 + decoded_bytes = bytearray() + + # Perversely, bytearrays are a lot more convenient across Python 2 and + # Python 3 because they behave *the same way* on both platforms. Given that + # we really do want numerical bytes when we iterate here, let's use a + # bytearray. + huffman_string = bytearray(huffman_string) + + # This loop is unrolled somewhat. Because we use a nibble, not a byte, we + # need to handle each nibble twice. We unroll that: it makes the loop body + # a bit longer, but that's ok. + for input_byte in huffman_string: + index = (state * 16) + (input_byte >> 4) + state, flags, output_byte = HUFFMAN_TABLE[index] + + if flags & HUFFMAN_FAIL: + raise HPACKDecodingError("Invalid Huffman String") + + if flags & HUFFMAN_EMIT_SYMBOL: + decoded_bytes.append(output_byte) + + index = (state * 16) + (input_byte & 0x0F) + state, flags, output_byte = HUFFMAN_TABLE[index] + + if flags & HUFFMAN_FAIL: + raise HPACKDecodingError("Invalid Huffman String") + + if flags & HUFFMAN_EMIT_SYMBOL: + decoded_bytes.append(output_byte) + + if not (flags & HUFFMAN_COMPLETE): + raise HPACKDecodingError("Incomplete Huffman string") + + return bytes(decoded_bytes) + + +# Some decoder flags to control state transitions. +HUFFMAN_COMPLETE = 1 +HUFFMAN_EMIT_SYMBOL = (1 << 1) +HUFFMAN_FAIL = (1 << 2) + +# This is the monster table. Avert your eyes, children. +HUFFMAN_TABLE = [ + # Node 0 (Root Node, never emits symbols.) + (4, 0, 0), + (5, 0, 0), + (7, 0, 0), + (8, 0, 0), + (11, 0, 0), + (12, 0, 0), + (16, 0, 0), + (19, 0, 0), + (25, 0, 0), + (28, 0, 0), + (32, 0, 0), + (35, 0, 0), + (42, 0, 0), + (49, 0, 0), + (57, 0, 0), + (64, HUFFMAN_COMPLETE, 0), + + # Node 1 + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 48), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 49), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 50), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 97), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 99), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 101), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 105), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 111), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 115), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 116), + (13, 0, 0), + (14, 0, 0), + (17, 0, 0), + (18, 0, 0), + (20, 0, 0), + (21, 0, 0), + + # Node 2 + (1, HUFFMAN_EMIT_SYMBOL, 48), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 48), + (1, HUFFMAN_EMIT_SYMBOL, 49), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 49), + (1, HUFFMAN_EMIT_SYMBOL, 50), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 50), + (1, HUFFMAN_EMIT_SYMBOL, 97), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 97), + (1, HUFFMAN_EMIT_SYMBOL, 99), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 99), + (1, HUFFMAN_EMIT_SYMBOL, 101), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 101), + (1, HUFFMAN_EMIT_SYMBOL, 105), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 105), + (1, HUFFMAN_EMIT_SYMBOL, 111), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 111), + + # Node 3 + (2, HUFFMAN_EMIT_SYMBOL, 48), + (9, HUFFMAN_EMIT_SYMBOL, 48), + (23, HUFFMAN_EMIT_SYMBOL, 48), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 48), + (2, HUFFMAN_EMIT_SYMBOL, 49), + (9, HUFFMAN_EMIT_SYMBOL, 49), + (23, HUFFMAN_EMIT_SYMBOL, 49), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 49), + (2, HUFFMAN_EMIT_SYMBOL, 50), + (9, HUFFMAN_EMIT_SYMBOL, 50), + (23, HUFFMAN_EMIT_SYMBOL, 50), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 50), + (2, HUFFMAN_EMIT_SYMBOL, 97), + (9, HUFFMAN_EMIT_SYMBOL, 97), + (23, HUFFMAN_EMIT_SYMBOL, 97), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 97), + + # Node 4 + (3, HUFFMAN_EMIT_SYMBOL, 48), + (6, HUFFMAN_EMIT_SYMBOL, 48), + (10, HUFFMAN_EMIT_SYMBOL, 48), + (15, HUFFMAN_EMIT_SYMBOL, 48), + (24, HUFFMAN_EMIT_SYMBOL, 48), + (31, HUFFMAN_EMIT_SYMBOL, 48), + (41, HUFFMAN_EMIT_SYMBOL, 48), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 48), + (3, HUFFMAN_EMIT_SYMBOL, 49), + (6, HUFFMAN_EMIT_SYMBOL, 49), + (10, HUFFMAN_EMIT_SYMBOL, 49), + (15, HUFFMAN_EMIT_SYMBOL, 49), + (24, HUFFMAN_EMIT_SYMBOL, 49), + (31, HUFFMAN_EMIT_SYMBOL, 49), + (41, HUFFMAN_EMIT_SYMBOL, 49), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 49), + + # Node 5 + (3, HUFFMAN_EMIT_SYMBOL, 50), + (6, HUFFMAN_EMIT_SYMBOL, 50), + (10, HUFFMAN_EMIT_SYMBOL, 50), + (15, HUFFMAN_EMIT_SYMBOL, 50), + (24, HUFFMAN_EMIT_SYMBOL, 50), + (31, HUFFMAN_EMIT_SYMBOL, 50), + (41, HUFFMAN_EMIT_SYMBOL, 50), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 50), + (3, HUFFMAN_EMIT_SYMBOL, 97), + (6, HUFFMAN_EMIT_SYMBOL, 97), + (10, HUFFMAN_EMIT_SYMBOL, 97), + (15, HUFFMAN_EMIT_SYMBOL, 97), + (24, HUFFMAN_EMIT_SYMBOL, 97), + (31, HUFFMAN_EMIT_SYMBOL, 97), + (41, HUFFMAN_EMIT_SYMBOL, 97), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 97), + + # Node 6 + (2, HUFFMAN_EMIT_SYMBOL, 99), + (9, HUFFMAN_EMIT_SYMBOL, 99), + (23, HUFFMAN_EMIT_SYMBOL, 99), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 99), + (2, HUFFMAN_EMIT_SYMBOL, 101), + (9, HUFFMAN_EMIT_SYMBOL, 101), + (23, HUFFMAN_EMIT_SYMBOL, 101), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 101), + (2, HUFFMAN_EMIT_SYMBOL, 105), + (9, HUFFMAN_EMIT_SYMBOL, 105), + (23, HUFFMAN_EMIT_SYMBOL, 105), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 105), + (2, HUFFMAN_EMIT_SYMBOL, 111), + (9, HUFFMAN_EMIT_SYMBOL, 111), + (23, HUFFMAN_EMIT_SYMBOL, 111), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 111), + + # Node 7 + (3, HUFFMAN_EMIT_SYMBOL, 99), + (6, HUFFMAN_EMIT_SYMBOL, 99), + (10, HUFFMAN_EMIT_SYMBOL, 99), + (15, HUFFMAN_EMIT_SYMBOL, 99), + (24, HUFFMAN_EMIT_SYMBOL, 99), + (31, HUFFMAN_EMIT_SYMBOL, 99), + (41, HUFFMAN_EMIT_SYMBOL, 99), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 99), + (3, HUFFMAN_EMIT_SYMBOL, 101), + (6, HUFFMAN_EMIT_SYMBOL, 101), + (10, HUFFMAN_EMIT_SYMBOL, 101), + (15, HUFFMAN_EMIT_SYMBOL, 101), + (24, HUFFMAN_EMIT_SYMBOL, 101), + (31, HUFFMAN_EMIT_SYMBOL, 101), + (41, HUFFMAN_EMIT_SYMBOL, 101), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 101), + + # Node 8 + (3, HUFFMAN_EMIT_SYMBOL, 105), + (6, HUFFMAN_EMIT_SYMBOL, 105), + (10, HUFFMAN_EMIT_SYMBOL, 105), + (15, HUFFMAN_EMIT_SYMBOL, 105), + (24, HUFFMAN_EMIT_SYMBOL, 105), + (31, HUFFMAN_EMIT_SYMBOL, 105), + (41, HUFFMAN_EMIT_SYMBOL, 105), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 105), + (3, HUFFMAN_EMIT_SYMBOL, 111), + (6, HUFFMAN_EMIT_SYMBOL, 111), + (10, HUFFMAN_EMIT_SYMBOL, 111), + (15, HUFFMAN_EMIT_SYMBOL, 111), + (24, HUFFMAN_EMIT_SYMBOL, 111), + (31, HUFFMAN_EMIT_SYMBOL, 111), + (41, HUFFMAN_EMIT_SYMBOL, 111), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 111), + + # Node 9 + (1, HUFFMAN_EMIT_SYMBOL, 115), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 115), + (1, HUFFMAN_EMIT_SYMBOL, 116), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 116), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 32), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 37), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 45), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 46), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 47), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 51), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 52), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 53), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 54), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 55), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 56), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 57), + + # Node 10 + (2, HUFFMAN_EMIT_SYMBOL, 115), + (9, HUFFMAN_EMIT_SYMBOL, 115), + (23, HUFFMAN_EMIT_SYMBOL, 115), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 115), + (2, HUFFMAN_EMIT_SYMBOL, 116), + (9, HUFFMAN_EMIT_SYMBOL, 116), + (23, HUFFMAN_EMIT_SYMBOL, 116), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 116), + (1, HUFFMAN_EMIT_SYMBOL, 32), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 32), + (1, HUFFMAN_EMIT_SYMBOL, 37), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 37), + (1, HUFFMAN_EMIT_SYMBOL, 45), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 45), + (1, HUFFMAN_EMIT_SYMBOL, 46), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 46), + + # Node 11 + (3, HUFFMAN_EMIT_SYMBOL, 115), + (6, HUFFMAN_EMIT_SYMBOL, 115), + (10, HUFFMAN_EMIT_SYMBOL, 115), + (15, HUFFMAN_EMIT_SYMBOL, 115), + (24, HUFFMAN_EMIT_SYMBOL, 115), + (31, HUFFMAN_EMIT_SYMBOL, 115), + (41, HUFFMAN_EMIT_SYMBOL, 115), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 115), + (3, HUFFMAN_EMIT_SYMBOL, 116), + (6, HUFFMAN_EMIT_SYMBOL, 116), + (10, HUFFMAN_EMIT_SYMBOL, 116), + (15, HUFFMAN_EMIT_SYMBOL, 116), + (24, HUFFMAN_EMIT_SYMBOL, 116), + (31, HUFFMAN_EMIT_SYMBOL, 116), + (41, HUFFMAN_EMIT_SYMBOL, 116), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 116), + + # Node 12 + (2, HUFFMAN_EMIT_SYMBOL, 32), + (9, HUFFMAN_EMIT_SYMBOL, 32), + (23, HUFFMAN_EMIT_SYMBOL, 32), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 32), + (2, HUFFMAN_EMIT_SYMBOL, 37), + (9, HUFFMAN_EMIT_SYMBOL, 37), + (23, HUFFMAN_EMIT_SYMBOL, 37), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 37), + (2, HUFFMAN_EMIT_SYMBOL, 45), + (9, HUFFMAN_EMIT_SYMBOL, 45), + (23, HUFFMAN_EMIT_SYMBOL, 45), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 45), + (2, HUFFMAN_EMIT_SYMBOL, 46), + (9, HUFFMAN_EMIT_SYMBOL, 46), + (23, HUFFMAN_EMIT_SYMBOL, 46), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 46), + + # Node 13 + (3, HUFFMAN_EMIT_SYMBOL, 32), + (6, HUFFMAN_EMIT_SYMBOL, 32), + (10, HUFFMAN_EMIT_SYMBOL, 32), + (15, HUFFMAN_EMIT_SYMBOL, 32), + (24, HUFFMAN_EMIT_SYMBOL, 32), + (31, HUFFMAN_EMIT_SYMBOL, 32), + (41, HUFFMAN_EMIT_SYMBOL, 32), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 32), + (3, HUFFMAN_EMIT_SYMBOL, 37), + (6, HUFFMAN_EMIT_SYMBOL, 37), + (10, HUFFMAN_EMIT_SYMBOL, 37), + (15, HUFFMAN_EMIT_SYMBOL, 37), + (24, HUFFMAN_EMIT_SYMBOL, 37), + (31, HUFFMAN_EMIT_SYMBOL, 37), + (41, HUFFMAN_EMIT_SYMBOL, 37), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 37), + + # Node 14 + (3, HUFFMAN_EMIT_SYMBOL, 45), + (6, HUFFMAN_EMIT_SYMBOL, 45), + (10, HUFFMAN_EMIT_SYMBOL, 45), + (15, HUFFMAN_EMIT_SYMBOL, 45), + (24, HUFFMAN_EMIT_SYMBOL, 45), + (31, HUFFMAN_EMIT_SYMBOL, 45), + (41, HUFFMAN_EMIT_SYMBOL, 45), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 45), + (3, HUFFMAN_EMIT_SYMBOL, 46), + (6, HUFFMAN_EMIT_SYMBOL, 46), + (10, HUFFMAN_EMIT_SYMBOL, 46), + (15, HUFFMAN_EMIT_SYMBOL, 46), + (24, HUFFMAN_EMIT_SYMBOL, 46), + (31, HUFFMAN_EMIT_SYMBOL, 46), + (41, HUFFMAN_EMIT_SYMBOL, 46), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 46), + + # Node 15 + (1, HUFFMAN_EMIT_SYMBOL, 47), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 47), + (1, HUFFMAN_EMIT_SYMBOL, 51), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 51), + (1, HUFFMAN_EMIT_SYMBOL, 52), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 52), + (1, HUFFMAN_EMIT_SYMBOL, 53), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 53), + (1, HUFFMAN_EMIT_SYMBOL, 54), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 54), + (1, HUFFMAN_EMIT_SYMBOL, 55), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 55), + (1, HUFFMAN_EMIT_SYMBOL, 56), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 56), + (1, HUFFMAN_EMIT_SYMBOL, 57), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 57), + + # Node 16 + (2, HUFFMAN_EMIT_SYMBOL, 47), + (9, HUFFMAN_EMIT_SYMBOL, 47), + (23, HUFFMAN_EMIT_SYMBOL, 47), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 47), + (2, HUFFMAN_EMIT_SYMBOL, 51), + (9, HUFFMAN_EMIT_SYMBOL, 51), + (23, HUFFMAN_EMIT_SYMBOL, 51), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 51), + (2, HUFFMAN_EMIT_SYMBOL, 52), + (9, HUFFMAN_EMIT_SYMBOL, 52), + (23, HUFFMAN_EMIT_SYMBOL, 52), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 52), + (2, HUFFMAN_EMIT_SYMBOL, 53), + (9, HUFFMAN_EMIT_SYMBOL, 53), + (23, HUFFMAN_EMIT_SYMBOL, 53), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 53), + + # Node 17 + (3, HUFFMAN_EMIT_SYMBOL, 47), + (6, HUFFMAN_EMIT_SYMBOL, 47), + (10, HUFFMAN_EMIT_SYMBOL, 47), + (15, HUFFMAN_EMIT_SYMBOL, 47), + (24, HUFFMAN_EMIT_SYMBOL, 47), + (31, HUFFMAN_EMIT_SYMBOL, 47), + (41, HUFFMAN_EMIT_SYMBOL, 47), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 47), + (3, HUFFMAN_EMIT_SYMBOL, 51), + (6, HUFFMAN_EMIT_SYMBOL, 51), + (10, HUFFMAN_EMIT_SYMBOL, 51), + (15, HUFFMAN_EMIT_SYMBOL, 51), + (24, HUFFMAN_EMIT_SYMBOL, 51), + (31, HUFFMAN_EMIT_SYMBOL, 51), + (41, HUFFMAN_EMIT_SYMBOL, 51), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 51), + + # Node 18 + (3, HUFFMAN_EMIT_SYMBOL, 52), + (6, HUFFMAN_EMIT_SYMBOL, 52), + (10, HUFFMAN_EMIT_SYMBOL, 52), + (15, HUFFMAN_EMIT_SYMBOL, 52), + (24, HUFFMAN_EMIT_SYMBOL, 52), + (31, HUFFMAN_EMIT_SYMBOL, 52), + (41, HUFFMAN_EMIT_SYMBOL, 52), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 52), + (3, HUFFMAN_EMIT_SYMBOL, 53), + (6, HUFFMAN_EMIT_SYMBOL, 53), + (10, HUFFMAN_EMIT_SYMBOL, 53), + (15, HUFFMAN_EMIT_SYMBOL, 53), + (24, HUFFMAN_EMIT_SYMBOL, 53), + (31, HUFFMAN_EMIT_SYMBOL, 53), + (41, HUFFMAN_EMIT_SYMBOL, 53), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 53), + + # Node 19 + (2, HUFFMAN_EMIT_SYMBOL, 54), + (9, HUFFMAN_EMIT_SYMBOL, 54), + (23, HUFFMAN_EMIT_SYMBOL, 54), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 54), + (2, HUFFMAN_EMIT_SYMBOL, 55), + (9, HUFFMAN_EMIT_SYMBOL, 55), + (23, HUFFMAN_EMIT_SYMBOL, 55), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 55), + (2, HUFFMAN_EMIT_SYMBOL, 56), + (9, HUFFMAN_EMIT_SYMBOL, 56), + (23, HUFFMAN_EMIT_SYMBOL, 56), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 56), + (2, HUFFMAN_EMIT_SYMBOL, 57), + (9, HUFFMAN_EMIT_SYMBOL, 57), + (23, HUFFMAN_EMIT_SYMBOL, 57), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 57), + + # Node 20 + (3, HUFFMAN_EMIT_SYMBOL, 54), + (6, HUFFMAN_EMIT_SYMBOL, 54), + (10, HUFFMAN_EMIT_SYMBOL, 54), + (15, HUFFMAN_EMIT_SYMBOL, 54), + (24, HUFFMAN_EMIT_SYMBOL, 54), + (31, HUFFMAN_EMIT_SYMBOL, 54), + (41, HUFFMAN_EMIT_SYMBOL, 54), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 54), + (3, HUFFMAN_EMIT_SYMBOL, 55), + (6, HUFFMAN_EMIT_SYMBOL, 55), + (10, HUFFMAN_EMIT_SYMBOL, 55), + (15, HUFFMAN_EMIT_SYMBOL, 55), + (24, HUFFMAN_EMIT_SYMBOL, 55), + (31, HUFFMAN_EMIT_SYMBOL, 55), + (41, HUFFMAN_EMIT_SYMBOL, 55), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 55), + + # Node 21 + (3, HUFFMAN_EMIT_SYMBOL, 56), + (6, HUFFMAN_EMIT_SYMBOL, 56), + (10, HUFFMAN_EMIT_SYMBOL, 56), + (15, HUFFMAN_EMIT_SYMBOL, 56), + (24, HUFFMAN_EMIT_SYMBOL, 56), + (31, HUFFMAN_EMIT_SYMBOL, 56), + (41, HUFFMAN_EMIT_SYMBOL, 56), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 56), + (3, HUFFMAN_EMIT_SYMBOL, 57), + (6, HUFFMAN_EMIT_SYMBOL, 57), + (10, HUFFMAN_EMIT_SYMBOL, 57), + (15, HUFFMAN_EMIT_SYMBOL, 57), + (24, HUFFMAN_EMIT_SYMBOL, 57), + (31, HUFFMAN_EMIT_SYMBOL, 57), + (41, HUFFMAN_EMIT_SYMBOL, 57), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 57), + + # Node 22 + (26, 0, 0), + (27, 0, 0), + (29, 0, 0), + (30, 0, 0), + (33, 0, 0), + (34, 0, 0), + (36, 0, 0), + (37, 0, 0), + (43, 0, 0), + (46, 0, 0), + (50, 0, 0), + (53, 0, 0), + (58, 0, 0), + (61, 0, 0), + (65, 0, 0), + (68, HUFFMAN_COMPLETE, 0), + + # Node 23 + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 61), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 65), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 95), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 98), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 100), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 102), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 103), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 104), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 108), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 109), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 110), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 112), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 114), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 117), + (38, 0, 0), + (39, 0, 0), + + # Node 24 + (1, HUFFMAN_EMIT_SYMBOL, 61), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 61), + (1, HUFFMAN_EMIT_SYMBOL, 65), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 65), + (1, HUFFMAN_EMIT_SYMBOL, 95), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 95), + (1, HUFFMAN_EMIT_SYMBOL, 98), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 98), + (1, HUFFMAN_EMIT_SYMBOL, 100), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 100), + (1, HUFFMAN_EMIT_SYMBOL, 102), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 102), + (1, HUFFMAN_EMIT_SYMBOL, 103), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 103), + (1, HUFFMAN_EMIT_SYMBOL, 104), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 104), + + # Node 25 + (2, HUFFMAN_EMIT_SYMBOL, 61), + (9, HUFFMAN_EMIT_SYMBOL, 61), + (23, HUFFMAN_EMIT_SYMBOL, 61), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 61), + (2, HUFFMAN_EMIT_SYMBOL, 65), + (9, HUFFMAN_EMIT_SYMBOL, 65), + (23, HUFFMAN_EMIT_SYMBOL, 65), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 65), + (2, HUFFMAN_EMIT_SYMBOL, 95), + (9, HUFFMAN_EMIT_SYMBOL, 95), + (23, HUFFMAN_EMIT_SYMBOL, 95), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 95), + (2, HUFFMAN_EMIT_SYMBOL, 98), + (9, HUFFMAN_EMIT_SYMBOL, 98), + (23, HUFFMAN_EMIT_SYMBOL, 98), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 98), + + # Node 26 + (3, HUFFMAN_EMIT_SYMBOL, 61), + (6, HUFFMAN_EMIT_SYMBOL, 61), + (10, HUFFMAN_EMIT_SYMBOL, 61), + (15, HUFFMAN_EMIT_SYMBOL, 61), + (24, HUFFMAN_EMIT_SYMBOL, 61), + (31, HUFFMAN_EMIT_SYMBOL, 61), + (41, HUFFMAN_EMIT_SYMBOL, 61), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 61), + (3, HUFFMAN_EMIT_SYMBOL, 65), + (6, HUFFMAN_EMIT_SYMBOL, 65), + (10, HUFFMAN_EMIT_SYMBOL, 65), + (15, HUFFMAN_EMIT_SYMBOL, 65), + (24, HUFFMAN_EMIT_SYMBOL, 65), + (31, HUFFMAN_EMIT_SYMBOL, 65), + (41, HUFFMAN_EMIT_SYMBOL, 65), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 65), + + # Node 27 + (3, HUFFMAN_EMIT_SYMBOL, 95), + (6, HUFFMAN_EMIT_SYMBOL, 95), + (10, HUFFMAN_EMIT_SYMBOL, 95), + (15, HUFFMAN_EMIT_SYMBOL, 95), + (24, HUFFMAN_EMIT_SYMBOL, 95), + (31, HUFFMAN_EMIT_SYMBOL, 95), + (41, HUFFMAN_EMIT_SYMBOL, 95), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 95), + (3, HUFFMAN_EMIT_SYMBOL, 98), + (6, HUFFMAN_EMIT_SYMBOL, 98), + (10, HUFFMAN_EMIT_SYMBOL, 98), + (15, HUFFMAN_EMIT_SYMBOL, 98), + (24, HUFFMAN_EMIT_SYMBOL, 98), + (31, HUFFMAN_EMIT_SYMBOL, 98), + (41, HUFFMAN_EMIT_SYMBOL, 98), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 98), + + # Node 28 + (2, HUFFMAN_EMIT_SYMBOL, 100), + (9, HUFFMAN_EMIT_SYMBOL, 100), + (23, HUFFMAN_EMIT_SYMBOL, 100), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 100), + (2, HUFFMAN_EMIT_SYMBOL, 102), + (9, HUFFMAN_EMIT_SYMBOL, 102), + (23, HUFFMAN_EMIT_SYMBOL, 102), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 102), + (2, HUFFMAN_EMIT_SYMBOL, 103), + (9, HUFFMAN_EMIT_SYMBOL, 103), + (23, HUFFMAN_EMIT_SYMBOL, 103), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 103), + (2, HUFFMAN_EMIT_SYMBOL, 104), + (9, HUFFMAN_EMIT_SYMBOL, 104), + (23, HUFFMAN_EMIT_SYMBOL, 104), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 104), + + # Node 29 + (3, HUFFMAN_EMIT_SYMBOL, 100), + (6, HUFFMAN_EMIT_SYMBOL, 100), + (10, HUFFMAN_EMIT_SYMBOL, 100), + (15, HUFFMAN_EMIT_SYMBOL, 100), + (24, HUFFMAN_EMIT_SYMBOL, 100), + (31, HUFFMAN_EMIT_SYMBOL, 100), + (41, HUFFMAN_EMIT_SYMBOL, 100), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 100), + (3, HUFFMAN_EMIT_SYMBOL, 102), + (6, HUFFMAN_EMIT_SYMBOL, 102), + (10, HUFFMAN_EMIT_SYMBOL, 102), + (15, HUFFMAN_EMIT_SYMBOL, 102), + (24, HUFFMAN_EMIT_SYMBOL, 102), + (31, HUFFMAN_EMIT_SYMBOL, 102), + (41, HUFFMAN_EMIT_SYMBOL, 102), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 102), + + # Node 30 + (3, HUFFMAN_EMIT_SYMBOL, 103), + (6, HUFFMAN_EMIT_SYMBOL, 103), + (10, HUFFMAN_EMIT_SYMBOL, 103), + (15, HUFFMAN_EMIT_SYMBOL, 103), + (24, HUFFMAN_EMIT_SYMBOL, 103), + (31, HUFFMAN_EMIT_SYMBOL, 103), + (41, HUFFMAN_EMIT_SYMBOL, 103), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 103), + (3, HUFFMAN_EMIT_SYMBOL, 104), + (6, HUFFMAN_EMIT_SYMBOL, 104), + (10, HUFFMAN_EMIT_SYMBOL, 104), + (15, HUFFMAN_EMIT_SYMBOL, 104), + (24, HUFFMAN_EMIT_SYMBOL, 104), + (31, HUFFMAN_EMIT_SYMBOL, 104), + (41, HUFFMAN_EMIT_SYMBOL, 104), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 104), + + # Node 31 + (1, HUFFMAN_EMIT_SYMBOL, 108), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 108), + (1, HUFFMAN_EMIT_SYMBOL, 109), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 109), + (1, HUFFMAN_EMIT_SYMBOL, 110), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 110), + (1, HUFFMAN_EMIT_SYMBOL, 112), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 112), + (1, HUFFMAN_EMIT_SYMBOL, 114), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 114), + (1, HUFFMAN_EMIT_SYMBOL, 117), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 117), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 58), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 66), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 67), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 68), + + # Node 32 + (2, HUFFMAN_EMIT_SYMBOL, 108), + (9, HUFFMAN_EMIT_SYMBOL, 108), + (23, HUFFMAN_EMIT_SYMBOL, 108), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 108), + (2, HUFFMAN_EMIT_SYMBOL, 109), + (9, HUFFMAN_EMIT_SYMBOL, 109), + (23, HUFFMAN_EMIT_SYMBOL, 109), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 109), + (2, HUFFMAN_EMIT_SYMBOL, 110), + (9, HUFFMAN_EMIT_SYMBOL, 110), + (23, HUFFMAN_EMIT_SYMBOL, 110), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 110), + (2, HUFFMAN_EMIT_SYMBOL, 112), + (9, HUFFMAN_EMIT_SYMBOL, 112), + (23, HUFFMAN_EMIT_SYMBOL, 112), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 112), + + # Node 33 + (3, HUFFMAN_EMIT_SYMBOL, 108), + (6, HUFFMAN_EMIT_SYMBOL, 108), + (10, HUFFMAN_EMIT_SYMBOL, 108), + (15, HUFFMAN_EMIT_SYMBOL, 108), + (24, HUFFMAN_EMIT_SYMBOL, 108), + (31, HUFFMAN_EMIT_SYMBOL, 108), + (41, HUFFMAN_EMIT_SYMBOL, 108), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 108), + (3, HUFFMAN_EMIT_SYMBOL, 109), + (6, HUFFMAN_EMIT_SYMBOL, 109), + (10, HUFFMAN_EMIT_SYMBOL, 109), + (15, HUFFMAN_EMIT_SYMBOL, 109), + (24, HUFFMAN_EMIT_SYMBOL, 109), + (31, HUFFMAN_EMIT_SYMBOL, 109), + (41, HUFFMAN_EMIT_SYMBOL, 109), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 109), + + # Node 34 + (3, HUFFMAN_EMIT_SYMBOL, 110), + (6, HUFFMAN_EMIT_SYMBOL, 110), + (10, HUFFMAN_EMIT_SYMBOL, 110), + (15, HUFFMAN_EMIT_SYMBOL, 110), + (24, HUFFMAN_EMIT_SYMBOL, 110), + (31, HUFFMAN_EMIT_SYMBOL, 110), + (41, HUFFMAN_EMIT_SYMBOL, 110), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 110), + (3, HUFFMAN_EMIT_SYMBOL, 112), + (6, HUFFMAN_EMIT_SYMBOL, 112), + (10, HUFFMAN_EMIT_SYMBOL, 112), + (15, HUFFMAN_EMIT_SYMBOL, 112), + (24, HUFFMAN_EMIT_SYMBOL, 112), + (31, HUFFMAN_EMIT_SYMBOL, 112), + (41, HUFFMAN_EMIT_SYMBOL, 112), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 112), + + # Node 35 + (2, HUFFMAN_EMIT_SYMBOL, 114), + (9, HUFFMAN_EMIT_SYMBOL, 114), + (23, HUFFMAN_EMIT_SYMBOL, 114), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 114), + (2, HUFFMAN_EMIT_SYMBOL, 117), + (9, HUFFMAN_EMIT_SYMBOL, 117), + (23, HUFFMAN_EMIT_SYMBOL, 117), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 117), + (1, HUFFMAN_EMIT_SYMBOL, 58), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 58), + (1, HUFFMAN_EMIT_SYMBOL, 66), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 66), + (1, HUFFMAN_EMIT_SYMBOL, 67), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 67), + (1, HUFFMAN_EMIT_SYMBOL, 68), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 68), + + # Node 36 + (3, HUFFMAN_EMIT_SYMBOL, 114), + (6, HUFFMAN_EMIT_SYMBOL, 114), + (10, HUFFMAN_EMIT_SYMBOL, 114), + (15, HUFFMAN_EMIT_SYMBOL, 114), + (24, HUFFMAN_EMIT_SYMBOL, 114), + (31, HUFFMAN_EMIT_SYMBOL, 114), + (41, HUFFMAN_EMIT_SYMBOL, 114), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 114), + (3, HUFFMAN_EMIT_SYMBOL, 117), + (6, HUFFMAN_EMIT_SYMBOL, 117), + (10, HUFFMAN_EMIT_SYMBOL, 117), + (15, HUFFMAN_EMIT_SYMBOL, 117), + (24, HUFFMAN_EMIT_SYMBOL, 117), + (31, HUFFMAN_EMIT_SYMBOL, 117), + (41, HUFFMAN_EMIT_SYMBOL, 117), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 117), + + # Node 37 + (2, HUFFMAN_EMIT_SYMBOL, 58), + (9, HUFFMAN_EMIT_SYMBOL, 58), + (23, HUFFMAN_EMIT_SYMBOL, 58), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 58), + (2, HUFFMAN_EMIT_SYMBOL, 66), + (9, HUFFMAN_EMIT_SYMBOL, 66), + (23, HUFFMAN_EMIT_SYMBOL, 66), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 66), + (2, HUFFMAN_EMIT_SYMBOL, 67), + (9, HUFFMAN_EMIT_SYMBOL, 67), + (23, HUFFMAN_EMIT_SYMBOL, 67), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 67), + (2, HUFFMAN_EMIT_SYMBOL, 68), + (9, HUFFMAN_EMIT_SYMBOL, 68), + (23, HUFFMAN_EMIT_SYMBOL, 68), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 68), + + # Node 38 + (3, HUFFMAN_EMIT_SYMBOL, 58), + (6, HUFFMAN_EMIT_SYMBOL, 58), + (10, HUFFMAN_EMIT_SYMBOL, 58), + (15, HUFFMAN_EMIT_SYMBOL, 58), + (24, HUFFMAN_EMIT_SYMBOL, 58), + (31, HUFFMAN_EMIT_SYMBOL, 58), + (41, HUFFMAN_EMIT_SYMBOL, 58), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 58), + (3, HUFFMAN_EMIT_SYMBOL, 66), + (6, HUFFMAN_EMIT_SYMBOL, 66), + (10, HUFFMAN_EMIT_SYMBOL, 66), + (15, HUFFMAN_EMIT_SYMBOL, 66), + (24, HUFFMAN_EMIT_SYMBOL, 66), + (31, HUFFMAN_EMIT_SYMBOL, 66), + (41, HUFFMAN_EMIT_SYMBOL, 66), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 66), + + # Node 39 + (3, HUFFMAN_EMIT_SYMBOL, 67), + (6, HUFFMAN_EMIT_SYMBOL, 67), + (10, HUFFMAN_EMIT_SYMBOL, 67), + (15, HUFFMAN_EMIT_SYMBOL, 67), + (24, HUFFMAN_EMIT_SYMBOL, 67), + (31, HUFFMAN_EMIT_SYMBOL, 67), + (41, HUFFMAN_EMIT_SYMBOL, 67), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 67), + (3, HUFFMAN_EMIT_SYMBOL, 68), + (6, HUFFMAN_EMIT_SYMBOL, 68), + (10, HUFFMAN_EMIT_SYMBOL, 68), + (15, HUFFMAN_EMIT_SYMBOL, 68), + (24, HUFFMAN_EMIT_SYMBOL, 68), + (31, HUFFMAN_EMIT_SYMBOL, 68), + (41, HUFFMAN_EMIT_SYMBOL, 68), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 68), + + # Node 40 + (44, 0, 0), + (45, 0, 0), + (47, 0, 0), + (48, 0, 0), + (51, 0, 0), + (52, 0, 0), + (54, 0, 0), + (55, 0, 0), + (59, 0, 0), + (60, 0, 0), + (62, 0, 0), + (63, 0, 0), + (66, 0, 0), + (67, 0, 0), + (69, 0, 0), + (72, HUFFMAN_COMPLETE, 0), + + # Node 41 + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 69), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 70), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 71), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 72), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 73), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 74), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 75), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 76), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 77), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 78), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 79), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 80), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 81), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 82), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 83), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 84), + + # Node 42 + (1, HUFFMAN_EMIT_SYMBOL, 69), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 69), + (1, HUFFMAN_EMIT_SYMBOL, 70), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 70), + (1, HUFFMAN_EMIT_SYMBOL, 71), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 71), + (1, HUFFMAN_EMIT_SYMBOL, 72), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 72), + (1, HUFFMAN_EMIT_SYMBOL, 73), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 73), + (1, HUFFMAN_EMIT_SYMBOL, 74), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 74), + (1, HUFFMAN_EMIT_SYMBOL, 75), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 75), + (1, HUFFMAN_EMIT_SYMBOL, 76), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 76), + + # Node 43 + (2, HUFFMAN_EMIT_SYMBOL, 69), + (9, HUFFMAN_EMIT_SYMBOL, 69), + (23, HUFFMAN_EMIT_SYMBOL, 69), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 69), + (2, HUFFMAN_EMIT_SYMBOL, 70), + (9, HUFFMAN_EMIT_SYMBOL, 70), + (23, HUFFMAN_EMIT_SYMBOL, 70), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 70), + (2, HUFFMAN_EMIT_SYMBOL, 71), + (9, HUFFMAN_EMIT_SYMBOL, 71), + (23, HUFFMAN_EMIT_SYMBOL, 71), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 71), + (2, HUFFMAN_EMIT_SYMBOL, 72), + (9, HUFFMAN_EMIT_SYMBOL, 72), + (23, HUFFMAN_EMIT_SYMBOL, 72), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 72), + + # Node 44 + (3, HUFFMAN_EMIT_SYMBOL, 69), + (6, HUFFMAN_EMIT_SYMBOL, 69), + (10, HUFFMAN_EMIT_SYMBOL, 69), + (15, HUFFMAN_EMIT_SYMBOL, 69), + (24, HUFFMAN_EMIT_SYMBOL, 69), + (31, HUFFMAN_EMIT_SYMBOL, 69), + (41, HUFFMAN_EMIT_SYMBOL, 69), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 69), + (3, HUFFMAN_EMIT_SYMBOL, 70), + (6, HUFFMAN_EMIT_SYMBOL, 70), + (10, HUFFMAN_EMIT_SYMBOL, 70), + (15, HUFFMAN_EMIT_SYMBOL, 70), + (24, HUFFMAN_EMIT_SYMBOL, 70), + (31, HUFFMAN_EMIT_SYMBOL, 70), + (41, HUFFMAN_EMIT_SYMBOL, 70), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 70), + + # Node 45 + (3, HUFFMAN_EMIT_SYMBOL, 71), + (6, HUFFMAN_EMIT_SYMBOL, 71), + (10, HUFFMAN_EMIT_SYMBOL, 71), + (15, HUFFMAN_EMIT_SYMBOL, 71), + (24, HUFFMAN_EMIT_SYMBOL, 71), + (31, HUFFMAN_EMIT_SYMBOL, 71), + (41, HUFFMAN_EMIT_SYMBOL, 71), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 71), + (3, HUFFMAN_EMIT_SYMBOL, 72), + (6, HUFFMAN_EMIT_SYMBOL, 72), + (10, HUFFMAN_EMIT_SYMBOL, 72), + (15, HUFFMAN_EMIT_SYMBOL, 72), + (24, HUFFMAN_EMIT_SYMBOL, 72), + (31, HUFFMAN_EMIT_SYMBOL, 72), + (41, HUFFMAN_EMIT_SYMBOL, 72), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 72), + + # Node 46 + (2, HUFFMAN_EMIT_SYMBOL, 73), + (9, HUFFMAN_EMIT_SYMBOL, 73), + (23, HUFFMAN_EMIT_SYMBOL, 73), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 73), + (2, HUFFMAN_EMIT_SYMBOL, 74), + (9, HUFFMAN_EMIT_SYMBOL, 74), + (23, HUFFMAN_EMIT_SYMBOL, 74), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 74), + (2, HUFFMAN_EMIT_SYMBOL, 75), + (9, HUFFMAN_EMIT_SYMBOL, 75), + (23, HUFFMAN_EMIT_SYMBOL, 75), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 75), + (2, HUFFMAN_EMIT_SYMBOL, 76), + (9, HUFFMAN_EMIT_SYMBOL, 76), + (23, HUFFMAN_EMIT_SYMBOL, 76), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 76), + + # Node 47 + (3, HUFFMAN_EMIT_SYMBOL, 73), + (6, HUFFMAN_EMIT_SYMBOL, 73), + (10, HUFFMAN_EMIT_SYMBOL, 73), + (15, HUFFMAN_EMIT_SYMBOL, 73), + (24, HUFFMAN_EMIT_SYMBOL, 73), + (31, HUFFMAN_EMIT_SYMBOL, 73), + (41, HUFFMAN_EMIT_SYMBOL, 73), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 73), + (3, HUFFMAN_EMIT_SYMBOL, 74), + (6, HUFFMAN_EMIT_SYMBOL, 74), + (10, HUFFMAN_EMIT_SYMBOL, 74), + (15, HUFFMAN_EMIT_SYMBOL, 74), + (24, HUFFMAN_EMIT_SYMBOL, 74), + (31, HUFFMAN_EMIT_SYMBOL, 74), + (41, HUFFMAN_EMIT_SYMBOL, 74), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 74), + + # Node 48 + (3, HUFFMAN_EMIT_SYMBOL, 75), + (6, HUFFMAN_EMIT_SYMBOL, 75), + (10, HUFFMAN_EMIT_SYMBOL, 75), + (15, HUFFMAN_EMIT_SYMBOL, 75), + (24, HUFFMAN_EMIT_SYMBOL, 75), + (31, HUFFMAN_EMIT_SYMBOL, 75), + (41, HUFFMAN_EMIT_SYMBOL, 75), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 75), + (3, HUFFMAN_EMIT_SYMBOL, 76), + (6, HUFFMAN_EMIT_SYMBOL, 76), + (10, HUFFMAN_EMIT_SYMBOL, 76), + (15, HUFFMAN_EMIT_SYMBOL, 76), + (24, HUFFMAN_EMIT_SYMBOL, 76), + (31, HUFFMAN_EMIT_SYMBOL, 76), + (41, HUFFMAN_EMIT_SYMBOL, 76), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 76), + + # Node 49 + (1, HUFFMAN_EMIT_SYMBOL, 77), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 77), + (1, HUFFMAN_EMIT_SYMBOL, 78), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 78), + (1, HUFFMAN_EMIT_SYMBOL, 79), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 79), + (1, HUFFMAN_EMIT_SYMBOL, 80), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 80), + (1, HUFFMAN_EMIT_SYMBOL, 81), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 81), + (1, HUFFMAN_EMIT_SYMBOL, 82), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 82), + (1, HUFFMAN_EMIT_SYMBOL, 83), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 83), + (1, HUFFMAN_EMIT_SYMBOL, 84), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 84), + + # Node 50 + (2, HUFFMAN_EMIT_SYMBOL, 77), + (9, HUFFMAN_EMIT_SYMBOL, 77), + (23, HUFFMAN_EMIT_SYMBOL, 77), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 77), + (2, HUFFMAN_EMIT_SYMBOL, 78), + (9, HUFFMAN_EMIT_SYMBOL, 78), + (23, HUFFMAN_EMIT_SYMBOL, 78), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 78), + (2, HUFFMAN_EMIT_SYMBOL, 79), + (9, HUFFMAN_EMIT_SYMBOL, 79), + (23, HUFFMAN_EMIT_SYMBOL, 79), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 79), + (2, HUFFMAN_EMIT_SYMBOL, 80), + (9, HUFFMAN_EMIT_SYMBOL, 80), + (23, HUFFMAN_EMIT_SYMBOL, 80), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 80), + + # Node 51 + (3, HUFFMAN_EMIT_SYMBOL, 77), + (6, HUFFMAN_EMIT_SYMBOL, 77), + (10, HUFFMAN_EMIT_SYMBOL, 77), + (15, HUFFMAN_EMIT_SYMBOL, 77), + (24, HUFFMAN_EMIT_SYMBOL, 77), + (31, HUFFMAN_EMIT_SYMBOL, 77), + (41, HUFFMAN_EMIT_SYMBOL, 77), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 77), + (3, HUFFMAN_EMIT_SYMBOL, 78), + (6, HUFFMAN_EMIT_SYMBOL, 78), + (10, HUFFMAN_EMIT_SYMBOL, 78), + (15, HUFFMAN_EMIT_SYMBOL, 78), + (24, HUFFMAN_EMIT_SYMBOL, 78), + (31, HUFFMAN_EMIT_SYMBOL, 78), + (41, HUFFMAN_EMIT_SYMBOL, 78), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 78), + + # Node 52 + (3, HUFFMAN_EMIT_SYMBOL, 79), + (6, HUFFMAN_EMIT_SYMBOL, 79), + (10, HUFFMAN_EMIT_SYMBOL, 79), + (15, HUFFMAN_EMIT_SYMBOL, 79), + (24, HUFFMAN_EMIT_SYMBOL, 79), + (31, HUFFMAN_EMIT_SYMBOL, 79), + (41, HUFFMAN_EMIT_SYMBOL, 79), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 79), + (3, HUFFMAN_EMIT_SYMBOL, 80), + (6, HUFFMAN_EMIT_SYMBOL, 80), + (10, HUFFMAN_EMIT_SYMBOL, 80), + (15, HUFFMAN_EMIT_SYMBOL, 80), + (24, HUFFMAN_EMIT_SYMBOL, 80), + (31, HUFFMAN_EMIT_SYMBOL, 80), + (41, HUFFMAN_EMIT_SYMBOL, 80), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 80), + + # Node 53 + (2, HUFFMAN_EMIT_SYMBOL, 81), + (9, HUFFMAN_EMIT_SYMBOL, 81), + (23, HUFFMAN_EMIT_SYMBOL, 81), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 81), + (2, HUFFMAN_EMIT_SYMBOL, 82), + (9, HUFFMAN_EMIT_SYMBOL, 82), + (23, HUFFMAN_EMIT_SYMBOL, 82), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 82), + (2, HUFFMAN_EMIT_SYMBOL, 83), + (9, HUFFMAN_EMIT_SYMBOL, 83), + (23, HUFFMAN_EMIT_SYMBOL, 83), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 83), + (2, HUFFMAN_EMIT_SYMBOL, 84), + (9, HUFFMAN_EMIT_SYMBOL, 84), + (23, HUFFMAN_EMIT_SYMBOL, 84), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 84), + + # Node 54 + (3, HUFFMAN_EMIT_SYMBOL, 81), + (6, HUFFMAN_EMIT_SYMBOL, 81), + (10, HUFFMAN_EMIT_SYMBOL, 81), + (15, HUFFMAN_EMIT_SYMBOL, 81), + (24, HUFFMAN_EMIT_SYMBOL, 81), + (31, HUFFMAN_EMIT_SYMBOL, 81), + (41, HUFFMAN_EMIT_SYMBOL, 81), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 81), + (3, HUFFMAN_EMIT_SYMBOL, 82), + (6, HUFFMAN_EMIT_SYMBOL, 82), + (10, HUFFMAN_EMIT_SYMBOL, 82), + (15, HUFFMAN_EMIT_SYMBOL, 82), + (24, HUFFMAN_EMIT_SYMBOL, 82), + (31, HUFFMAN_EMIT_SYMBOL, 82), + (41, HUFFMAN_EMIT_SYMBOL, 82), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 82), + + # Node 55 + (3, HUFFMAN_EMIT_SYMBOL, 83), + (6, HUFFMAN_EMIT_SYMBOL, 83), + (10, HUFFMAN_EMIT_SYMBOL, 83), + (15, HUFFMAN_EMIT_SYMBOL, 83), + (24, HUFFMAN_EMIT_SYMBOL, 83), + (31, HUFFMAN_EMIT_SYMBOL, 83), + (41, HUFFMAN_EMIT_SYMBOL, 83), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 83), + (3, HUFFMAN_EMIT_SYMBOL, 84), + (6, HUFFMAN_EMIT_SYMBOL, 84), + (10, HUFFMAN_EMIT_SYMBOL, 84), + (15, HUFFMAN_EMIT_SYMBOL, 84), + (24, HUFFMAN_EMIT_SYMBOL, 84), + (31, HUFFMAN_EMIT_SYMBOL, 84), + (41, HUFFMAN_EMIT_SYMBOL, 84), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 84), + + # Node 56 + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 85), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 86), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 87), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 89), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 106), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 107), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 113), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 118), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 119), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 120), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 121), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 122), + (70, 0, 0), + (71, 0, 0), + (73, 0, 0), + (74, HUFFMAN_COMPLETE, 0), + + # Node 57 + (1, HUFFMAN_EMIT_SYMBOL, 85), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 85), + (1, HUFFMAN_EMIT_SYMBOL, 86), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 86), + (1, HUFFMAN_EMIT_SYMBOL, 87), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 87), + (1, HUFFMAN_EMIT_SYMBOL, 89), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 89), + (1, HUFFMAN_EMIT_SYMBOL, 106), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 106), + (1, HUFFMAN_EMIT_SYMBOL, 107), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 107), + (1, HUFFMAN_EMIT_SYMBOL, 113), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 113), + (1, HUFFMAN_EMIT_SYMBOL, 118), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 118), + + # Node 58 + (2, HUFFMAN_EMIT_SYMBOL, 85), + (9, HUFFMAN_EMIT_SYMBOL, 85), + (23, HUFFMAN_EMIT_SYMBOL, 85), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 85), + (2, HUFFMAN_EMIT_SYMBOL, 86), + (9, HUFFMAN_EMIT_SYMBOL, 86), + (23, HUFFMAN_EMIT_SYMBOL, 86), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 86), + (2, HUFFMAN_EMIT_SYMBOL, 87), + (9, HUFFMAN_EMIT_SYMBOL, 87), + (23, HUFFMAN_EMIT_SYMBOL, 87), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 87), + (2, HUFFMAN_EMIT_SYMBOL, 89), + (9, HUFFMAN_EMIT_SYMBOL, 89), + (23, HUFFMAN_EMIT_SYMBOL, 89), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 89), + + # Node 59 + (3, HUFFMAN_EMIT_SYMBOL, 85), + (6, HUFFMAN_EMIT_SYMBOL, 85), + (10, HUFFMAN_EMIT_SYMBOL, 85), + (15, HUFFMAN_EMIT_SYMBOL, 85), + (24, HUFFMAN_EMIT_SYMBOL, 85), + (31, HUFFMAN_EMIT_SYMBOL, 85), + (41, HUFFMAN_EMIT_SYMBOL, 85), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 85), + (3, HUFFMAN_EMIT_SYMBOL, 86), + (6, HUFFMAN_EMIT_SYMBOL, 86), + (10, HUFFMAN_EMIT_SYMBOL, 86), + (15, HUFFMAN_EMIT_SYMBOL, 86), + (24, HUFFMAN_EMIT_SYMBOL, 86), + (31, HUFFMAN_EMIT_SYMBOL, 86), + (41, HUFFMAN_EMIT_SYMBOL, 86), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 86), + + # Node 60 + (3, HUFFMAN_EMIT_SYMBOL, 87), + (6, HUFFMAN_EMIT_SYMBOL, 87), + (10, HUFFMAN_EMIT_SYMBOL, 87), + (15, HUFFMAN_EMIT_SYMBOL, 87), + (24, HUFFMAN_EMIT_SYMBOL, 87), + (31, HUFFMAN_EMIT_SYMBOL, 87), + (41, HUFFMAN_EMIT_SYMBOL, 87), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 87), + (3, HUFFMAN_EMIT_SYMBOL, 89), + (6, HUFFMAN_EMIT_SYMBOL, 89), + (10, HUFFMAN_EMIT_SYMBOL, 89), + (15, HUFFMAN_EMIT_SYMBOL, 89), + (24, HUFFMAN_EMIT_SYMBOL, 89), + (31, HUFFMAN_EMIT_SYMBOL, 89), + (41, HUFFMAN_EMIT_SYMBOL, 89), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 89), + + # Node 61 + (2, HUFFMAN_EMIT_SYMBOL, 106), + (9, HUFFMAN_EMIT_SYMBOL, 106), + (23, HUFFMAN_EMIT_SYMBOL, 106), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 106), + (2, HUFFMAN_EMIT_SYMBOL, 107), + (9, HUFFMAN_EMIT_SYMBOL, 107), + (23, HUFFMAN_EMIT_SYMBOL, 107), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 107), + (2, HUFFMAN_EMIT_SYMBOL, 113), + (9, HUFFMAN_EMIT_SYMBOL, 113), + (23, HUFFMAN_EMIT_SYMBOL, 113), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 113), + (2, HUFFMAN_EMIT_SYMBOL, 118), + (9, HUFFMAN_EMIT_SYMBOL, 118), + (23, HUFFMAN_EMIT_SYMBOL, 118), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 118), + + # Node 62 + (3, HUFFMAN_EMIT_SYMBOL, 106), + (6, HUFFMAN_EMIT_SYMBOL, 106), + (10, HUFFMAN_EMIT_SYMBOL, 106), + (15, HUFFMAN_EMIT_SYMBOL, 106), + (24, HUFFMAN_EMIT_SYMBOL, 106), + (31, HUFFMAN_EMIT_SYMBOL, 106), + (41, HUFFMAN_EMIT_SYMBOL, 106), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 106), + (3, HUFFMAN_EMIT_SYMBOL, 107), + (6, HUFFMAN_EMIT_SYMBOL, 107), + (10, HUFFMAN_EMIT_SYMBOL, 107), + (15, HUFFMAN_EMIT_SYMBOL, 107), + (24, HUFFMAN_EMIT_SYMBOL, 107), + (31, HUFFMAN_EMIT_SYMBOL, 107), + (41, HUFFMAN_EMIT_SYMBOL, 107), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 107), + + # Node 63 + (3, HUFFMAN_EMIT_SYMBOL, 113), + (6, HUFFMAN_EMIT_SYMBOL, 113), + (10, HUFFMAN_EMIT_SYMBOL, 113), + (15, HUFFMAN_EMIT_SYMBOL, 113), + (24, HUFFMAN_EMIT_SYMBOL, 113), + (31, HUFFMAN_EMIT_SYMBOL, 113), + (41, HUFFMAN_EMIT_SYMBOL, 113), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 113), + (3, HUFFMAN_EMIT_SYMBOL, 118), + (6, HUFFMAN_EMIT_SYMBOL, 118), + (10, HUFFMAN_EMIT_SYMBOL, 118), + (15, HUFFMAN_EMIT_SYMBOL, 118), + (24, HUFFMAN_EMIT_SYMBOL, 118), + (31, HUFFMAN_EMIT_SYMBOL, 118), + (41, HUFFMAN_EMIT_SYMBOL, 118), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 118), + + # Node 64 + (1, HUFFMAN_EMIT_SYMBOL, 119), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 119), + (1, HUFFMAN_EMIT_SYMBOL, 120), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 120), + (1, HUFFMAN_EMIT_SYMBOL, 121), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 121), + (1, HUFFMAN_EMIT_SYMBOL, 122), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 122), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 38), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 42), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 44), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 59), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 88), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 90), + (75, 0, 0), + (78, 0, 0), + + # Node 65 + (2, HUFFMAN_EMIT_SYMBOL, 119), + (9, HUFFMAN_EMIT_SYMBOL, 119), + (23, HUFFMAN_EMIT_SYMBOL, 119), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 119), + (2, HUFFMAN_EMIT_SYMBOL, 120), + (9, HUFFMAN_EMIT_SYMBOL, 120), + (23, HUFFMAN_EMIT_SYMBOL, 120), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 120), + (2, HUFFMAN_EMIT_SYMBOL, 121), + (9, HUFFMAN_EMIT_SYMBOL, 121), + (23, HUFFMAN_EMIT_SYMBOL, 121), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 121), + (2, HUFFMAN_EMIT_SYMBOL, 122), + (9, HUFFMAN_EMIT_SYMBOL, 122), + (23, HUFFMAN_EMIT_SYMBOL, 122), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 122), + + # Node 66 + (3, HUFFMAN_EMIT_SYMBOL, 119), + (6, HUFFMAN_EMIT_SYMBOL, 119), + (10, HUFFMAN_EMIT_SYMBOL, 119), + (15, HUFFMAN_EMIT_SYMBOL, 119), + (24, HUFFMAN_EMIT_SYMBOL, 119), + (31, HUFFMAN_EMIT_SYMBOL, 119), + (41, HUFFMAN_EMIT_SYMBOL, 119), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 119), + (3, HUFFMAN_EMIT_SYMBOL, 120), + (6, HUFFMAN_EMIT_SYMBOL, 120), + (10, HUFFMAN_EMIT_SYMBOL, 120), + (15, HUFFMAN_EMIT_SYMBOL, 120), + (24, HUFFMAN_EMIT_SYMBOL, 120), + (31, HUFFMAN_EMIT_SYMBOL, 120), + (41, HUFFMAN_EMIT_SYMBOL, 120), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 120), + + # Node 67 + (3, HUFFMAN_EMIT_SYMBOL, 121), + (6, HUFFMAN_EMIT_SYMBOL, 121), + (10, HUFFMAN_EMIT_SYMBOL, 121), + (15, HUFFMAN_EMIT_SYMBOL, 121), + (24, HUFFMAN_EMIT_SYMBOL, 121), + (31, HUFFMAN_EMIT_SYMBOL, 121), + (41, HUFFMAN_EMIT_SYMBOL, 121), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 121), + (3, HUFFMAN_EMIT_SYMBOL, 122), + (6, HUFFMAN_EMIT_SYMBOL, 122), + (10, HUFFMAN_EMIT_SYMBOL, 122), + (15, HUFFMAN_EMIT_SYMBOL, 122), + (24, HUFFMAN_EMIT_SYMBOL, 122), + (31, HUFFMAN_EMIT_SYMBOL, 122), + (41, HUFFMAN_EMIT_SYMBOL, 122), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 122), + + # Node 68 + (1, HUFFMAN_EMIT_SYMBOL, 38), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 38), + (1, HUFFMAN_EMIT_SYMBOL, 42), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 42), + (1, HUFFMAN_EMIT_SYMBOL, 44), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 44), + (1, HUFFMAN_EMIT_SYMBOL, 59), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 59), + (1, HUFFMAN_EMIT_SYMBOL, 88), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 88), + (1, HUFFMAN_EMIT_SYMBOL, 90), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 90), + (76, 0, 0), + (77, 0, 0), + (79, 0, 0), + (81, 0, 0), + + # Node 69 + (2, HUFFMAN_EMIT_SYMBOL, 38), + (9, HUFFMAN_EMIT_SYMBOL, 38), + (23, HUFFMAN_EMIT_SYMBOL, 38), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 38), + (2, HUFFMAN_EMIT_SYMBOL, 42), + (9, HUFFMAN_EMIT_SYMBOL, 42), + (23, HUFFMAN_EMIT_SYMBOL, 42), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 42), + (2, HUFFMAN_EMIT_SYMBOL, 44), + (9, HUFFMAN_EMIT_SYMBOL, 44), + (23, HUFFMAN_EMIT_SYMBOL, 44), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 44), + (2, HUFFMAN_EMIT_SYMBOL, 59), + (9, HUFFMAN_EMIT_SYMBOL, 59), + (23, HUFFMAN_EMIT_SYMBOL, 59), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 59), + + # Node 70 + (3, HUFFMAN_EMIT_SYMBOL, 38), + (6, HUFFMAN_EMIT_SYMBOL, 38), + (10, HUFFMAN_EMIT_SYMBOL, 38), + (15, HUFFMAN_EMIT_SYMBOL, 38), + (24, HUFFMAN_EMIT_SYMBOL, 38), + (31, HUFFMAN_EMIT_SYMBOL, 38), + (41, HUFFMAN_EMIT_SYMBOL, 38), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 38), + (3, HUFFMAN_EMIT_SYMBOL, 42), + (6, HUFFMAN_EMIT_SYMBOL, 42), + (10, HUFFMAN_EMIT_SYMBOL, 42), + (15, HUFFMAN_EMIT_SYMBOL, 42), + (24, HUFFMAN_EMIT_SYMBOL, 42), + (31, HUFFMAN_EMIT_SYMBOL, 42), + (41, HUFFMAN_EMIT_SYMBOL, 42), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 42), + + # Node 71 + (3, HUFFMAN_EMIT_SYMBOL, 44), + (6, HUFFMAN_EMIT_SYMBOL, 44), + (10, HUFFMAN_EMIT_SYMBOL, 44), + (15, HUFFMAN_EMIT_SYMBOL, 44), + (24, HUFFMAN_EMIT_SYMBOL, 44), + (31, HUFFMAN_EMIT_SYMBOL, 44), + (41, HUFFMAN_EMIT_SYMBOL, 44), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 44), + (3, HUFFMAN_EMIT_SYMBOL, 59), + (6, HUFFMAN_EMIT_SYMBOL, 59), + (10, HUFFMAN_EMIT_SYMBOL, 59), + (15, HUFFMAN_EMIT_SYMBOL, 59), + (24, HUFFMAN_EMIT_SYMBOL, 59), + (31, HUFFMAN_EMIT_SYMBOL, 59), + (41, HUFFMAN_EMIT_SYMBOL, 59), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 59), + + # Node 72 + (2, HUFFMAN_EMIT_SYMBOL, 88), + (9, HUFFMAN_EMIT_SYMBOL, 88), + (23, HUFFMAN_EMIT_SYMBOL, 88), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 88), + (2, HUFFMAN_EMIT_SYMBOL, 90), + (9, HUFFMAN_EMIT_SYMBOL, 90), + (23, HUFFMAN_EMIT_SYMBOL, 90), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 90), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 33), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 34), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 40), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 41), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 63), + (80, 0, 0), + (82, 0, 0), + (84, 0, 0), + + # Node 73 + (3, HUFFMAN_EMIT_SYMBOL, 88), + (6, HUFFMAN_EMIT_SYMBOL, 88), + (10, HUFFMAN_EMIT_SYMBOL, 88), + (15, HUFFMAN_EMIT_SYMBOL, 88), + (24, HUFFMAN_EMIT_SYMBOL, 88), + (31, HUFFMAN_EMIT_SYMBOL, 88), + (41, HUFFMAN_EMIT_SYMBOL, 88), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 88), + (3, HUFFMAN_EMIT_SYMBOL, 90), + (6, HUFFMAN_EMIT_SYMBOL, 90), + (10, HUFFMAN_EMIT_SYMBOL, 90), + (15, HUFFMAN_EMIT_SYMBOL, 90), + (24, HUFFMAN_EMIT_SYMBOL, 90), + (31, HUFFMAN_EMIT_SYMBOL, 90), + (41, HUFFMAN_EMIT_SYMBOL, 90), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 90), + + # Node 74 + (1, HUFFMAN_EMIT_SYMBOL, 33), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 33), + (1, HUFFMAN_EMIT_SYMBOL, 34), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 34), + (1, HUFFMAN_EMIT_SYMBOL, 40), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 40), + (1, HUFFMAN_EMIT_SYMBOL, 41), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 41), + (1, HUFFMAN_EMIT_SYMBOL, 63), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 63), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 39), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 43), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 124), + (83, 0, 0), + (85, 0, 0), + (88, 0, 0), + + # Node 75 + (2, HUFFMAN_EMIT_SYMBOL, 33), + (9, HUFFMAN_EMIT_SYMBOL, 33), + (23, HUFFMAN_EMIT_SYMBOL, 33), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 33), + (2, HUFFMAN_EMIT_SYMBOL, 34), + (9, HUFFMAN_EMIT_SYMBOL, 34), + (23, HUFFMAN_EMIT_SYMBOL, 34), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 34), + (2, HUFFMAN_EMIT_SYMBOL, 40), + (9, HUFFMAN_EMIT_SYMBOL, 40), + (23, HUFFMAN_EMIT_SYMBOL, 40), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 40), + (2, HUFFMAN_EMIT_SYMBOL, 41), + (9, HUFFMAN_EMIT_SYMBOL, 41), + (23, HUFFMAN_EMIT_SYMBOL, 41), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 41), + + # Node 76 + (3, HUFFMAN_EMIT_SYMBOL, 33), + (6, HUFFMAN_EMIT_SYMBOL, 33), + (10, HUFFMAN_EMIT_SYMBOL, 33), + (15, HUFFMAN_EMIT_SYMBOL, 33), + (24, HUFFMAN_EMIT_SYMBOL, 33), + (31, HUFFMAN_EMIT_SYMBOL, 33), + (41, HUFFMAN_EMIT_SYMBOL, 33), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 33), + (3, HUFFMAN_EMIT_SYMBOL, 34), + (6, HUFFMAN_EMIT_SYMBOL, 34), + (10, HUFFMAN_EMIT_SYMBOL, 34), + (15, HUFFMAN_EMIT_SYMBOL, 34), + (24, HUFFMAN_EMIT_SYMBOL, 34), + (31, HUFFMAN_EMIT_SYMBOL, 34), + (41, HUFFMAN_EMIT_SYMBOL, 34), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 34), + + # Node 77 + (3, HUFFMAN_EMIT_SYMBOL, 40), + (6, HUFFMAN_EMIT_SYMBOL, 40), + (10, HUFFMAN_EMIT_SYMBOL, 40), + (15, HUFFMAN_EMIT_SYMBOL, 40), + (24, HUFFMAN_EMIT_SYMBOL, 40), + (31, HUFFMAN_EMIT_SYMBOL, 40), + (41, HUFFMAN_EMIT_SYMBOL, 40), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 40), + (3, HUFFMAN_EMIT_SYMBOL, 41), + (6, HUFFMAN_EMIT_SYMBOL, 41), + (10, HUFFMAN_EMIT_SYMBOL, 41), + (15, HUFFMAN_EMIT_SYMBOL, 41), + (24, HUFFMAN_EMIT_SYMBOL, 41), + (31, HUFFMAN_EMIT_SYMBOL, 41), + (41, HUFFMAN_EMIT_SYMBOL, 41), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 41), + + # Node 78 + (2, HUFFMAN_EMIT_SYMBOL, 63), + (9, HUFFMAN_EMIT_SYMBOL, 63), + (23, HUFFMAN_EMIT_SYMBOL, 63), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 63), + (1, HUFFMAN_EMIT_SYMBOL, 39), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 39), + (1, HUFFMAN_EMIT_SYMBOL, 43), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 43), + (1, HUFFMAN_EMIT_SYMBOL, 124), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 124), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 35), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 62), + (86, 0, 0), + (87, 0, 0), + (89, 0, 0), + (90, 0, 0), + + # Node 79 + (3, HUFFMAN_EMIT_SYMBOL, 63), + (6, HUFFMAN_EMIT_SYMBOL, 63), + (10, HUFFMAN_EMIT_SYMBOL, 63), + (15, HUFFMAN_EMIT_SYMBOL, 63), + (24, HUFFMAN_EMIT_SYMBOL, 63), + (31, HUFFMAN_EMIT_SYMBOL, 63), + (41, HUFFMAN_EMIT_SYMBOL, 63), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 63), + (2, HUFFMAN_EMIT_SYMBOL, 39), + (9, HUFFMAN_EMIT_SYMBOL, 39), + (23, HUFFMAN_EMIT_SYMBOL, 39), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 39), + (2, HUFFMAN_EMIT_SYMBOL, 43), + (9, HUFFMAN_EMIT_SYMBOL, 43), + (23, HUFFMAN_EMIT_SYMBOL, 43), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 43), + + # Node 80 + (3, HUFFMAN_EMIT_SYMBOL, 39), + (6, HUFFMAN_EMIT_SYMBOL, 39), + (10, HUFFMAN_EMIT_SYMBOL, 39), + (15, HUFFMAN_EMIT_SYMBOL, 39), + (24, HUFFMAN_EMIT_SYMBOL, 39), + (31, HUFFMAN_EMIT_SYMBOL, 39), + (41, HUFFMAN_EMIT_SYMBOL, 39), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 39), + (3, HUFFMAN_EMIT_SYMBOL, 43), + (6, HUFFMAN_EMIT_SYMBOL, 43), + (10, HUFFMAN_EMIT_SYMBOL, 43), + (15, HUFFMAN_EMIT_SYMBOL, 43), + (24, HUFFMAN_EMIT_SYMBOL, 43), + (31, HUFFMAN_EMIT_SYMBOL, 43), + (41, HUFFMAN_EMIT_SYMBOL, 43), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 43), + + # Node 81 + (2, HUFFMAN_EMIT_SYMBOL, 124), + (9, HUFFMAN_EMIT_SYMBOL, 124), + (23, HUFFMAN_EMIT_SYMBOL, 124), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 124), + (1, HUFFMAN_EMIT_SYMBOL, 35), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 35), + (1, HUFFMAN_EMIT_SYMBOL, 62), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 62), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 0), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 36), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 64), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 91), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 93), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 126), + (91, 0, 0), + (92, 0, 0), + + # Node 82 + (3, HUFFMAN_EMIT_SYMBOL, 124), + (6, HUFFMAN_EMIT_SYMBOL, 124), + (10, HUFFMAN_EMIT_SYMBOL, 124), + (15, HUFFMAN_EMIT_SYMBOL, 124), + (24, HUFFMAN_EMIT_SYMBOL, 124), + (31, HUFFMAN_EMIT_SYMBOL, 124), + (41, HUFFMAN_EMIT_SYMBOL, 124), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 124), + (2, HUFFMAN_EMIT_SYMBOL, 35), + (9, HUFFMAN_EMIT_SYMBOL, 35), + (23, HUFFMAN_EMIT_SYMBOL, 35), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 35), + (2, HUFFMAN_EMIT_SYMBOL, 62), + (9, HUFFMAN_EMIT_SYMBOL, 62), + (23, HUFFMAN_EMIT_SYMBOL, 62), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 62), + + # Node 83 + (3, HUFFMAN_EMIT_SYMBOL, 35), + (6, HUFFMAN_EMIT_SYMBOL, 35), + (10, HUFFMAN_EMIT_SYMBOL, 35), + (15, HUFFMAN_EMIT_SYMBOL, 35), + (24, HUFFMAN_EMIT_SYMBOL, 35), + (31, HUFFMAN_EMIT_SYMBOL, 35), + (41, HUFFMAN_EMIT_SYMBOL, 35), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 35), + (3, HUFFMAN_EMIT_SYMBOL, 62), + (6, HUFFMAN_EMIT_SYMBOL, 62), + (10, HUFFMAN_EMIT_SYMBOL, 62), + (15, HUFFMAN_EMIT_SYMBOL, 62), + (24, HUFFMAN_EMIT_SYMBOL, 62), + (31, HUFFMAN_EMIT_SYMBOL, 62), + (41, HUFFMAN_EMIT_SYMBOL, 62), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 62), + + # Node 84 + (1, HUFFMAN_EMIT_SYMBOL, 0), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 0), + (1, HUFFMAN_EMIT_SYMBOL, 36), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 36), + (1, HUFFMAN_EMIT_SYMBOL, 64), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 64), + (1, HUFFMAN_EMIT_SYMBOL, 91), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 91), + (1, HUFFMAN_EMIT_SYMBOL, 93), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 93), + (1, HUFFMAN_EMIT_SYMBOL, 126), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 126), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 94), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 125), + (93, 0, 0), + (94, 0, 0), + + # Node 85 + (2, HUFFMAN_EMIT_SYMBOL, 0), + (9, HUFFMAN_EMIT_SYMBOL, 0), + (23, HUFFMAN_EMIT_SYMBOL, 0), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 0), + (2, HUFFMAN_EMIT_SYMBOL, 36), + (9, HUFFMAN_EMIT_SYMBOL, 36), + (23, HUFFMAN_EMIT_SYMBOL, 36), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 36), + (2, HUFFMAN_EMIT_SYMBOL, 64), + (9, HUFFMAN_EMIT_SYMBOL, 64), + (23, HUFFMAN_EMIT_SYMBOL, 64), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 64), + (2, HUFFMAN_EMIT_SYMBOL, 91), + (9, HUFFMAN_EMIT_SYMBOL, 91), + (23, HUFFMAN_EMIT_SYMBOL, 91), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 91), + + # Node 86 + (3, HUFFMAN_EMIT_SYMBOL, 0), + (6, HUFFMAN_EMIT_SYMBOL, 0), + (10, HUFFMAN_EMIT_SYMBOL, 0), + (15, HUFFMAN_EMIT_SYMBOL, 0), + (24, HUFFMAN_EMIT_SYMBOL, 0), + (31, HUFFMAN_EMIT_SYMBOL, 0), + (41, HUFFMAN_EMIT_SYMBOL, 0), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 0), + (3, HUFFMAN_EMIT_SYMBOL, 36), + (6, HUFFMAN_EMIT_SYMBOL, 36), + (10, HUFFMAN_EMIT_SYMBOL, 36), + (15, HUFFMAN_EMIT_SYMBOL, 36), + (24, HUFFMAN_EMIT_SYMBOL, 36), + (31, HUFFMAN_EMIT_SYMBOL, 36), + (41, HUFFMAN_EMIT_SYMBOL, 36), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 36), + + # Node 87 + (3, HUFFMAN_EMIT_SYMBOL, 64), + (6, HUFFMAN_EMIT_SYMBOL, 64), + (10, HUFFMAN_EMIT_SYMBOL, 64), + (15, HUFFMAN_EMIT_SYMBOL, 64), + (24, HUFFMAN_EMIT_SYMBOL, 64), + (31, HUFFMAN_EMIT_SYMBOL, 64), + (41, HUFFMAN_EMIT_SYMBOL, 64), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 64), + (3, HUFFMAN_EMIT_SYMBOL, 91), + (6, HUFFMAN_EMIT_SYMBOL, 91), + (10, HUFFMAN_EMIT_SYMBOL, 91), + (15, HUFFMAN_EMIT_SYMBOL, 91), + (24, HUFFMAN_EMIT_SYMBOL, 91), + (31, HUFFMAN_EMIT_SYMBOL, 91), + (41, HUFFMAN_EMIT_SYMBOL, 91), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 91), + + # Node 88 + (2, HUFFMAN_EMIT_SYMBOL, 93), + (9, HUFFMAN_EMIT_SYMBOL, 93), + (23, HUFFMAN_EMIT_SYMBOL, 93), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 93), + (2, HUFFMAN_EMIT_SYMBOL, 126), + (9, HUFFMAN_EMIT_SYMBOL, 126), + (23, HUFFMAN_EMIT_SYMBOL, 126), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 126), + (1, HUFFMAN_EMIT_SYMBOL, 94), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 94), + (1, HUFFMAN_EMIT_SYMBOL, 125), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 125), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 60), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 96), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 123), + (95, 0, 0), + + # Node 89 + (3, HUFFMAN_EMIT_SYMBOL, 93), + (6, HUFFMAN_EMIT_SYMBOL, 93), + (10, HUFFMAN_EMIT_SYMBOL, 93), + (15, HUFFMAN_EMIT_SYMBOL, 93), + (24, HUFFMAN_EMIT_SYMBOL, 93), + (31, HUFFMAN_EMIT_SYMBOL, 93), + (41, HUFFMAN_EMIT_SYMBOL, 93), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 93), + (3, HUFFMAN_EMIT_SYMBOL, 126), + (6, HUFFMAN_EMIT_SYMBOL, 126), + (10, HUFFMAN_EMIT_SYMBOL, 126), + (15, HUFFMAN_EMIT_SYMBOL, 126), + (24, HUFFMAN_EMIT_SYMBOL, 126), + (31, HUFFMAN_EMIT_SYMBOL, 126), + (41, HUFFMAN_EMIT_SYMBOL, 126), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 126), + + # Node 90 + (2, HUFFMAN_EMIT_SYMBOL, 94), + (9, HUFFMAN_EMIT_SYMBOL, 94), + (23, HUFFMAN_EMIT_SYMBOL, 94), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 94), + (2, HUFFMAN_EMIT_SYMBOL, 125), + (9, HUFFMAN_EMIT_SYMBOL, 125), + (23, HUFFMAN_EMIT_SYMBOL, 125), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 125), + (1, HUFFMAN_EMIT_SYMBOL, 60), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 60), + (1, HUFFMAN_EMIT_SYMBOL, 96), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 96), + (1, HUFFMAN_EMIT_SYMBOL, 123), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 123), + (96, 0, 0), + (110, 0, 0), + + # Node 91 + (3, HUFFMAN_EMIT_SYMBOL, 94), + (6, HUFFMAN_EMIT_SYMBOL, 94), + (10, HUFFMAN_EMIT_SYMBOL, 94), + (15, HUFFMAN_EMIT_SYMBOL, 94), + (24, HUFFMAN_EMIT_SYMBOL, 94), + (31, HUFFMAN_EMIT_SYMBOL, 94), + (41, HUFFMAN_EMIT_SYMBOL, 94), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 94), + (3, HUFFMAN_EMIT_SYMBOL, 125), + (6, HUFFMAN_EMIT_SYMBOL, 125), + (10, HUFFMAN_EMIT_SYMBOL, 125), + (15, HUFFMAN_EMIT_SYMBOL, 125), + (24, HUFFMAN_EMIT_SYMBOL, 125), + (31, HUFFMAN_EMIT_SYMBOL, 125), + (41, HUFFMAN_EMIT_SYMBOL, 125), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 125), + + # Node 92 + (2, HUFFMAN_EMIT_SYMBOL, 60), + (9, HUFFMAN_EMIT_SYMBOL, 60), + (23, HUFFMAN_EMIT_SYMBOL, 60), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 60), + (2, HUFFMAN_EMIT_SYMBOL, 96), + (9, HUFFMAN_EMIT_SYMBOL, 96), + (23, HUFFMAN_EMIT_SYMBOL, 96), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 96), + (2, HUFFMAN_EMIT_SYMBOL, 123), + (9, HUFFMAN_EMIT_SYMBOL, 123), + (23, HUFFMAN_EMIT_SYMBOL, 123), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 123), + (97, 0, 0), + (101, 0, 0), + (111, 0, 0), + (133, 0, 0), + + # Node 93 + (3, HUFFMAN_EMIT_SYMBOL, 60), + (6, HUFFMAN_EMIT_SYMBOL, 60), + (10, HUFFMAN_EMIT_SYMBOL, 60), + (15, HUFFMAN_EMIT_SYMBOL, 60), + (24, HUFFMAN_EMIT_SYMBOL, 60), + (31, HUFFMAN_EMIT_SYMBOL, 60), + (41, HUFFMAN_EMIT_SYMBOL, 60), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 60), + (3, HUFFMAN_EMIT_SYMBOL, 96), + (6, HUFFMAN_EMIT_SYMBOL, 96), + (10, HUFFMAN_EMIT_SYMBOL, 96), + (15, HUFFMAN_EMIT_SYMBOL, 96), + (24, HUFFMAN_EMIT_SYMBOL, 96), + (31, HUFFMAN_EMIT_SYMBOL, 96), + (41, HUFFMAN_EMIT_SYMBOL, 96), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 96), + + # Node 94 + (3, HUFFMAN_EMIT_SYMBOL, 123), + (6, HUFFMAN_EMIT_SYMBOL, 123), + (10, HUFFMAN_EMIT_SYMBOL, 123), + (15, HUFFMAN_EMIT_SYMBOL, 123), + (24, HUFFMAN_EMIT_SYMBOL, 123), + (31, HUFFMAN_EMIT_SYMBOL, 123), + (41, HUFFMAN_EMIT_SYMBOL, 123), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 123), + (98, 0, 0), + (99, 0, 0), + (102, 0, 0), + (105, 0, 0), + (112, 0, 0), + (119, 0, 0), + (134, 0, 0), + (153, 0, 0), + + # Node 95 + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 92), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 195), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 208), + (100, 0, 0), + (103, 0, 0), + (104, 0, 0), + (106, 0, 0), + (107, 0, 0), + (113, 0, 0), + (116, 0, 0), + (120, 0, 0), + (126, 0, 0), + (135, 0, 0), + (142, 0, 0), + (154, 0, 0), + (169, 0, 0), + + # Node 96 + (1, HUFFMAN_EMIT_SYMBOL, 92), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 92), + (1, HUFFMAN_EMIT_SYMBOL, 195), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 195), + (1, HUFFMAN_EMIT_SYMBOL, 208), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 208), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 128), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 130), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 131), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 162), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 184), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 194), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 224), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 226), + (108, 0, 0), + (109, 0, 0), + + # Node 97 + (2, HUFFMAN_EMIT_SYMBOL, 92), + (9, HUFFMAN_EMIT_SYMBOL, 92), + (23, HUFFMAN_EMIT_SYMBOL, 92), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 92), + (2, HUFFMAN_EMIT_SYMBOL, 195), + (9, HUFFMAN_EMIT_SYMBOL, 195), + (23, HUFFMAN_EMIT_SYMBOL, 195), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 195), + (2, HUFFMAN_EMIT_SYMBOL, 208), + (9, HUFFMAN_EMIT_SYMBOL, 208), + (23, HUFFMAN_EMIT_SYMBOL, 208), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 208), + (1, HUFFMAN_EMIT_SYMBOL, 128), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 128), + (1, HUFFMAN_EMIT_SYMBOL, 130), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 130), + + # Node 98 + (3, HUFFMAN_EMIT_SYMBOL, 92), + (6, HUFFMAN_EMIT_SYMBOL, 92), + (10, HUFFMAN_EMIT_SYMBOL, 92), + (15, HUFFMAN_EMIT_SYMBOL, 92), + (24, HUFFMAN_EMIT_SYMBOL, 92), + (31, HUFFMAN_EMIT_SYMBOL, 92), + (41, HUFFMAN_EMIT_SYMBOL, 92), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 92), + (3, HUFFMAN_EMIT_SYMBOL, 195), + (6, HUFFMAN_EMIT_SYMBOL, 195), + (10, HUFFMAN_EMIT_SYMBOL, 195), + (15, HUFFMAN_EMIT_SYMBOL, 195), + (24, HUFFMAN_EMIT_SYMBOL, 195), + (31, HUFFMAN_EMIT_SYMBOL, 195), + (41, HUFFMAN_EMIT_SYMBOL, 195), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 195), + + # Node 99 + (3, HUFFMAN_EMIT_SYMBOL, 208), + (6, HUFFMAN_EMIT_SYMBOL, 208), + (10, HUFFMAN_EMIT_SYMBOL, 208), + (15, HUFFMAN_EMIT_SYMBOL, 208), + (24, HUFFMAN_EMIT_SYMBOL, 208), + (31, HUFFMAN_EMIT_SYMBOL, 208), + (41, HUFFMAN_EMIT_SYMBOL, 208), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 208), + (2, HUFFMAN_EMIT_SYMBOL, 128), + (9, HUFFMAN_EMIT_SYMBOL, 128), + (23, HUFFMAN_EMIT_SYMBOL, 128), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 128), + (2, HUFFMAN_EMIT_SYMBOL, 130), + (9, HUFFMAN_EMIT_SYMBOL, 130), + (23, HUFFMAN_EMIT_SYMBOL, 130), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 130), + + # Node 100 + (3, HUFFMAN_EMIT_SYMBOL, 128), + (6, HUFFMAN_EMIT_SYMBOL, 128), + (10, HUFFMAN_EMIT_SYMBOL, 128), + (15, HUFFMAN_EMIT_SYMBOL, 128), + (24, HUFFMAN_EMIT_SYMBOL, 128), + (31, HUFFMAN_EMIT_SYMBOL, 128), + (41, HUFFMAN_EMIT_SYMBOL, 128), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 128), + (3, HUFFMAN_EMIT_SYMBOL, 130), + (6, HUFFMAN_EMIT_SYMBOL, 130), + (10, HUFFMAN_EMIT_SYMBOL, 130), + (15, HUFFMAN_EMIT_SYMBOL, 130), + (24, HUFFMAN_EMIT_SYMBOL, 130), + (31, HUFFMAN_EMIT_SYMBOL, 130), + (41, HUFFMAN_EMIT_SYMBOL, 130), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 130), + + # Node 101 + (1, HUFFMAN_EMIT_SYMBOL, 131), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 131), + (1, HUFFMAN_EMIT_SYMBOL, 162), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 162), + (1, HUFFMAN_EMIT_SYMBOL, 184), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 184), + (1, HUFFMAN_EMIT_SYMBOL, 194), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 194), + (1, HUFFMAN_EMIT_SYMBOL, 224), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 224), + (1, HUFFMAN_EMIT_SYMBOL, 226), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 226), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 153), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 161), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 167), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 172), + + # Node 102 + (2, HUFFMAN_EMIT_SYMBOL, 131), + (9, HUFFMAN_EMIT_SYMBOL, 131), + (23, HUFFMAN_EMIT_SYMBOL, 131), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 131), + (2, HUFFMAN_EMIT_SYMBOL, 162), + (9, HUFFMAN_EMIT_SYMBOL, 162), + (23, HUFFMAN_EMIT_SYMBOL, 162), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 162), + (2, HUFFMAN_EMIT_SYMBOL, 184), + (9, HUFFMAN_EMIT_SYMBOL, 184), + (23, HUFFMAN_EMIT_SYMBOL, 184), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 184), + (2, HUFFMAN_EMIT_SYMBOL, 194), + (9, HUFFMAN_EMIT_SYMBOL, 194), + (23, HUFFMAN_EMIT_SYMBOL, 194), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 194), + + # Node 103 + (3, HUFFMAN_EMIT_SYMBOL, 131), + (6, HUFFMAN_EMIT_SYMBOL, 131), + (10, HUFFMAN_EMIT_SYMBOL, 131), + (15, HUFFMAN_EMIT_SYMBOL, 131), + (24, HUFFMAN_EMIT_SYMBOL, 131), + (31, HUFFMAN_EMIT_SYMBOL, 131), + (41, HUFFMAN_EMIT_SYMBOL, 131), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 131), + (3, HUFFMAN_EMIT_SYMBOL, 162), + (6, HUFFMAN_EMIT_SYMBOL, 162), + (10, HUFFMAN_EMIT_SYMBOL, 162), + (15, HUFFMAN_EMIT_SYMBOL, 162), + (24, HUFFMAN_EMIT_SYMBOL, 162), + (31, HUFFMAN_EMIT_SYMBOL, 162), + (41, HUFFMAN_EMIT_SYMBOL, 162), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 162), + + # Node 104 + (3, HUFFMAN_EMIT_SYMBOL, 184), + (6, HUFFMAN_EMIT_SYMBOL, 184), + (10, HUFFMAN_EMIT_SYMBOL, 184), + (15, HUFFMAN_EMIT_SYMBOL, 184), + (24, HUFFMAN_EMIT_SYMBOL, 184), + (31, HUFFMAN_EMIT_SYMBOL, 184), + (41, HUFFMAN_EMIT_SYMBOL, 184), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 184), + (3, HUFFMAN_EMIT_SYMBOL, 194), + (6, HUFFMAN_EMIT_SYMBOL, 194), + (10, HUFFMAN_EMIT_SYMBOL, 194), + (15, HUFFMAN_EMIT_SYMBOL, 194), + (24, HUFFMAN_EMIT_SYMBOL, 194), + (31, HUFFMAN_EMIT_SYMBOL, 194), + (41, HUFFMAN_EMIT_SYMBOL, 194), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 194), + + # Node 105 + (2, HUFFMAN_EMIT_SYMBOL, 224), + (9, HUFFMAN_EMIT_SYMBOL, 224), + (23, HUFFMAN_EMIT_SYMBOL, 224), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 224), + (2, HUFFMAN_EMIT_SYMBOL, 226), + (9, HUFFMAN_EMIT_SYMBOL, 226), + (23, HUFFMAN_EMIT_SYMBOL, 226), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 226), + (1, HUFFMAN_EMIT_SYMBOL, 153), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 153), + (1, HUFFMAN_EMIT_SYMBOL, 161), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 161), + (1, HUFFMAN_EMIT_SYMBOL, 167), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 167), + (1, HUFFMAN_EMIT_SYMBOL, 172), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 172), + + # Node 106 + (3, HUFFMAN_EMIT_SYMBOL, 224), + (6, HUFFMAN_EMIT_SYMBOL, 224), + (10, HUFFMAN_EMIT_SYMBOL, 224), + (15, HUFFMAN_EMIT_SYMBOL, 224), + (24, HUFFMAN_EMIT_SYMBOL, 224), + (31, HUFFMAN_EMIT_SYMBOL, 224), + (41, HUFFMAN_EMIT_SYMBOL, 224), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 224), + (3, HUFFMAN_EMIT_SYMBOL, 226), + (6, HUFFMAN_EMIT_SYMBOL, 226), + (10, HUFFMAN_EMIT_SYMBOL, 226), + (15, HUFFMAN_EMIT_SYMBOL, 226), + (24, HUFFMAN_EMIT_SYMBOL, 226), + (31, HUFFMAN_EMIT_SYMBOL, 226), + (41, HUFFMAN_EMIT_SYMBOL, 226), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 226), + + # Node 107 + (2, HUFFMAN_EMIT_SYMBOL, 153), + (9, HUFFMAN_EMIT_SYMBOL, 153), + (23, HUFFMAN_EMIT_SYMBOL, 153), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 153), + (2, HUFFMAN_EMIT_SYMBOL, 161), + (9, HUFFMAN_EMIT_SYMBOL, 161), + (23, HUFFMAN_EMIT_SYMBOL, 161), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 161), + (2, HUFFMAN_EMIT_SYMBOL, 167), + (9, HUFFMAN_EMIT_SYMBOL, 167), + (23, HUFFMAN_EMIT_SYMBOL, 167), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 167), + (2, HUFFMAN_EMIT_SYMBOL, 172), + (9, HUFFMAN_EMIT_SYMBOL, 172), + (23, HUFFMAN_EMIT_SYMBOL, 172), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 172), + + # Node 108 + (3, HUFFMAN_EMIT_SYMBOL, 153), + (6, HUFFMAN_EMIT_SYMBOL, 153), + (10, HUFFMAN_EMIT_SYMBOL, 153), + (15, HUFFMAN_EMIT_SYMBOL, 153), + (24, HUFFMAN_EMIT_SYMBOL, 153), + (31, HUFFMAN_EMIT_SYMBOL, 153), + (41, HUFFMAN_EMIT_SYMBOL, 153), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 153), + (3, HUFFMAN_EMIT_SYMBOL, 161), + (6, HUFFMAN_EMIT_SYMBOL, 161), + (10, HUFFMAN_EMIT_SYMBOL, 161), + (15, HUFFMAN_EMIT_SYMBOL, 161), + (24, HUFFMAN_EMIT_SYMBOL, 161), + (31, HUFFMAN_EMIT_SYMBOL, 161), + (41, HUFFMAN_EMIT_SYMBOL, 161), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 161), + + # Node 109 + (3, HUFFMAN_EMIT_SYMBOL, 167), + (6, HUFFMAN_EMIT_SYMBOL, 167), + (10, HUFFMAN_EMIT_SYMBOL, 167), + (15, HUFFMAN_EMIT_SYMBOL, 167), + (24, HUFFMAN_EMIT_SYMBOL, 167), + (31, HUFFMAN_EMIT_SYMBOL, 167), + (41, HUFFMAN_EMIT_SYMBOL, 167), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 167), + (3, HUFFMAN_EMIT_SYMBOL, 172), + (6, HUFFMAN_EMIT_SYMBOL, 172), + (10, HUFFMAN_EMIT_SYMBOL, 172), + (15, HUFFMAN_EMIT_SYMBOL, 172), + (24, HUFFMAN_EMIT_SYMBOL, 172), + (31, HUFFMAN_EMIT_SYMBOL, 172), + (41, HUFFMAN_EMIT_SYMBOL, 172), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 172), + + # Node 110 + (114, 0, 0), + (115, 0, 0), + (117, 0, 0), + (118, 0, 0), + (121, 0, 0), + (123, 0, 0), + (127, 0, 0), + (130, 0, 0), + (136, 0, 0), + (139, 0, 0), + (143, 0, 0), + (146, 0, 0), + (155, 0, 0), + (162, 0, 0), + (170, 0, 0), + (180, 0, 0), + + # Node 111 + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 176), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 177), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 179), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 209), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 216), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 217), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 227), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 229), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 230), + (122, 0, 0), + (124, 0, 0), + (125, 0, 0), + (128, 0, 0), + (129, 0, 0), + (131, 0, 0), + (132, 0, 0), + + # Node 112 + (1, HUFFMAN_EMIT_SYMBOL, 176), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 176), + (1, HUFFMAN_EMIT_SYMBOL, 177), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 177), + (1, HUFFMAN_EMIT_SYMBOL, 179), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 179), + (1, HUFFMAN_EMIT_SYMBOL, 209), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 209), + (1, HUFFMAN_EMIT_SYMBOL, 216), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 216), + (1, HUFFMAN_EMIT_SYMBOL, 217), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 217), + (1, HUFFMAN_EMIT_SYMBOL, 227), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 227), + (1, HUFFMAN_EMIT_SYMBOL, 229), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 229), + + # Node 113 + (2, HUFFMAN_EMIT_SYMBOL, 176), + (9, HUFFMAN_EMIT_SYMBOL, 176), + (23, HUFFMAN_EMIT_SYMBOL, 176), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 176), + (2, HUFFMAN_EMIT_SYMBOL, 177), + (9, HUFFMAN_EMIT_SYMBOL, 177), + (23, HUFFMAN_EMIT_SYMBOL, 177), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 177), + (2, HUFFMAN_EMIT_SYMBOL, 179), + (9, HUFFMAN_EMIT_SYMBOL, 179), + (23, HUFFMAN_EMIT_SYMBOL, 179), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 179), + (2, HUFFMAN_EMIT_SYMBOL, 209), + (9, HUFFMAN_EMIT_SYMBOL, 209), + (23, HUFFMAN_EMIT_SYMBOL, 209), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 209), + + # Node 114 + (3, HUFFMAN_EMIT_SYMBOL, 176), + (6, HUFFMAN_EMIT_SYMBOL, 176), + (10, HUFFMAN_EMIT_SYMBOL, 176), + (15, HUFFMAN_EMIT_SYMBOL, 176), + (24, HUFFMAN_EMIT_SYMBOL, 176), + (31, HUFFMAN_EMIT_SYMBOL, 176), + (41, HUFFMAN_EMIT_SYMBOL, 176), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 176), + (3, HUFFMAN_EMIT_SYMBOL, 177), + (6, HUFFMAN_EMIT_SYMBOL, 177), + (10, HUFFMAN_EMIT_SYMBOL, 177), + (15, HUFFMAN_EMIT_SYMBOL, 177), + (24, HUFFMAN_EMIT_SYMBOL, 177), + (31, HUFFMAN_EMIT_SYMBOL, 177), + (41, HUFFMAN_EMIT_SYMBOL, 177), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 177), + + # Node 115 + (3, HUFFMAN_EMIT_SYMBOL, 179), + (6, HUFFMAN_EMIT_SYMBOL, 179), + (10, HUFFMAN_EMIT_SYMBOL, 179), + (15, HUFFMAN_EMIT_SYMBOL, 179), + (24, HUFFMAN_EMIT_SYMBOL, 179), + (31, HUFFMAN_EMIT_SYMBOL, 179), + (41, HUFFMAN_EMIT_SYMBOL, 179), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 179), + (3, HUFFMAN_EMIT_SYMBOL, 209), + (6, HUFFMAN_EMIT_SYMBOL, 209), + (10, HUFFMAN_EMIT_SYMBOL, 209), + (15, HUFFMAN_EMIT_SYMBOL, 209), + (24, HUFFMAN_EMIT_SYMBOL, 209), + (31, HUFFMAN_EMIT_SYMBOL, 209), + (41, HUFFMAN_EMIT_SYMBOL, 209), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 209), + + # Node 116 + (2, HUFFMAN_EMIT_SYMBOL, 216), + (9, HUFFMAN_EMIT_SYMBOL, 216), + (23, HUFFMAN_EMIT_SYMBOL, 216), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 216), + (2, HUFFMAN_EMIT_SYMBOL, 217), + (9, HUFFMAN_EMIT_SYMBOL, 217), + (23, HUFFMAN_EMIT_SYMBOL, 217), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 217), + (2, HUFFMAN_EMIT_SYMBOL, 227), + (9, HUFFMAN_EMIT_SYMBOL, 227), + (23, HUFFMAN_EMIT_SYMBOL, 227), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 227), + (2, HUFFMAN_EMIT_SYMBOL, 229), + (9, HUFFMAN_EMIT_SYMBOL, 229), + (23, HUFFMAN_EMIT_SYMBOL, 229), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 229), + + # Node 117 + (3, HUFFMAN_EMIT_SYMBOL, 216), + (6, HUFFMAN_EMIT_SYMBOL, 216), + (10, HUFFMAN_EMIT_SYMBOL, 216), + (15, HUFFMAN_EMIT_SYMBOL, 216), + (24, HUFFMAN_EMIT_SYMBOL, 216), + (31, HUFFMAN_EMIT_SYMBOL, 216), + (41, HUFFMAN_EMIT_SYMBOL, 216), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 216), + (3, HUFFMAN_EMIT_SYMBOL, 217), + (6, HUFFMAN_EMIT_SYMBOL, 217), + (10, HUFFMAN_EMIT_SYMBOL, 217), + (15, HUFFMAN_EMIT_SYMBOL, 217), + (24, HUFFMAN_EMIT_SYMBOL, 217), + (31, HUFFMAN_EMIT_SYMBOL, 217), + (41, HUFFMAN_EMIT_SYMBOL, 217), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 217), + + # Node 118 + (3, HUFFMAN_EMIT_SYMBOL, 227), + (6, HUFFMAN_EMIT_SYMBOL, 227), + (10, HUFFMAN_EMIT_SYMBOL, 227), + (15, HUFFMAN_EMIT_SYMBOL, 227), + (24, HUFFMAN_EMIT_SYMBOL, 227), + (31, HUFFMAN_EMIT_SYMBOL, 227), + (41, HUFFMAN_EMIT_SYMBOL, 227), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 227), + (3, HUFFMAN_EMIT_SYMBOL, 229), + (6, HUFFMAN_EMIT_SYMBOL, 229), + (10, HUFFMAN_EMIT_SYMBOL, 229), + (15, HUFFMAN_EMIT_SYMBOL, 229), + (24, HUFFMAN_EMIT_SYMBOL, 229), + (31, HUFFMAN_EMIT_SYMBOL, 229), + (41, HUFFMAN_EMIT_SYMBOL, 229), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 229), + + # Node 119 + (1, HUFFMAN_EMIT_SYMBOL, 230), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 230), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 129), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 132), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 133), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 134), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 136), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 146), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 154), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 156), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 160), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 163), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 164), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 169), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 170), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 173), + + # Node 120 + (2, HUFFMAN_EMIT_SYMBOL, 230), + (9, HUFFMAN_EMIT_SYMBOL, 230), + (23, HUFFMAN_EMIT_SYMBOL, 230), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 230), + (1, HUFFMAN_EMIT_SYMBOL, 129), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 129), + (1, HUFFMAN_EMIT_SYMBOL, 132), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 132), + (1, HUFFMAN_EMIT_SYMBOL, 133), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 133), + (1, HUFFMAN_EMIT_SYMBOL, 134), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 134), + (1, HUFFMAN_EMIT_SYMBOL, 136), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 136), + (1, HUFFMAN_EMIT_SYMBOL, 146), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 146), + + # Node 121 + (3, HUFFMAN_EMIT_SYMBOL, 230), + (6, HUFFMAN_EMIT_SYMBOL, 230), + (10, HUFFMAN_EMIT_SYMBOL, 230), + (15, HUFFMAN_EMIT_SYMBOL, 230), + (24, HUFFMAN_EMIT_SYMBOL, 230), + (31, HUFFMAN_EMIT_SYMBOL, 230), + (41, HUFFMAN_EMIT_SYMBOL, 230), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 230), + (2, HUFFMAN_EMIT_SYMBOL, 129), + (9, HUFFMAN_EMIT_SYMBOL, 129), + (23, HUFFMAN_EMIT_SYMBOL, 129), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 129), + (2, HUFFMAN_EMIT_SYMBOL, 132), + (9, HUFFMAN_EMIT_SYMBOL, 132), + (23, HUFFMAN_EMIT_SYMBOL, 132), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 132), + + # Node 122 + (3, HUFFMAN_EMIT_SYMBOL, 129), + (6, HUFFMAN_EMIT_SYMBOL, 129), + (10, HUFFMAN_EMIT_SYMBOL, 129), + (15, HUFFMAN_EMIT_SYMBOL, 129), + (24, HUFFMAN_EMIT_SYMBOL, 129), + (31, HUFFMAN_EMIT_SYMBOL, 129), + (41, HUFFMAN_EMIT_SYMBOL, 129), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 129), + (3, HUFFMAN_EMIT_SYMBOL, 132), + (6, HUFFMAN_EMIT_SYMBOL, 132), + (10, HUFFMAN_EMIT_SYMBOL, 132), + (15, HUFFMAN_EMIT_SYMBOL, 132), + (24, HUFFMAN_EMIT_SYMBOL, 132), + (31, HUFFMAN_EMIT_SYMBOL, 132), + (41, HUFFMAN_EMIT_SYMBOL, 132), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 132), + + # Node 123 + (2, HUFFMAN_EMIT_SYMBOL, 133), + (9, HUFFMAN_EMIT_SYMBOL, 133), + (23, HUFFMAN_EMIT_SYMBOL, 133), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 133), + (2, HUFFMAN_EMIT_SYMBOL, 134), + (9, HUFFMAN_EMIT_SYMBOL, 134), + (23, HUFFMAN_EMIT_SYMBOL, 134), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 134), + (2, HUFFMAN_EMIT_SYMBOL, 136), + (9, HUFFMAN_EMIT_SYMBOL, 136), + (23, HUFFMAN_EMIT_SYMBOL, 136), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 136), + (2, HUFFMAN_EMIT_SYMBOL, 146), + (9, HUFFMAN_EMIT_SYMBOL, 146), + (23, HUFFMAN_EMIT_SYMBOL, 146), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 146), + + # Node 124 + (3, HUFFMAN_EMIT_SYMBOL, 133), + (6, HUFFMAN_EMIT_SYMBOL, 133), + (10, HUFFMAN_EMIT_SYMBOL, 133), + (15, HUFFMAN_EMIT_SYMBOL, 133), + (24, HUFFMAN_EMIT_SYMBOL, 133), + (31, HUFFMAN_EMIT_SYMBOL, 133), + (41, HUFFMAN_EMIT_SYMBOL, 133), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 133), + (3, HUFFMAN_EMIT_SYMBOL, 134), + (6, HUFFMAN_EMIT_SYMBOL, 134), + (10, HUFFMAN_EMIT_SYMBOL, 134), + (15, HUFFMAN_EMIT_SYMBOL, 134), + (24, HUFFMAN_EMIT_SYMBOL, 134), + (31, HUFFMAN_EMIT_SYMBOL, 134), + (41, HUFFMAN_EMIT_SYMBOL, 134), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 134), + + # Node 125 + (3, HUFFMAN_EMIT_SYMBOL, 136), + (6, HUFFMAN_EMIT_SYMBOL, 136), + (10, HUFFMAN_EMIT_SYMBOL, 136), + (15, HUFFMAN_EMIT_SYMBOL, 136), + (24, HUFFMAN_EMIT_SYMBOL, 136), + (31, HUFFMAN_EMIT_SYMBOL, 136), + (41, HUFFMAN_EMIT_SYMBOL, 136), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 136), + (3, HUFFMAN_EMIT_SYMBOL, 146), + (6, HUFFMAN_EMIT_SYMBOL, 146), + (10, HUFFMAN_EMIT_SYMBOL, 146), + (15, HUFFMAN_EMIT_SYMBOL, 146), + (24, HUFFMAN_EMIT_SYMBOL, 146), + (31, HUFFMAN_EMIT_SYMBOL, 146), + (41, HUFFMAN_EMIT_SYMBOL, 146), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 146), + + # Node 126 + (1, HUFFMAN_EMIT_SYMBOL, 154), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 154), + (1, HUFFMAN_EMIT_SYMBOL, 156), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 156), + (1, HUFFMAN_EMIT_SYMBOL, 160), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 160), + (1, HUFFMAN_EMIT_SYMBOL, 163), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 163), + (1, HUFFMAN_EMIT_SYMBOL, 164), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 164), + (1, HUFFMAN_EMIT_SYMBOL, 169), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 169), + (1, HUFFMAN_EMIT_SYMBOL, 170), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 170), + (1, HUFFMAN_EMIT_SYMBOL, 173), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 173), + + # Node 127 + (2, HUFFMAN_EMIT_SYMBOL, 154), + (9, HUFFMAN_EMIT_SYMBOL, 154), + (23, HUFFMAN_EMIT_SYMBOL, 154), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 154), + (2, HUFFMAN_EMIT_SYMBOL, 156), + (9, HUFFMAN_EMIT_SYMBOL, 156), + (23, HUFFMAN_EMIT_SYMBOL, 156), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 156), + (2, HUFFMAN_EMIT_SYMBOL, 160), + (9, HUFFMAN_EMIT_SYMBOL, 160), + (23, HUFFMAN_EMIT_SYMBOL, 160), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 160), + (2, HUFFMAN_EMIT_SYMBOL, 163), + (9, HUFFMAN_EMIT_SYMBOL, 163), + (23, HUFFMAN_EMIT_SYMBOL, 163), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 163), + + # Node 128 + (3, HUFFMAN_EMIT_SYMBOL, 154), + (6, HUFFMAN_EMIT_SYMBOL, 154), + (10, HUFFMAN_EMIT_SYMBOL, 154), + (15, HUFFMAN_EMIT_SYMBOL, 154), + (24, HUFFMAN_EMIT_SYMBOL, 154), + (31, HUFFMAN_EMIT_SYMBOL, 154), + (41, HUFFMAN_EMIT_SYMBOL, 154), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 154), + (3, HUFFMAN_EMIT_SYMBOL, 156), + (6, HUFFMAN_EMIT_SYMBOL, 156), + (10, HUFFMAN_EMIT_SYMBOL, 156), + (15, HUFFMAN_EMIT_SYMBOL, 156), + (24, HUFFMAN_EMIT_SYMBOL, 156), + (31, HUFFMAN_EMIT_SYMBOL, 156), + (41, HUFFMAN_EMIT_SYMBOL, 156), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 156), + + # Node 129 + (3, HUFFMAN_EMIT_SYMBOL, 160), + (6, HUFFMAN_EMIT_SYMBOL, 160), + (10, HUFFMAN_EMIT_SYMBOL, 160), + (15, HUFFMAN_EMIT_SYMBOL, 160), + (24, HUFFMAN_EMIT_SYMBOL, 160), + (31, HUFFMAN_EMIT_SYMBOL, 160), + (41, HUFFMAN_EMIT_SYMBOL, 160), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 160), + (3, HUFFMAN_EMIT_SYMBOL, 163), + (6, HUFFMAN_EMIT_SYMBOL, 163), + (10, HUFFMAN_EMIT_SYMBOL, 163), + (15, HUFFMAN_EMIT_SYMBOL, 163), + (24, HUFFMAN_EMIT_SYMBOL, 163), + (31, HUFFMAN_EMIT_SYMBOL, 163), + (41, HUFFMAN_EMIT_SYMBOL, 163), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 163), + + # Node 130 + (2, HUFFMAN_EMIT_SYMBOL, 164), + (9, HUFFMAN_EMIT_SYMBOL, 164), + (23, HUFFMAN_EMIT_SYMBOL, 164), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 164), + (2, HUFFMAN_EMIT_SYMBOL, 169), + (9, HUFFMAN_EMIT_SYMBOL, 169), + (23, HUFFMAN_EMIT_SYMBOL, 169), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 169), + (2, HUFFMAN_EMIT_SYMBOL, 170), + (9, HUFFMAN_EMIT_SYMBOL, 170), + (23, HUFFMAN_EMIT_SYMBOL, 170), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 170), + (2, HUFFMAN_EMIT_SYMBOL, 173), + (9, HUFFMAN_EMIT_SYMBOL, 173), + (23, HUFFMAN_EMIT_SYMBOL, 173), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 173), + + # Node 131 + (3, HUFFMAN_EMIT_SYMBOL, 164), + (6, HUFFMAN_EMIT_SYMBOL, 164), + (10, HUFFMAN_EMIT_SYMBOL, 164), + (15, HUFFMAN_EMIT_SYMBOL, 164), + (24, HUFFMAN_EMIT_SYMBOL, 164), + (31, HUFFMAN_EMIT_SYMBOL, 164), + (41, HUFFMAN_EMIT_SYMBOL, 164), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 164), + (3, HUFFMAN_EMIT_SYMBOL, 169), + (6, HUFFMAN_EMIT_SYMBOL, 169), + (10, HUFFMAN_EMIT_SYMBOL, 169), + (15, HUFFMAN_EMIT_SYMBOL, 169), + (24, HUFFMAN_EMIT_SYMBOL, 169), + (31, HUFFMAN_EMIT_SYMBOL, 169), + (41, HUFFMAN_EMIT_SYMBOL, 169), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 169), + + # Node 132 + (3, HUFFMAN_EMIT_SYMBOL, 170), + (6, HUFFMAN_EMIT_SYMBOL, 170), + (10, HUFFMAN_EMIT_SYMBOL, 170), + (15, HUFFMAN_EMIT_SYMBOL, 170), + (24, HUFFMAN_EMIT_SYMBOL, 170), + (31, HUFFMAN_EMIT_SYMBOL, 170), + (41, HUFFMAN_EMIT_SYMBOL, 170), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 170), + (3, HUFFMAN_EMIT_SYMBOL, 173), + (6, HUFFMAN_EMIT_SYMBOL, 173), + (10, HUFFMAN_EMIT_SYMBOL, 173), + (15, HUFFMAN_EMIT_SYMBOL, 173), + (24, HUFFMAN_EMIT_SYMBOL, 173), + (31, HUFFMAN_EMIT_SYMBOL, 173), + (41, HUFFMAN_EMIT_SYMBOL, 173), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 173), + + # Node 133 + (137, 0, 0), + (138, 0, 0), + (140, 0, 0), + (141, 0, 0), + (144, 0, 0), + (145, 0, 0), + (147, 0, 0), + (150, 0, 0), + (156, 0, 0), + (159, 0, 0), + (163, 0, 0), + (166, 0, 0), + (171, 0, 0), + (174, 0, 0), + (181, 0, 0), + (190, 0, 0), + + # Node 134 + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 178), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 181), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 185), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 186), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 187), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 189), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 190), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 196), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 198), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 228), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 232), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 233), + (148, 0, 0), + (149, 0, 0), + (151, 0, 0), + (152, 0, 0), + + # Node 135 + (1, HUFFMAN_EMIT_SYMBOL, 178), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 178), + (1, HUFFMAN_EMIT_SYMBOL, 181), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 181), + (1, HUFFMAN_EMIT_SYMBOL, 185), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 185), + (1, HUFFMAN_EMIT_SYMBOL, 186), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 186), + (1, HUFFMAN_EMIT_SYMBOL, 187), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 187), + (1, HUFFMAN_EMIT_SYMBOL, 189), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 189), + (1, HUFFMAN_EMIT_SYMBOL, 190), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 190), + (1, HUFFMAN_EMIT_SYMBOL, 196), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 196), + + # Node 136 + (2, HUFFMAN_EMIT_SYMBOL, 178), + (9, HUFFMAN_EMIT_SYMBOL, 178), + (23, HUFFMAN_EMIT_SYMBOL, 178), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 178), + (2, HUFFMAN_EMIT_SYMBOL, 181), + (9, HUFFMAN_EMIT_SYMBOL, 181), + (23, HUFFMAN_EMIT_SYMBOL, 181), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 181), + (2, HUFFMAN_EMIT_SYMBOL, 185), + (9, HUFFMAN_EMIT_SYMBOL, 185), + (23, HUFFMAN_EMIT_SYMBOL, 185), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 185), + (2, HUFFMAN_EMIT_SYMBOL, 186), + (9, HUFFMAN_EMIT_SYMBOL, 186), + (23, HUFFMAN_EMIT_SYMBOL, 186), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 186), + + # Node 137 + (3, HUFFMAN_EMIT_SYMBOL, 178), + (6, HUFFMAN_EMIT_SYMBOL, 178), + (10, HUFFMAN_EMIT_SYMBOL, 178), + (15, HUFFMAN_EMIT_SYMBOL, 178), + (24, HUFFMAN_EMIT_SYMBOL, 178), + (31, HUFFMAN_EMIT_SYMBOL, 178), + (41, HUFFMAN_EMIT_SYMBOL, 178), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 178), + (3, HUFFMAN_EMIT_SYMBOL, 181), + (6, HUFFMAN_EMIT_SYMBOL, 181), + (10, HUFFMAN_EMIT_SYMBOL, 181), + (15, HUFFMAN_EMIT_SYMBOL, 181), + (24, HUFFMAN_EMIT_SYMBOL, 181), + (31, HUFFMAN_EMIT_SYMBOL, 181), + (41, HUFFMAN_EMIT_SYMBOL, 181), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 181), + + # Node 138 + (3, HUFFMAN_EMIT_SYMBOL, 185), + (6, HUFFMAN_EMIT_SYMBOL, 185), + (10, HUFFMAN_EMIT_SYMBOL, 185), + (15, HUFFMAN_EMIT_SYMBOL, 185), + (24, HUFFMAN_EMIT_SYMBOL, 185), + (31, HUFFMAN_EMIT_SYMBOL, 185), + (41, HUFFMAN_EMIT_SYMBOL, 185), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 185), + (3, HUFFMAN_EMIT_SYMBOL, 186), + (6, HUFFMAN_EMIT_SYMBOL, 186), + (10, HUFFMAN_EMIT_SYMBOL, 186), + (15, HUFFMAN_EMIT_SYMBOL, 186), + (24, HUFFMAN_EMIT_SYMBOL, 186), + (31, HUFFMAN_EMIT_SYMBOL, 186), + (41, HUFFMAN_EMIT_SYMBOL, 186), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 186), + + # Node 139 + (2, HUFFMAN_EMIT_SYMBOL, 187), + (9, HUFFMAN_EMIT_SYMBOL, 187), + (23, HUFFMAN_EMIT_SYMBOL, 187), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 187), + (2, HUFFMAN_EMIT_SYMBOL, 189), + (9, HUFFMAN_EMIT_SYMBOL, 189), + (23, HUFFMAN_EMIT_SYMBOL, 189), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 189), + (2, HUFFMAN_EMIT_SYMBOL, 190), + (9, HUFFMAN_EMIT_SYMBOL, 190), + (23, HUFFMAN_EMIT_SYMBOL, 190), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 190), + (2, HUFFMAN_EMIT_SYMBOL, 196), + (9, HUFFMAN_EMIT_SYMBOL, 196), + (23, HUFFMAN_EMIT_SYMBOL, 196), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 196), + + # Node 140 + (3, HUFFMAN_EMIT_SYMBOL, 187), + (6, HUFFMAN_EMIT_SYMBOL, 187), + (10, HUFFMAN_EMIT_SYMBOL, 187), + (15, HUFFMAN_EMIT_SYMBOL, 187), + (24, HUFFMAN_EMIT_SYMBOL, 187), + (31, HUFFMAN_EMIT_SYMBOL, 187), + (41, HUFFMAN_EMIT_SYMBOL, 187), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 187), + (3, HUFFMAN_EMIT_SYMBOL, 189), + (6, HUFFMAN_EMIT_SYMBOL, 189), + (10, HUFFMAN_EMIT_SYMBOL, 189), + (15, HUFFMAN_EMIT_SYMBOL, 189), + (24, HUFFMAN_EMIT_SYMBOL, 189), + (31, HUFFMAN_EMIT_SYMBOL, 189), + (41, HUFFMAN_EMIT_SYMBOL, 189), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 189), + + # Node 141 + (3, HUFFMAN_EMIT_SYMBOL, 190), + (6, HUFFMAN_EMIT_SYMBOL, 190), + (10, HUFFMAN_EMIT_SYMBOL, 190), + (15, HUFFMAN_EMIT_SYMBOL, 190), + (24, HUFFMAN_EMIT_SYMBOL, 190), + (31, HUFFMAN_EMIT_SYMBOL, 190), + (41, HUFFMAN_EMIT_SYMBOL, 190), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 190), + (3, HUFFMAN_EMIT_SYMBOL, 196), + (6, HUFFMAN_EMIT_SYMBOL, 196), + (10, HUFFMAN_EMIT_SYMBOL, 196), + (15, HUFFMAN_EMIT_SYMBOL, 196), + (24, HUFFMAN_EMIT_SYMBOL, 196), + (31, HUFFMAN_EMIT_SYMBOL, 196), + (41, HUFFMAN_EMIT_SYMBOL, 196), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 196), + + # Node 142 + (1, HUFFMAN_EMIT_SYMBOL, 198), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 198), + (1, HUFFMAN_EMIT_SYMBOL, 228), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 228), + (1, HUFFMAN_EMIT_SYMBOL, 232), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 232), + (1, HUFFMAN_EMIT_SYMBOL, 233), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 233), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 1), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 135), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 137), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 138), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 139), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 140), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 141), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 143), + + # Node 143 + (2, HUFFMAN_EMIT_SYMBOL, 198), + (9, HUFFMAN_EMIT_SYMBOL, 198), + (23, HUFFMAN_EMIT_SYMBOL, 198), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 198), + (2, HUFFMAN_EMIT_SYMBOL, 228), + (9, HUFFMAN_EMIT_SYMBOL, 228), + (23, HUFFMAN_EMIT_SYMBOL, 228), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 228), + (2, HUFFMAN_EMIT_SYMBOL, 232), + (9, HUFFMAN_EMIT_SYMBOL, 232), + (23, HUFFMAN_EMIT_SYMBOL, 232), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 232), + (2, HUFFMAN_EMIT_SYMBOL, 233), + (9, HUFFMAN_EMIT_SYMBOL, 233), + (23, HUFFMAN_EMIT_SYMBOL, 233), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 233), + + # Node 144 + (3, HUFFMAN_EMIT_SYMBOL, 198), + (6, HUFFMAN_EMIT_SYMBOL, 198), + (10, HUFFMAN_EMIT_SYMBOL, 198), + (15, HUFFMAN_EMIT_SYMBOL, 198), + (24, HUFFMAN_EMIT_SYMBOL, 198), + (31, HUFFMAN_EMIT_SYMBOL, 198), + (41, HUFFMAN_EMIT_SYMBOL, 198), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 198), + (3, HUFFMAN_EMIT_SYMBOL, 228), + (6, HUFFMAN_EMIT_SYMBOL, 228), + (10, HUFFMAN_EMIT_SYMBOL, 228), + (15, HUFFMAN_EMIT_SYMBOL, 228), + (24, HUFFMAN_EMIT_SYMBOL, 228), + (31, HUFFMAN_EMIT_SYMBOL, 228), + (41, HUFFMAN_EMIT_SYMBOL, 228), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 228), + + # Node 145 + (3, HUFFMAN_EMIT_SYMBOL, 232), + (6, HUFFMAN_EMIT_SYMBOL, 232), + (10, HUFFMAN_EMIT_SYMBOL, 232), + (15, HUFFMAN_EMIT_SYMBOL, 232), + (24, HUFFMAN_EMIT_SYMBOL, 232), + (31, HUFFMAN_EMIT_SYMBOL, 232), + (41, HUFFMAN_EMIT_SYMBOL, 232), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 232), + (3, HUFFMAN_EMIT_SYMBOL, 233), + (6, HUFFMAN_EMIT_SYMBOL, 233), + (10, HUFFMAN_EMIT_SYMBOL, 233), + (15, HUFFMAN_EMIT_SYMBOL, 233), + (24, HUFFMAN_EMIT_SYMBOL, 233), + (31, HUFFMAN_EMIT_SYMBOL, 233), + (41, HUFFMAN_EMIT_SYMBOL, 233), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 233), + + # Node 146 + (1, HUFFMAN_EMIT_SYMBOL, 1), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 1), + (1, HUFFMAN_EMIT_SYMBOL, 135), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 135), + (1, HUFFMAN_EMIT_SYMBOL, 137), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 137), + (1, HUFFMAN_EMIT_SYMBOL, 138), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 138), + (1, HUFFMAN_EMIT_SYMBOL, 139), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 139), + (1, HUFFMAN_EMIT_SYMBOL, 140), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 140), + (1, HUFFMAN_EMIT_SYMBOL, 141), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 141), + (1, HUFFMAN_EMIT_SYMBOL, 143), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 143), + + # Node 147 + (2, HUFFMAN_EMIT_SYMBOL, 1), + (9, HUFFMAN_EMIT_SYMBOL, 1), + (23, HUFFMAN_EMIT_SYMBOL, 1), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 1), + (2, HUFFMAN_EMIT_SYMBOL, 135), + (9, HUFFMAN_EMIT_SYMBOL, 135), + (23, HUFFMAN_EMIT_SYMBOL, 135), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 135), + (2, HUFFMAN_EMIT_SYMBOL, 137), + (9, HUFFMAN_EMIT_SYMBOL, 137), + (23, HUFFMAN_EMIT_SYMBOL, 137), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 137), + (2, HUFFMAN_EMIT_SYMBOL, 138), + (9, HUFFMAN_EMIT_SYMBOL, 138), + (23, HUFFMAN_EMIT_SYMBOL, 138), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 138), + + # Node 148 + (3, HUFFMAN_EMIT_SYMBOL, 1), + (6, HUFFMAN_EMIT_SYMBOL, 1), + (10, HUFFMAN_EMIT_SYMBOL, 1), + (15, HUFFMAN_EMIT_SYMBOL, 1), + (24, HUFFMAN_EMIT_SYMBOL, 1), + (31, HUFFMAN_EMIT_SYMBOL, 1), + (41, HUFFMAN_EMIT_SYMBOL, 1), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 1), + (3, HUFFMAN_EMIT_SYMBOL, 135), + (6, HUFFMAN_EMIT_SYMBOL, 135), + (10, HUFFMAN_EMIT_SYMBOL, 135), + (15, HUFFMAN_EMIT_SYMBOL, 135), + (24, HUFFMAN_EMIT_SYMBOL, 135), + (31, HUFFMAN_EMIT_SYMBOL, 135), + (41, HUFFMAN_EMIT_SYMBOL, 135), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 135), + + # Node 149 + (3, HUFFMAN_EMIT_SYMBOL, 137), + (6, HUFFMAN_EMIT_SYMBOL, 137), + (10, HUFFMAN_EMIT_SYMBOL, 137), + (15, HUFFMAN_EMIT_SYMBOL, 137), + (24, HUFFMAN_EMIT_SYMBOL, 137), + (31, HUFFMAN_EMIT_SYMBOL, 137), + (41, HUFFMAN_EMIT_SYMBOL, 137), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 137), + (3, HUFFMAN_EMIT_SYMBOL, 138), + (6, HUFFMAN_EMIT_SYMBOL, 138), + (10, HUFFMAN_EMIT_SYMBOL, 138), + (15, HUFFMAN_EMIT_SYMBOL, 138), + (24, HUFFMAN_EMIT_SYMBOL, 138), + (31, HUFFMAN_EMIT_SYMBOL, 138), + (41, HUFFMAN_EMIT_SYMBOL, 138), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 138), + + # Node 150 + (2, HUFFMAN_EMIT_SYMBOL, 139), + (9, HUFFMAN_EMIT_SYMBOL, 139), + (23, HUFFMAN_EMIT_SYMBOL, 139), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 139), + (2, HUFFMAN_EMIT_SYMBOL, 140), + (9, HUFFMAN_EMIT_SYMBOL, 140), + (23, HUFFMAN_EMIT_SYMBOL, 140), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 140), + (2, HUFFMAN_EMIT_SYMBOL, 141), + (9, HUFFMAN_EMIT_SYMBOL, 141), + (23, HUFFMAN_EMIT_SYMBOL, 141), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 141), + (2, HUFFMAN_EMIT_SYMBOL, 143), + (9, HUFFMAN_EMIT_SYMBOL, 143), + (23, HUFFMAN_EMIT_SYMBOL, 143), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 143), + + # Node 151 + (3, HUFFMAN_EMIT_SYMBOL, 139), + (6, HUFFMAN_EMIT_SYMBOL, 139), + (10, HUFFMAN_EMIT_SYMBOL, 139), + (15, HUFFMAN_EMIT_SYMBOL, 139), + (24, HUFFMAN_EMIT_SYMBOL, 139), + (31, HUFFMAN_EMIT_SYMBOL, 139), + (41, HUFFMAN_EMIT_SYMBOL, 139), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 139), + (3, HUFFMAN_EMIT_SYMBOL, 140), + (6, HUFFMAN_EMIT_SYMBOL, 140), + (10, HUFFMAN_EMIT_SYMBOL, 140), + (15, HUFFMAN_EMIT_SYMBOL, 140), + (24, HUFFMAN_EMIT_SYMBOL, 140), + (31, HUFFMAN_EMIT_SYMBOL, 140), + (41, HUFFMAN_EMIT_SYMBOL, 140), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 140), + + # Node 152 + (3, HUFFMAN_EMIT_SYMBOL, 141), + (6, HUFFMAN_EMIT_SYMBOL, 141), + (10, HUFFMAN_EMIT_SYMBOL, 141), + (15, HUFFMAN_EMIT_SYMBOL, 141), + (24, HUFFMAN_EMIT_SYMBOL, 141), + (31, HUFFMAN_EMIT_SYMBOL, 141), + (41, HUFFMAN_EMIT_SYMBOL, 141), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 141), + (3, HUFFMAN_EMIT_SYMBOL, 143), + (6, HUFFMAN_EMIT_SYMBOL, 143), + (10, HUFFMAN_EMIT_SYMBOL, 143), + (15, HUFFMAN_EMIT_SYMBOL, 143), + (24, HUFFMAN_EMIT_SYMBOL, 143), + (31, HUFFMAN_EMIT_SYMBOL, 143), + (41, HUFFMAN_EMIT_SYMBOL, 143), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 143), + + # Node 153 + (157, 0, 0), + (158, 0, 0), + (160, 0, 0), + (161, 0, 0), + (164, 0, 0), + (165, 0, 0), + (167, 0, 0), + (168, 0, 0), + (172, 0, 0), + (173, 0, 0), + (175, 0, 0), + (177, 0, 0), + (182, 0, 0), + (185, 0, 0), + (191, 0, 0), + (207, 0, 0), + + # Node 154 + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 147), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 149), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 150), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 151), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 152), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 155), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 157), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 158), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 165), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 166), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 168), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 174), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 175), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 180), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 182), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 183), + + # Node 155 + (1, HUFFMAN_EMIT_SYMBOL, 147), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 147), + (1, HUFFMAN_EMIT_SYMBOL, 149), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 149), + (1, HUFFMAN_EMIT_SYMBOL, 150), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 150), + (1, HUFFMAN_EMIT_SYMBOL, 151), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 151), + (1, HUFFMAN_EMIT_SYMBOL, 152), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 152), + (1, HUFFMAN_EMIT_SYMBOL, 155), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 155), + (1, HUFFMAN_EMIT_SYMBOL, 157), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 157), + (1, HUFFMAN_EMIT_SYMBOL, 158), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 158), + + # Node 156 + (2, HUFFMAN_EMIT_SYMBOL, 147), + (9, HUFFMAN_EMIT_SYMBOL, 147), + (23, HUFFMAN_EMIT_SYMBOL, 147), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 147), + (2, HUFFMAN_EMIT_SYMBOL, 149), + (9, HUFFMAN_EMIT_SYMBOL, 149), + (23, HUFFMAN_EMIT_SYMBOL, 149), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 149), + (2, HUFFMAN_EMIT_SYMBOL, 150), + (9, HUFFMAN_EMIT_SYMBOL, 150), + (23, HUFFMAN_EMIT_SYMBOL, 150), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 150), + (2, HUFFMAN_EMIT_SYMBOL, 151), + (9, HUFFMAN_EMIT_SYMBOL, 151), + (23, HUFFMAN_EMIT_SYMBOL, 151), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 151), + + # Node 157 + (3, HUFFMAN_EMIT_SYMBOL, 147), + (6, HUFFMAN_EMIT_SYMBOL, 147), + (10, HUFFMAN_EMIT_SYMBOL, 147), + (15, HUFFMAN_EMIT_SYMBOL, 147), + (24, HUFFMAN_EMIT_SYMBOL, 147), + (31, HUFFMAN_EMIT_SYMBOL, 147), + (41, HUFFMAN_EMIT_SYMBOL, 147), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 147), + (3, HUFFMAN_EMIT_SYMBOL, 149), + (6, HUFFMAN_EMIT_SYMBOL, 149), + (10, HUFFMAN_EMIT_SYMBOL, 149), + (15, HUFFMAN_EMIT_SYMBOL, 149), + (24, HUFFMAN_EMIT_SYMBOL, 149), + (31, HUFFMAN_EMIT_SYMBOL, 149), + (41, HUFFMAN_EMIT_SYMBOL, 149), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 149), + + # Node 158 + (3, HUFFMAN_EMIT_SYMBOL, 150), + (6, HUFFMAN_EMIT_SYMBOL, 150), + (10, HUFFMAN_EMIT_SYMBOL, 150), + (15, HUFFMAN_EMIT_SYMBOL, 150), + (24, HUFFMAN_EMIT_SYMBOL, 150), + (31, HUFFMAN_EMIT_SYMBOL, 150), + (41, HUFFMAN_EMIT_SYMBOL, 150), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 150), + (3, HUFFMAN_EMIT_SYMBOL, 151), + (6, HUFFMAN_EMIT_SYMBOL, 151), + (10, HUFFMAN_EMIT_SYMBOL, 151), + (15, HUFFMAN_EMIT_SYMBOL, 151), + (24, HUFFMAN_EMIT_SYMBOL, 151), + (31, HUFFMAN_EMIT_SYMBOL, 151), + (41, HUFFMAN_EMIT_SYMBOL, 151), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 151), + + # Node 159 + (2, HUFFMAN_EMIT_SYMBOL, 152), + (9, HUFFMAN_EMIT_SYMBOL, 152), + (23, HUFFMAN_EMIT_SYMBOL, 152), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 152), + (2, HUFFMAN_EMIT_SYMBOL, 155), + (9, HUFFMAN_EMIT_SYMBOL, 155), + (23, HUFFMAN_EMIT_SYMBOL, 155), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 155), + (2, HUFFMAN_EMIT_SYMBOL, 157), + (9, HUFFMAN_EMIT_SYMBOL, 157), + (23, HUFFMAN_EMIT_SYMBOL, 157), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 157), + (2, HUFFMAN_EMIT_SYMBOL, 158), + (9, HUFFMAN_EMIT_SYMBOL, 158), + (23, HUFFMAN_EMIT_SYMBOL, 158), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 158), + + # Node 160 + (3, HUFFMAN_EMIT_SYMBOL, 152), + (6, HUFFMAN_EMIT_SYMBOL, 152), + (10, HUFFMAN_EMIT_SYMBOL, 152), + (15, HUFFMAN_EMIT_SYMBOL, 152), + (24, HUFFMAN_EMIT_SYMBOL, 152), + (31, HUFFMAN_EMIT_SYMBOL, 152), + (41, HUFFMAN_EMIT_SYMBOL, 152), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 152), + (3, HUFFMAN_EMIT_SYMBOL, 155), + (6, HUFFMAN_EMIT_SYMBOL, 155), + (10, HUFFMAN_EMIT_SYMBOL, 155), + (15, HUFFMAN_EMIT_SYMBOL, 155), + (24, HUFFMAN_EMIT_SYMBOL, 155), + (31, HUFFMAN_EMIT_SYMBOL, 155), + (41, HUFFMAN_EMIT_SYMBOL, 155), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 155), + + # Node 161 + (3, HUFFMAN_EMIT_SYMBOL, 157), + (6, HUFFMAN_EMIT_SYMBOL, 157), + (10, HUFFMAN_EMIT_SYMBOL, 157), + (15, HUFFMAN_EMIT_SYMBOL, 157), + (24, HUFFMAN_EMIT_SYMBOL, 157), + (31, HUFFMAN_EMIT_SYMBOL, 157), + (41, HUFFMAN_EMIT_SYMBOL, 157), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 157), + (3, HUFFMAN_EMIT_SYMBOL, 158), + (6, HUFFMAN_EMIT_SYMBOL, 158), + (10, HUFFMAN_EMIT_SYMBOL, 158), + (15, HUFFMAN_EMIT_SYMBOL, 158), + (24, HUFFMAN_EMIT_SYMBOL, 158), + (31, HUFFMAN_EMIT_SYMBOL, 158), + (41, HUFFMAN_EMIT_SYMBOL, 158), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 158), + + # Node 162 + (1, HUFFMAN_EMIT_SYMBOL, 165), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 165), + (1, HUFFMAN_EMIT_SYMBOL, 166), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 166), + (1, HUFFMAN_EMIT_SYMBOL, 168), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 168), + (1, HUFFMAN_EMIT_SYMBOL, 174), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 174), + (1, HUFFMAN_EMIT_SYMBOL, 175), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 175), + (1, HUFFMAN_EMIT_SYMBOL, 180), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 180), + (1, HUFFMAN_EMIT_SYMBOL, 182), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 182), + (1, HUFFMAN_EMIT_SYMBOL, 183), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 183), + + # Node 163 + (2, HUFFMAN_EMIT_SYMBOL, 165), + (9, HUFFMAN_EMIT_SYMBOL, 165), + (23, HUFFMAN_EMIT_SYMBOL, 165), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 165), + (2, HUFFMAN_EMIT_SYMBOL, 166), + (9, HUFFMAN_EMIT_SYMBOL, 166), + (23, HUFFMAN_EMIT_SYMBOL, 166), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 166), + (2, HUFFMAN_EMIT_SYMBOL, 168), + (9, HUFFMAN_EMIT_SYMBOL, 168), + (23, HUFFMAN_EMIT_SYMBOL, 168), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 168), + (2, HUFFMAN_EMIT_SYMBOL, 174), + (9, HUFFMAN_EMIT_SYMBOL, 174), + (23, HUFFMAN_EMIT_SYMBOL, 174), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 174), + + # Node 164 + (3, HUFFMAN_EMIT_SYMBOL, 165), + (6, HUFFMAN_EMIT_SYMBOL, 165), + (10, HUFFMAN_EMIT_SYMBOL, 165), + (15, HUFFMAN_EMIT_SYMBOL, 165), + (24, HUFFMAN_EMIT_SYMBOL, 165), + (31, HUFFMAN_EMIT_SYMBOL, 165), + (41, HUFFMAN_EMIT_SYMBOL, 165), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 165), + (3, HUFFMAN_EMIT_SYMBOL, 166), + (6, HUFFMAN_EMIT_SYMBOL, 166), + (10, HUFFMAN_EMIT_SYMBOL, 166), + (15, HUFFMAN_EMIT_SYMBOL, 166), + (24, HUFFMAN_EMIT_SYMBOL, 166), + (31, HUFFMAN_EMIT_SYMBOL, 166), + (41, HUFFMAN_EMIT_SYMBOL, 166), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 166), + + # Node 165 + (3, HUFFMAN_EMIT_SYMBOL, 168), + (6, HUFFMAN_EMIT_SYMBOL, 168), + (10, HUFFMAN_EMIT_SYMBOL, 168), + (15, HUFFMAN_EMIT_SYMBOL, 168), + (24, HUFFMAN_EMIT_SYMBOL, 168), + (31, HUFFMAN_EMIT_SYMBOL, 168), + (41, HUFFMAN_EMIT_SYMBOL, 168), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 168), + (3, HUFFMAN_EMIT_SYMBOL, 174), + (6, HUFFMAN_EMIT_SYMBOL, 174), + (10, HUFFMAN_EMIT_SYMBOL, 174), + (15, HUFFMAN_EMIT_SYMBOL, 174), + (24, HUFFMAN_EMIT_SYMBOL, 174), + (31, HUFFMAN_EMIT_SYMBOL, 174), + (41, HUFFMAN_EMIT_SYMBOL, 174), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 174), + + # Node 166 + (2, HUFFMAN_EMIT_SYMBOL, 175), + (9, HUFFMAN_EMIT_SYMBOL, 175), + (23, HUFFMAN_EMIT_SYMBOL, 175), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 175), + (2, HUFFMAN_EMIT_SYMBOL, 180), + (9, HUFFMAN_EMIT_SYMBOL, 180), + (23, HUFFMAN_EMIT_SYMBOL, 180), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 180), + (2, HUFFMAN_EMIT_SYMBOL, 182), + (9, HUFFMAN_EMIT_SYMBOL, 182), + (23, HUFFMAN_EMIT_SYMBOL, 182), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 182), + (2, HUFFMAN_EMIT_SYMBOL, 183), + (9, HUFFMAN_EMIT_SYMBOL, 183), + (23, HUFFMAN_EMIT_SYMBOL, 183), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 183), + + # Node 167 + (3, HUFFMAN_EMIT_SYMBOL, 175), + (6, HUFFMAN_EMIT_SYMBOL, 175), + (10, HUFFMAN_EMIT_SYMBOL, 175), + (15, HUFFMAN_EMIT_SYMBOL, 175), + (24, HUFFMAN_EMIT_SYMBOL, 175), + (31, HUFFMAN_EMIT_SYMBOL, 175), + (41, HUFFMAN_EMIT_SYMBOL, 175), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 175), + (3, HUFFMAN_EMIT_SYMBOL, 180), + (6, HUFFMAN_EMIT_SYMBOL, 180), + (10, HUFFMAN_EMIT_SYMBOL, 180), + (15, HUFFMAN_EMIT_SYMBOL, 180), + (24, HUFFMAN_EMIT_SYMBOL, 180), + (31, HUFFMAN_EMIT_SYMBOL, 180), + (41, HUFFMAN_EMIT_SYMBOL, 180), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 180), + + # Node 168 + (3, HUFFMAN_EMIT_SYMBOL, 182), + (6, HUFFMAN_EMIT_SYMBOL, 182), + (10, HUFFMAN_EMIT_SYMBOL, 182), + (15, HUFFMAN_EMIT_SYMBOL, 182), + (24, HUFFMAN_EMIT_SYMBOL, 182), + (31, HUFFMAN_EMIT_SYMBOL, 182), + (41, HUFFMAN_EMIT_SYMBOL, 182), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 182), + (3, HUFFMAN_EMIT_SYMBOL, 183), + (6, HUFFMAN_EMIT_SYMBOL, 183), + (10, HUFFMAN_EMIT_SYMBOL, 183), + (15, HUFFMAN_EMIT_SYMBOL, 183), + (24, HUFFMAN_EMIT_SYMBOL, 183), + (31, HUFFMAN_EMIT_SYMBOL, 183), + (41, HUFFMAN_EMIT_SYMBOL, 183), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 183), + + # Node 169 + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 188), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 191), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 197), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 231), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 239), + (176, 0, 0), + (178, 0, 0), + (179, 0, 0), + (183, 0, 0), + (184, 0, 0), + (186, 0, 0), + (187, 0, 0), + (192, 0, 0), + (199, 0, 0), + (208, 0, 0), + (223, 0, 0), + + # Node 170 + (1, HUFFMAN_EMIT_SYMBOL, 188), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 188), + (1, HUFFMAN_EMIT_SYMBOL, 191), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 191), + (1, HUFFMAN_EMIT_SYMBOL, 197), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 197), + (1, HUFFMAN_EMIT_SYMBOL, 231), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 231), + (1, HUFFMAN_EMIT_SYMBOL, 239), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 239), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 9), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 142), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 144), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 145), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 148), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 159), + + # Node 171 + (2, HUFFMAN_EMIT_SYMBOL, 188), + (9, HUFFMAN_EMIT_SYMBOL, 188), + (23, HUFFMAN_EMIT_SYMBOL, 188), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 188), + (2, HUFFMAN_EMIT_SYMBOL, 191), + (9, HUFFMAN_EMIT_SYMBOL, 191), + (23, HUFFMAN_EMIT_SYMBOL, 191), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 191), + (2, HUFFMAN_EMIT_SYMBOL, 197), + (9, HUFFMAN_EMIT_SYMBOL, 197), + (23, HUFFMAN_EMIT_SYMBOL, 197), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 197), + (2, HUFFMAN_EMIT_SYMBOL, 231), + (9, HUFFMAN_EMIT_SYMBOL, 231), + (23, HUFFMAN_EMIT_SYMBOL, 231), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 231), + + # Node 172 + (3, HUFFMAN_EMIT_SYMBOL, 188), + (6, HUFFMAN_EMIT_SYMBOL, 188), + (10, HUFFMAN_EMIT_SYMBOL, 188), + (15, HUFFMAN_EMIT_SYMBOL, 188), + (24, HUFFMAN_EMIT_SYMBOL, 188), + (31, HUFFMAN_EMIT_SYMBOL, 188), + (41, HUFFMAN_EMIT_SYMBOL, 188), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 188), + (3, HUFFMAN_EMIT_SYMBOL, 191), + (6, HUFFMAN_EMIT_SYMBOL, 191), + (10, HUFFMAN_EMIT_SYMBOL, 191), + (15, HUFFMAN_EMIT_SYMBOL, 191), + (24, HUFFMAN_EMIT_SYMBOL, 191), + (31, HUFFMAN_EMIT_SYMBOL, 191), + (41, HUFFMAN_EMIT_SYMBOL, 191), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 191), + + # Node 173 + (3, HUFFMAN_EMIT_SYMBOL, 197), + (6, HUFFMAN_EMIT_SYMBOL, 197), + (10, HUFFMAN_EMIT_SYMBOL, 197), + (15, HUFFMAN_EMIT_SYMBOL, 197), + (24, HUFFMAN_EMIT_SYMBOL, 197), + (31, HUFFMAN_EMIT_SYMBOL, 197), + (41, HUFFMAN_EMIT_SYMBOL, 197), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 197), + (3, HUFFMAN_EMIT_SYMBOL, 231), + (6, HUFFMAN_EMIT_SYMBOL, 231), + (10, HUFFMAN_EMIT_SYMBOL, 231), + (15, HUFFMAN_EMIT_SYMBOL, 231), + (24, HUFFMAN_EMIT_SYMBOL, 231), + (31, HUFFMAN_EMIT_SYMBOL, 231), + (41, HUFFMAN_EMIT_SYMBOL, 231), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 231), + + # Node 174 + (2, HUFFMAN_EMIT_SYMBOL, 239), + (9, HUFFMAN_EMIT_SYMBOL, 239), + (23, HUFFMAN_EMIT_SYMBOL, 239), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 239), + (1, HUFFMAN_EMIT_SYMBOL, 9), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 9), + (1, HUFFMAN_EMIT_SYMBOL, 142), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 142), + (1, HUFFMAN_EMIT_SYMBOL, 144), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 144), + (1, HUFFMAN_EMIT_SYMBOL, 145), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 145), + (1, HUFFMAN_EMIT_SYMBOL, 148), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 148), + (1, HUFFMAN_EMIT_SYMBOL, 159), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 159), + + # Node 175 + (3, HUFFMAN_EMIT_SYMBOL, 239), + (6, HUFFMAN_EMIT_SYMBOL, 239), + (10, HUFFMAN_EMIT_SYMBOL, 239), + (15, HUFFMAN_EMIT_SYMBOL, 239), + (24, HUFFMAN_EMIT_SYMBOL, 239), + (31, HUFFMAN_EMIT_SYMBOL, 239), + (41, HUFFMAN_EMIT_SYMBOL, 239), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 239), + (2, HUFFMAN_EMIT_SYMBOL, 9), + (9, HUFFMAN_EMIT_SYMBOL, 9), + (23, HUFFMAN_EMIT_SYMBOL, 9), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 9), + (2, HUFFMAN_EMIT_SYMBOL, 142), + (9, HUFFMAN_EMIT_SYMBOL, 142), + (23, HUFFMAN_EMIT_SYMBOL, 142), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 142), + + # Node 176 + (3, HUFFMAN_EMIT_SYMBOL, 9), + (6, HUFFMAN_EMIT_SYMBOL, 9), + (10, HUFFMAN_EMIT_SYMBOL, 9), + (15, HUFFMAN_EMIT_SYMBOL, 9), + (24, HUFFMAN_EMIT_SYMBOL, 9), + (31, HUFFMAN_EMIT_SYMBOL, 9), + (41, HUFFMAN_EMIT_SYMBOL, 9), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 9), + (3, HUFFMAN_EMIT_SYMBOL, 142), + (6, HUFFMAN_EMIT_SYMBOL, 142), + (10, HUFFMAN_EMIT_SYMBOL, 142), + (15, HUFFMAN_EMIT_SYMBOL, 142), + (24, HUFFMAN_EMIT_SYMBOL, 142), + (31, HUFFMAN_EMIT_SYMBOL, 142), + (41, HUFFMAN_EMIT_SYMBOL, 142), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 142), + + # Node 177 + (2, HUFFMAN_EMIT_SYMBOL, 144), + (9, HUFFMAN_EMIT_SYMBOL, 144), + (23, HUFFMAN_EMIT_SYMBOL, 144), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 144), + (2, HUFFMAN_EMIT_SYMBOL, 145), + (9, HUFFMAN_EMIT_SYMBOL, 145), + (23, HUFFMAN_EMIT_SYMBOL, 145), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 145), + (2, HUFFMAN_EMIT_SYMBOL, 148), + (9, HUFFMAN_EMIT_SYMBOL, 148), + (23, HUFFMAN_EMIT_SYMBOL, 148), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 148), + (2, HUFFMAN_EMIT_SYMBOL, 159), + (9, HUFFMAN_EMIT_SYMBOL, 159), + (23, HUFFMAN_EMIT_SYMBOL, 159), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 159), + + # Node 178 + (3, HUFFMAN_EMIT_SYMBOL, 144), + (6, HUFFMAN_EMIT_SYMBOL, 144), + (10, HUFFMAN_EMIT_SYMBOL, 144), + (15, HUFFMAN_EMIT_SYMBOL, 144), + (24, HUFFMAN_EMIT_SYMBOL, 144), + (31, HUFFMAN_EMIT_SYMBOL, 144), + (41, HUFFMAN_EMIT_SYMBOL, 144), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 144), + (3, HUFFMAN_EMIT_SYMBOL, 145), + (6, HUFFMAN_EMIT_SYMBOL, 145), + (10, HUFFMAN_EMIT_SYMBOL, 145), + (15, HUFFMAN_EMIT_SYMBOL, 145), + (24, HUFFMAN_EMIT_SYMBOL, 145), + (31, HUFFMAN_EMIT_SYMBOL, 145), + (41, HUFFMAN_EMIT_SYMBOL, 145), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 145), + + # Node 179 + (3, HUFFMAN_EMIT_SYMBOL, 148), + (6, HUFFMAN_EMIT_SYMBOL, 148), + (10, HUFFMAN_EMIT_SYMBOL, 148), + (15, HUFFMAN_EMIT_SYMBOL, 148), + (24, HUFFMAN_EMIT_SYMBOL, 148), + (31, HUFFMAN_EMIT_SYMBOL, 148), + (41, HUFFMAN_EMIT_SYMBOL, 148), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 148), + (3, HUFFMAN_EMIT_SYMBOL, 159), + (6, HUFFMAN_EMIT_SYMBOL, 159), + (10, HUFFMAN_EMIT_SYMBOL, 159), + (15, HUFFMAN_EMIT_SYMBOL, 159), + (24, HUFFMAN_EMIT_SYMBOL, 159), + (31, HUFFMAN_EMIT_SYMBOL, 159), + (41, HUFFMAN_EMIT_SYMBOL, 159), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 159), + + # Node 180 + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 171), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 206), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 215), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 225), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 236), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 237), + (188, 0, 0), + (189, 0, 0), + (193, 0, 0), + (196, 0, 0), + (200, 0, 0), + (203, 0, 0), + (209, 0, 0), + (216, 0, 0), + (224, 0, 0), + (238, 0, 0), + + # Node 181 + (1, HUFFMAN_EMIT_SYMBOL, 171), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 171), + (1, HUFFMAN_EMIT_SYMBOL, 206), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 206), + (1, HUFFMAN_EMIT_SYMBOL, 215), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 215), + (1, HUFFMAN_EMIT_SYMBOL, 225), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 225), + (1, HUFFMAN_EMIT_SYMBOL, 236), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 236), + (1, HUFFMAN_EMIT_SYMBOL, 237), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 237), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 199), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 207), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 234), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 235), + + # Node 182 + (2, HUFFMAN_EMIT_SYMBOL, 171), + (9, HUFFMAN_EMIT_SYMBOL, 171), + (23, HUFFMAN_EMIT_SYMBOL, 171), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 171), + (2, HUFFMAN_EMIT_SYMBOL, 206), + (9, HUFFMAN_EMIT_SYMBOL, 206), + (23, HUFFMAN_EMIT_SYMBOL, 206), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 206), + (2, HUFFMAN_EMIT_SYMBOL, 215), + (9, HUFFMAN_EMIT_SYMBOL, 215), + (23, HUFFMAN_EMIT_SYMBOL, 215), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 215), + (2, HUFFMAN_EMIT_SYMBOL, 225), + (9, HUFFMAN_EMIT_SYMBOL, 225), + (23, HUFFMAN_EMIT_SYMBOL, 225), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 225), + + # Node 183 + (3, HUFFMAN_EMIT_SYMBOL, 171), + (6, HUFFMAN_EMIT_SYMBOL, 171), + (10, HUFFMAN_EMIT_SYMBOL, 171), + (15, HUFFMAN_EMIT_SYMBOL, 171), + (24, HUFFMAN_EMIT_SYMBOL, 171), + (31, HUFFMAN_EMIT_SYMBOL, 171), + (41, HUFFMAN_EMIT_SYMBOL, 171), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 171), + (3, HUFFMAN_EMIT_SYMBOL, 206), + (6, HUFFMAN_EMIT_SYMBOL, 206), + (10, HUFFMAN_EMIT_SYMBOL, 206), + (15, HUFFMAN_EMIT_SYMBOL, 206), + (24, HUFFMAN_EMIT_SYMBOL, 206), + (31, HUFFMAN_EMIT_SYMBOL, 206), + (41, HUFFMAN_EMIT_SYMBOL, 206), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 206), + + # Node 184 + (3, HUFFMAN_EMIT_SYMBOL, 215), + (6, HUFFMAN_EMIT_SYMBOL, 215), + (10, HUFFMAN_EMIT_SYMBOL, 215), + (15, HUFFMAN_EMIT_SYMBOL, 215), + (24, HUFFMAN_EMIT_SYMBOL, 215), + (31, HUFFMAN_EMIT_SYMBOL, 215), + (41, HUFFMAN_EMIT_SYMBOL, 215), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 215), + (3, HUFFMAN_EMIT_SYMBOL, 225), + (6, HUFFMAN_EMIT_SYMBOL, 225), + (10, HUFFMAN_EMIT_SYMBOL, 225), + (15, HUFFMAN_EMIT_SYMBOL, 225), + (24, HUFFMAN_EMIT_SYMBOL, 225), + (31, HUFFMAN_EMIT_SYMBOL, 225), + (41, HUFFMAN_EMIT_SYMBOL, 225), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 225), + + # Node 185 + (2, HUFFMAN_EMIT_SYMBOL, 236), + (9, HUFFMAN_EMIT_SYMBOL, 236), + (23, HUFFMAN_EMIT_SYMBOL, 236), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 236), + (2, HUFFMAN_EMIT_SYMBOL, 237), + (9, HUFFMAN_EMIT_SYMBOL, 237), + (23, HUFFMAN_EMIT_SYMBOL, 237), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 237), + (1, HUFFMAN_EMIT_SYMBOL, 199), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 199), + (1, HUFFMAN_EMIT_SYMBOL, 207), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 207), + (1, HUFFMAN_EMIT_SYMBOL, 234), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 234), + (1, HUFFMAN_EMIT_SYMBOL, 235), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 235), + + # Node 186 + (3, HUFFMAN_EMIT_SYMBOL, 236), + (6, HUFFMAN_EMIT_SYMBOL, 236), + (10, HUFFMAN_EMIT_SYMBOL, 236), + (15, HUFFMAN_EMIT_SYMBOL, 236), + (24, HUFFMAN_EMIT_SYMBOL, 236), + (31, HUFFMAN_EMIT_SYMBOL, 236), + (41, HUFFMAN_EMIT_SYMBOL, 236), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 236), + (3, HUFFMAN_EMIT_SYMBOL, 237), + (6, HUFFMAN_EMIT_SYMBOL, 237), + (10, HUFFMAN_EMIT_SYMBOL, 237), + (15, HUFFMAN_EMIT_SYMBOL, 237), + (24, HUFFMAN_EMIT_SYMBOL, 237), + (31, HUFFMAN_EMIT_SYMBOL, 237), + (41, HUFFMAN_EMIT_SYMBOL, 237), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 237), + + # Node 187 + (2, HUFFMAN_EMIT_SYMBOL, 199), + (9, HUFFMAN_EMIT_SYMBOL, 199), + (23, HUFFMAN_EMIT_SYMBOL, 199), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 199), + (2, HUFFMAN_EMIT_SYMBOL, 207), + (9, HUFFMAN_EMIT_SYMBOL, 207), + (23, HUFFMAN_EMIT_SYMBOL, 207), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 207), + (2, HUFFMAN_EMIT_SYMBOL, 234), + (9, HUFFMAN_EMIT_SYMBOL, 234), + (23, HUFFMAN_EMIT_SYMBOL, 234), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 234), + (2, HUFFMAN_EMIT_SYMBOL, 235), + (9, HUFFMAN_EMIT_SYMBOL, 235), + (23, HUFFMAN_EMIT_SYMBOL, 235), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 235), + + # Node 188 + (3, HUFFMAN_EMIT_SYMBOL, 199), + (6, HUFFMAN_EMIT_SYMBOL, 199), + (10, HUFFMAN_EMIT_SYMBOL, 199), + (15, HUFFMAN_EMIT_SYMBOL, 199), + (24, HUFFMAN_EMIT_SYMBOL, 199), + (31, HUFFMAN_EMIT_SYMBOL, 199), + (41, HUFFMAN_EMIT_SYMBOL, 199), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 199), + (3, HUFFMAN_EMIT_SYMBOL, 207), + (6, HUFFMAN_EMIT_SYMBOL, 207), + (10, HUFFMAN_EMIT_SYMBOL, 207), + (15, HUFFMAN_EMIT_SYMBOL, 207), + (24, HUFFMAN_EMIT_SYMBOL, 207), + (31, HUFFMAN_EMIT_SYMBOL, 207), + (41, HUFFMAN_EMIT_SYMBOL, 207), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 207), + + # Node 189 + (3, HUFFMAN_EMIT_SYMBOL, 234), + (6, HUFFMAN_EMIT_SYMBOL, 234), + (10, HUFFMAN_EMIT_SYMBOL, 234), + (15, HUFFMAN_EMIT_SYMBOL, 234), + (24, HUFFMAN_EMIT_SYMBOL, 234), + (31, HUFFMAN_EMIT_SYMBOL, 234), + (41, HUFFMAN_EMIT_SYMBOL, 234), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 234), + (3, HUFFMAN_EMIT_SYMBOL, 235), + (6, HUFFMAN_EMIT_SYMBOL, 235), + (10, HUFFMAN_EMIT_SYMBOL, 235), + (15, HUFFMAN_EMIT_SYMBOL, 235), + (24, HUFFMAN_EMIT_SYMBOL, 235), + (31, HUFFMAN_EMIT_SYMBOL, 235), + (41, HUFFMAN_EMIT_SYMBOL, 235), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 235), + + # Node 190 + (194, 0, 0), + (195, 0, 0), + (197, 0, 0), + (198, 0, 0), + (201, 0, 0), + (202, 0, 0), + (204, 0, 0), + (205, 0, 0), + (210, 0, 0), + (213, 0, 0), + (217, 0, 0), + (220, 0, 0), + (225, 0, 0), + (231, 0, 0), + (239, 0, 0), + (246, 0, 0), + + # Node 191 + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 192), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 193), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 200), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 201), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 202), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 205), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 210), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 213), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 218), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 219), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 238), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 240), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 242), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 243), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 255), + (206, 0, 0), + + # Node 192 + (1, HUFFMAN_EMIT_SYMBOL, 192), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 192), + (1, HUFFMAN_EMIT_SYMBOL, 193), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 193), + (1, HUFFMAN_EMIT_SYMBOL, 200), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 200), + (1, HUFFMAN_EMIT_SYMBOL, 201), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 201), + (1, HUFFMAN_EMIT_SYMBOL, 202), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 202), + (1, HUFFMAN_EMIT_SYMBOL, 205), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 205), + (1, HUFFMAN_EMIT_SYMBOL, 210), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 210), + (1, HUFFMAN_EMIT_SYMBOL, 213), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 213), + + # Node 193 + (2, HUFFMAN_EMIT_SYMBOL, 192), + (9, HUFFMAN_EMIT_SYMBOL, 192), + (23, HUFFMAN_EMIT_SYMBOL, 192), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 192), + (2, HUFFMAN_EMIT_SYMBOL, 193), + (9, HUFFMAN_EMIT_SYMBOL, 193), + (23, HUFFMAN_EMIT_SYMBOL, 193), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 193), + (2, HUFFMAN_EMIT_SYMBOL, 200), + (9, HUFFMAN_EMIT_SYMBOL, 200), + (23, HUFFMAN_EMIT_SYMBOL, 200), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 200), + (2, HUFFMAN_EMIT_SYMBOL, 201), + (9, HUFFMAN_EMIT_SYMBOL, 201), + (23, HUFFMAN_EMIT_SYMBOL, 201), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 201), + + # Node 194 + (3, HUFFMAN_EMIT_SYMBOL, 192), + (6, HUFFMAN_EMIT_SYMBOL, 192), + (10, HUFFMAN_EMIT_SYMBOL, 192), + (15, HUFFMAN_EMIT_SYMBOL, 192), + (24, HUFFMAN_EMIT_SYMBOL, 192), + (31, HUFFMAN_EMIT_SYMBOL, 192), + (41, HUFFMAN_EMIT_SYMBOL, 192), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 192), + (3, HUFFMAN_EMIT_SYMBOL, 193), + (6, HUFFMAN_EMIT_SYMBOL, 193), + (10, HUFFMAN_EMIT_SYMBOL, 193), + (15, HUFFMAN_EMIT_SYMBOL, 193), + (24, HUFFMAN_EMIT_SYMBOL, 193), + (31, HUFFMAN_EMIT_SYMBOL, 193), + (41, HUFFMAN_EMIT_SYMBOL, 193), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 193), + + # Node 195 + (3, HUFFMAN_EMIT_SYMBOL, 200), + (6, HUFFMAN_EMIT_SYMBOL, 200), + (10, HUFFMAN_EMIT_SYMBOL, 200), + (15, HUFFMAN_EMIT_SYMBOL, 200), + (24, HUFFMAN_EMIT_SYMBOL, 200), + (31, HUFFMAN_EMIT_SYMBOL, 200), + (41, HUFFMAN_EMIT_SYMBOL, 200), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 200), + (3, HUFFMAN_EMIT_SYMBOL, 201), + (6, HUFFMAN_EMIT_SYMBOL, 201), + (10, HUFFMAN_EMIT_SYMBOL, 201), + (15, HUFFMAN_EMIT_SYMBOL, 201), + (24, HUFFMAN_EMIT_SYMBOL, 201), + (31, HUFFMAN_EMIT_SYMBOL, 201), + (41, HUFFMAN_EMIT_SYMBOL, 201), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 201), + + # Node 196 + (2, HUFFMAN_EMIT_SYMBOL, 202), + (9, HUFFMAN_EMIT_SYMBOL, 202), + (23, HUFFMAN_EMIT_SYMBOL, 202), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 202), + (2, HUFFMAN_EMIT_SYMBOL, 205), + (9, HUFFMAN_EMIT_SYMBOL, 205), + (23, HUFFMAN_EMIT_SYMBOL, 205), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 205), + (2, HUFFMAN_EMIT_SYMBOL, 210), + (9, HUFFMAN_EMIT_SYMBOL, 210), + (23, HUFFMAN_EMIT_SYMBOL, 210), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 210), + (2, HUFFMAN_EMIT_SYMBOL, 213), + (9, HUFFMAN_EMIT_SYMBOL, 213), + (23, HUFFMAN_EMIT_SYMBOL, 213), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 213), + + # Node 197 + (3, HUFFMAN_EMIT_SYMBOL, 202), + (6, HUFFMAN_EMIT_SYMBOL, 202), + (10, HUFFMAN_EMIT_SYMBOL, 202), + (15, HUFFMAN_EMIT_SYMBOL, 202), + (24, HUFFMAN_EMIT_SYMBOL, 202), + (31, HUFFMAN_EMIT_SYMBOL, 202), + (41, HUFFMAN_EMIT_SYMBOL, 202), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 202), + (3, HUFFMAN_EMIT_SYMBOL, 205), + (6, HUFFMAN_EMIT_SYMBOL, 205), + (10, HUFFMAN_EMIT_SYMBOL, 205), + (15, HUFFMAN_EMIT_SYMBOL, 205), + (24, HUFFMAN_EMIT_SYMBOL, 205), + (31, HUFFMAN_EMIT_SYMBOL, 205), + (41, HUFFMAN_EMIT_SYMBOL, 205), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 205), + + # Node 198 + (3, HUFFMAN_EMIT_SYMBOL, 210), + (6, HUFFMAN_EMIT_SYMBOL, 210), + (10, HUFFMAN_EMIT_SYMBOL, 210), + (15, HUFFMAN_EMIT_SYMBOL, 210), + (24, HUFFMAN_EMIT_SYMBOL, 210), + (31, HUFFMAN_EMIT_SYMBOL, 210), + (41, HUFFMAN_EMIT_SYMBOL, 210), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 210), + (3, HUFFMAN_EMIT_SYMBOL, 213), + (6, HUFFMAN_EMIT_SYMBOL, 213), + (10, HUFFMAN_EMIT_SYMBOL, 213), + (15, HUFFMAN_EMIT_SYMBOL, 213), + (24, HUFFMAN_EMIT_SYMBOL, 213), + (31, HUFFMAN_EMIT_SYMBOL, 213), + (41, HUFFMAN_EMIT_SYMBOL, 213), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 213), + + # Node 199 + (1, HUFFMAN_EMIT_SYMBOL, 218), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 218), + (1, HUFFMAN_EMIT_SYMBOL, 219), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 219), + (1, HUFFMAN_EMIT_SYMBOL, 238), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 238), + (1, HUFFMAN_EMIT_SYMBOL, 240), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 240), + (1, HUFFMAN_EMIT_SYMBOL, 242), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 242), + (1, HUFFMAN_EMIT_SYMBOL, 243), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 243), + (1, HUFFMAN_EMIT_SYMBOL, 255), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 255), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 203), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 204), + + # Node 200 + (2, HUFFMAN_EMIT_SYMBOL, 218), + (9, HUFFMAN_EMIT_SYMBOL, 218), + (23, HUFFMAN_EMIT_SYMBOL, 218), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 218), + (2, HUFFMAN_EMIT_SYMBOL, 219), + (9, HUFFMAN_EMIT_SYMBOL, 219), + (23, HUFFMAN_EMIT_SYMBOL, 219), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 219), + (2, HUFFMAN_EMIT_SYMBOL, 238), + (9, HUFFMAN_EMIT_SYMBOL, 238), + (23, HUFFMAN_EMIT_SYMBOL, 238), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 238), + (2, HUFFMAN_EMIT_SYMBOL, 240), + (9, HUFFMAN_EMIT_SYMBOL, 240), + (23, HUFFMAN_EMIT_SYMBOL, 240), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 240), + + # Node 201 + (3, HUFFMAN_EMIT_SYMBOL, 218), + (6, HUFFMAN_EMIT_SYMBOL, 218), + (10, HUFFMAN_EMIT_SYMBOL, 218), + (15, HUFFMAN_EMIT_SYMBOL, 218), + (24, HUFFMAN_EMIT_SYMBOL, 218), + (31, HUFFMAN_EMIT_SYMBOL, 218), + (41, HUFFMAN_EMIT_SYMBOL, 218), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 218), + (3, HUFFMAN_EMIT_SYMBOL, 219), + (6, HUFFMAN_EMIT_SYMBOL, 219), + (10, HUFFMAN_EMIT_SYMBOL, 219), + (15, HUFFMAN_EMIT_SYMBOL, 219), + (24, HUFFMAN_EMIT_SYMBOL, 219), + (31, HUFFMAN_EMIT_SYMBOL, 219), + (41, HUFFMAN_EMIT_SYMBOL, 219), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 219), + + # Node 202 + (3, HUFFMAN_EMIT_SYMBOL, 238), + (6, HUFFMAN_EMIT_SYMBOL, 238), + (10, HUFFMAN_EMIT_SYMBOL, 238), + (15, HUFFMAN_EMIT_SYMBOL, 238), + (24, HUFFMAN_EMIT_SYMBOL, 238), + (31, HUFFMAN_EMIT_SYMBOL, 238), + (41, HUFFMAN_EMIT_SYMBOL, 238), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 238), + (3, HUFFMAN_EMIT_SYMBOL, 240), + (6, HUFFMAN_EMIT_SYMBOL, 240), + (10, HUFFMAN_EMIT_SYMBOL, 240), + (15, HUFFMAN_EMIT_SYMBOL, 240), + (24, HUFFMAN_EMIT_SYMBOL, 240), + (31, HUFFMAN_EMIT_SYMBOL, 240), + (41, HUFFMAN_EMIT_SYMBOL, 240), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 240), + + # Node 203 + (2, HUFFMAN_EMIT_SYMBOL, 242), + (9, HUFFMAN_EMIT_SYMBOL, 242), + (23, HUFFMAN_EMIT_SYMBOL, 242), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 242), + (2, HUFFMAN_EMIT_SYMBOL, 243), + (9, HUFFMAN_EMIT_SYMBOL, 243), + (23, HUFFMAN_EMIT_SYMBOL, 243), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 243), + (2, HUFFMAN_EMIT_SYMBOL, 255), + (9, HUFFMAN_EMIT_SYMBOL, 255), + (23, HUFFMAN_EMIT_SYMBOL, 255), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 255), + (1, HUFFMAN_EMIT_SYMBOL, 203), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 203), + (1, HUFFMAN_EMIT_SYMBOL, 204), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 204), + + # Node 204 + (3, HUFFMAN_EMIT_SYMBOL, 242), + (6, HUFFMAN_EMIT_SYMBOL, 242), + (10, HUFFMAN_EMIT_SYMBOL, 242), + (15, HUFFMAN_EMIT_SYMBOL, 242), + (24, HUFFMAN_EMIT_SYMBOL, 242), + (31, HUFFMAN_EMIT_SYMBOL, 242), + (41, HUFFMAN_EMIT_SYMBOL, 242), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 242), + (3, HUFFMAN_EMIT_SYMBOL, 243), + (6, HUFFMAN_EMIT_SYMBOL, 243), + (10, HUFFMAN_EMIT_SYMBOL, 243), + (15, HUFFMAN_EMIT_SYMBOL, 243), + (24, HUFFMAN_EMIT_SYMBOL, 243), + (31, HUFFMAN_EMIT_SYMBOL, 243), + (41, HUFFMAN_EMIT_SYMBOL, 243), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 243), + + # Node 205 + (3, HUFFMAN_EMIT_SYMBOL, 255), + (6, HUFFMAN_EMIT_SYMBOL, 255), + (10, HUFFMAN_EMIT_SYMBOL, 255), + (15, HUFFMAN_EMIT_SYMBOL, 255), + (24, HUFFMAN_EMIT_SYMBOL, 255), + (31, HUFFMAN_EMIT_SYMBOL, 255), + (41, HUFFMAN_EMIT_SYMBOL, 255), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 255), + (2, HUFFMAN_EMIT_SYMBOL, 203), + (9, HUFFMAN_EMIT_SYMBOL, 203), + (23, HUFFMAN_EMIT_SYMBOL, 203), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 203), + (2, HUFFMAN_EMIT_SYMBOL, 204), + (9, HUFFMAN_EMIT_SYMBOL, 204), + (23, HUFFMAN_EMIT_SYMBOL, 204), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 204), + + # Node 206 + (3, HUFFMAN_EMIT_SYMBOL, 203), + (6, HUFFMAN_EMIT_SYMBOL, 203), + (10, HUFFMAN_EMIT_SYMBOL, 203), + (15, HUFFMAN_EMIT_SYMBOL, 203), + (24, HUFFMAN_EMIT_SYMBOL, 203), + (31, HUFFMAN_EMIT_SYMBOL, 203), + (41, HUFFMAN_EMIT_SYMBOL, 203), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 203), + (3, HUFFMAN_EMIT_SYMBOL, 204), + (6, HUFFMAN_EMIT_SYMBOL, 204), + (10, HUFFMAN_EMIT_SYMBOL, 204), + (15, HUFFMAN_EMIT_SYMBOL, 204), + (24, HUFFMAN_EMIT_SYMBOL, 204), + (31, HUFFMAN_EMIT_SYMBOL, 204), + (41, HUFFMAN_EMIT_SYMBOL, 204), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 204), + + # Node 207 + (211, 0, 0), + (212, 0, 0), + (214, 0, 0), + (215, 0, 0), + (218, 0, 0), + (219, 0, 0), + (221, 0, 0), + (222, 0, 0), + (226, 0, 0), + (228, 0, 0), + (232, 0, 0), + (235, 0, 0), + (240, 0, 0), + (243, 0, 0), + (247, 0, 0), + (250, 0, 0), + + # Node 208 + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 211), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 212), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 214), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 221), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 222), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 223), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 241), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 244), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 245), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 246), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 247), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 248), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 250), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 251), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 252), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 253), + + # Node 209 + (1, HUFFMAN_EMIT_SYMBOL, 211), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 211), + (1, HUFFMAN_EMIT_SYMBOL, 212), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 212), + (1, HUFFMAN_EMIT_SYMBOL, 214), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 214), + (1, HUFFMAN_EMIT_SYMBOL, 221), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 221), + (1, HUFFMAN_EMIT_SYMBOL, 222), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 222), + (1, HUFFMAN_EMIT_SYMBOL, 223), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 223), + (1, HUFFMAN_EMIT_SYMBOL, 241), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 241), + (1, HUFFMAN_EMIT_SYMBOL, 244), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 244), + + # Node 210 + (2, HUFFMAN_EMIT_SYMBOL, 211), + (9, HUFFMAN_EMIT_SYMBOL, 211), + (23, HUFFMAN_EMIT_SYMBOL, 211), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 211), + (2, HUFFMAN_EMIT_SYMBOL, 212), + (9, HUFFMAN_EMIT_SYMBOL, 212), + (23, HUFFMAN_EMIT_SYMBOL, 212), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 212), + (2, HUFFMAN_EMIT_SYMBOL, 214), + (9, HUFFMAN_EMIT_SYMBOL, 214), + (23, HUFFMAN_EMIT_SYMBOL, 214), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 214), + (2, HUFFMAN_EMIT_SYMBOL, 221), + (9, HUFFMAN_EMIT_SYMBOL, 221), + (23, HUFFMAN_EMIT_SYMBOL, 221), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 221), + + # Node 211 + (3, HUFFMAN_EMIT_SYMBOL, 211), + (6, HUFFMAN_EMIT_SYMBOL, 211), + (10, HUFFMAN_EMIT_SYMBOL, 211), + (15, HUFFMAN_EMIT_SYMBOL, 211), + (24, HUFFMAN_EMIT_SYMBOL, 211), + (31, HUFFMAN_EMIT_SYMBOL, 211), + (41, HUFFMAN_EMIT_SYMBOL, 211), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 211), + (3, HUFFMAN_EMIT_SYMBOL, 212), + (6, HUFFMAN_EMIT_SYMBOL, 212), + (10, HUFFMAN_EMIT_SYMBOL, 212), + (15, HUFFMAN_EMIT_SYMBOL, 212), + (24, HUFFMAN_EMIT_SYMBOL, 212), + (31, HUFFMAN_EMIT_SYMBOL, 212), + (41, HUFFMAN_EMIT_SYMBOL, 212), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 212), + + # Node 212 + (3, HUFFMAN_EMIT_SYMBOL, 214), + (6, HUFFMAN_EMIT_SYMBOL, 214), + (10, HUFFMAN_EMIT_SYMBOL, 214), + (15, HUFFMAN_EMIT_SYMBOL, 214), + (24, HUFFMAN_EMIT_SYMBOL, 214), + (31, HUFFMAN_EMIT_SYMBOL, 214), + (41, HUFFMAN_EMIT_SYMBOL, 214), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 214), + (3, HUFFMAN_EMIT_SYMBOL, 221), + (6, HUFFMAN_EMIT_SYMBOL, 221), + (10, HUFFMAN_EMIT_SYMBOL, 221), + (15, HUFFMAN_EMIT_SYMBOL, 221), + (24, HUFFMAN_EMIT_SYMBOL, 221), + (31, HUFFMAN_EMIT_SYMBOL, 221), + (41, HUFFMAN_EMIT_SYMBOL, 221), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 221), + + # Node 213 + (2, HUFFMAN_EMIT_SYMBOL, 222), + (9, HUFFMAN_EMIT_SYMBOL, 222), + (23, HUFFMAN_EMIT_SYMBOL, 222), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 222), + (2, HUFFMAN_EMIT_SYMBOL, 223), + (9, HUFFMAN_EMIT_SYMBOL, 223), + (23, HUFFMAN_EMIT_SYMBOL, 223), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 223), + (2, HUFFMAN_EMIT_SYMBOL, 241), + (9, HUFFMAN_EMIT_SYMBOL, 241), + (23, HUFFMAN_EMIT_SYMBOL, 241), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 241), + (2, HUFFMAN_EMIT_SYMBOL, 244), + (9, HUFFMAN_EMIT_SYMBOL, 244), + (23, HUFFMAN_EMIT_SYMBOL, 244), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 244), + + # Node 214 + (3, HUFFMAN_EMIT_SYMBOL, 222), + (6, HUFFMAN_EMIT_SYMBOL, 222), + (10, HUFFMAN_EMIT_SYMBOL, 222), + (15, HUFFMAN_EMIT_SYMBOL, 222), + (24, HUFFMAN_EMIT_SYMBOL, 222), + (31, HUFFMAN_EMIT_SYMBOL, 222), + (41, HUFFMAN_EMIT_SYMBOL, 222), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 222), + (3, HUFFMAN_EMIT_SYMBOL, 223), + (6, HUFFMAN_EMIT_SYMBOL, 223), + (10, HUFFMAN_EMIT_SYMBOL, 223), + (15, HUFFMAN_EMIT_SYMBOL, 223), + (24, HUFFMAN_EMIT_SYMBOL, 223), + (31, HUFFMAN_EMIT_SYMBOL, 223), + (41, HUFFMAN_EMIT_SYMBOL, 223), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 223), + + # Node 215 + (3, HUFFMAN_EMIT_SYMBOL, 241), + (6, HUFFMAN_EMIT_SYMBOL, 241), + (10, HUFFMAN_EMIT_SYMBOL, 241), + (15, HUFFMAN_EMIT_SYMBOL, 241), + (24, HUFFMAN_EMIT_SYMBOL, 241), + (31, HUFFMAN_EMIT_SYMBOL, 241), + (41, HUFFMAN_EMIT_SYMBOL, 241), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 241), + (3, HUFFMAN_EMIT_SYMBOL, 244), + (6, HUFFMAN_EMIT_SYMBOL, 244), + (10, HUFFMAN_EMIT_SYMBOL, 244), + (15, HUFFMAN_EMIT_SYMBOL, 244), + (24, HUFFMAN_EMIT_SYMBOL, 244), + (31, HUFFMAN_EMIT_SYMBOL, 244), + (41, HUFFMAN_EMIT_SYMBOL, 244), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 244), + + # Node 216 + (1, HUFFMAN_EMIT_SYMBOL, 245), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 245), + (1, HUFFMAN_EMIT_SYMBOL, 246), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 246), + (1, HUFFMAN_EMIT_SYMBOL, 247), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 247), + (1, HUFFMAN_EMIT_SYMBOL, 248), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 248), + (1, HUFFMAN_EMIT_SYMBOL, 250), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 250), + (1, HUFFMAN_EMIT_SYMBOL, 251), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 251), + (1, HUFFMAN_EMIT_SYMBOL, 252), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 252), + (1, HUFFMAN_EMIT_SYMBOL, 253), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 253), + + # Node 217 + (2, HUFFMAN_EMIT_SYMBOL, 245), + (9, HUFFMAN_EMIT_SYMBOL, 245), + (23, HUFFMAN_EMIT_SYMBOL, 245), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 245), + (2, HUFFMAN_EMIT_SYMBOL, 246), + (9, HUFFMAN_EMIT_SYMBOL, 246), + (23, HUFFMAN_EMIT_SYMBOL, 246), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 246), + (2, HUFFMAN_EMIT_SYMBOL, 247), + (9, HUFFMAN_EMIT_SYMBOL, 247), + (23, HUFFMAN_EMIT_SYMBOL, 247), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 247), + (2, HUFFMAN_EMIT_SYMBOL, 248), + (9, HUFFMAN_EMIT_SYMBOL, 248), + (23, HUFFMAN_EMIT_SYMBOL, 248), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 248), + + # Node 218 + (3, HUFFMAN_EMIT_SYMBOL, 245), + (6, HUFFMAN_EMIT_SYMBOL, 245), + (10, HUFFMAN_EMIT_SYMBOL, 245), + (15, HUFFMAN_EMIT_SYMBOL, 245), + (24, HUFFMAN_EMIT_SYMBOL, 245), + (31, HUFFMAN_EMIT_SYMBOL, 245), + (41, HUFFMAN_EMIT_SYMBOL, 245), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 245), + (3, HUFFMAN_EMIT_SYMBOL, 246), + (6, HUFFMAN_EMIT_SYMBOL, 246), + (10, HUFFMAN_EMIT_SYMBOL, 246), + (15, HUFFMAN_EMIT_SYMBOL, 246), + (24, HUFFMAN_EMIT_SYMBOL, 246), + (31, HUFFMAN_EMIT_SYMBOL, 246), + (41, HUFFMAN_EMIT_SYMBOL, 246), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 246), + + # Node 219 + (3, HUFFMAN_EMIT_SYMBOL, 247), + (6, HUFFMAN_EMIT_SYMBOL, 247), + (10, HUFFMAN_EMIT_SYMBOL, 247), + (15, HUFFMAN_EMIT_SYMBOL, 247), + (24, HUFFMAN_EMIT_SYMBOL, 247), + (31, HUFFMAN_EMIT_SYMBOL, 247), + (41, HUFFMAN_EMIT_SYMBOL, 247), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 247), + (3, HUFFMAN_EMIT_SYMBOL, 248), + (6, HUFFMAN_EMIT_SYMBOL, 248), + (10, HUFFMAN_EMIT_SYMBOL, 248), + (15, HUFFMAN_EMIT_SYMBOL, 248), + (24, HUFFMAN_EMIT_SYMBOL, 248), + (31, HUFFMAN_EMIT_SYMBOL, 248), + (41, HUFFMAN_EMIT_SYMBOL, 248), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 248), + + # Node 220 + (2, HUFFMAN_EMIT_SYMBOL, 250), + (9, HUFFMAN_EMIT_SYMBOL, 250), + (23, HUFFMAN_EMIT_SYMBOL, 250), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 250), + (2, HUFFMAN_EMIT_SYMBOL, 251), + (9, HUFFMAN_EMIT_SYMBOL, 251), + (23, HUFFMAN_EMIT_SYMBOL, 251), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 251), + (2, HUFFMAN_EMIT_SYMBOL, 252), + (9, HUFFMAN_EMIT_SYMBOL, 252), + (23, HUFFMAN_EMIT_SYMBOL, 252), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 252), + (2, HUFFMAN_EMIT_SYMBOL, 253), + (9, HUFFMAN_EMIT_SYMBOL, 253), + (23, HUFFMAN_EMIT_SYMBOL, 253), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 253), + + # Node 221 + (3, HUFFMAN_EMIT_SYMBOL, 250), + (6, HUFFMAN_EMIT_SYMBOL, 250), + (10, HUFFMAN_EMIT_SYMBOL, 250), + (15, HUFFMAN_EMIT_SYMBOL, 250), + (24, HUFFMAN_EMIT_SYMBOL, 250), + (31, HUFFMAN_EMIT_SYMBOL, 250), + (41, HUFFMAN_EMIT_SYMBOL, 250), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 250), + (3, HUFFMAN_EMIT_SYMBOL, 251), + (6, HUFFMAN_EMIT_SYMBOL, 251), + (10, HUFFMAN_EMIT_SYMBOL, 251), + (15, HUFFMAN_EMIT_SYMBOL, 251), + (24, HUFFMAN_EMIT_SYMBOL, 251), + (31, HUFFMAN_EMIT_SYMBOL, 251), + (41, HUFFMAN_EMIT_SYMBOL, 251), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 251), + + # Node 222 + (3, HUFFMAN_EMIT_SYMBOL, 252), + (6, HUFFMAN_EMIT_SYMBOL, 252), + (10, HUFFMAN_EMIT_SYMBOL, 252), + (15, HUFFMAN_EMIT_SYMBOL, 252), + (24, HUFFMAN_EMIT_SYMBOL, 252), + (31, HUFFMAN_EMIT_SYMBOL, 252), + (41, HUFFMAN_EMIT_SYMBOL, 252), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 252), + (3, HUFFMAN_EMIT_SYMBOL, 253), + (6, HUFFMAN_EMIT_SYMBOL, 253), + (10, HUFFMAN_EMIT_SYMBOL, 253), + (15, HUFFMAN_EMIT_SYMBOL, 253), + (24, HUFFMAN_EMIT_SYMBOL, 253), + (31, HUFFMAN_EMIT_SYMBOL, 253), + (41, HUFFMAN_EMIT_SYMBOL, 253), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 253), + + # Node 223 + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 254), + (227, 0, 0), + (229, 0, 0), + (230, 0, 0), + (233, 0, 0), + (234, 0, 0), + (236, 0, 0), + (237, 0, 0), + (241, 0, 0), + (242, 0, 0), + (244, 0, 0), + (245, 0, 0), + (248, 0, 0), + (249, 0, 0), + (251, 0, 0), + (252, 0, 0), + + # Node 224 + (1, HUFFMAN_EMIT_SYMBOL, 254), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 254), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 2), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 3), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 4), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 5), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 6), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 7), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 8), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 11), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 12), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 14), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 15), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 16), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 17), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 18), + + # Node 225 + (2, HUFFMAN_EMIT_SYMBOL, 254), + (9, HUFFMAN_EMIT_SYMBOL, 254), + (23, HUFFMAN_EMIT_SYMBOL, 254), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 254), + (1, HUFFMAN_EMIT_SYMBOL, 2), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 2), + (1, HUFFMAN_EMIT_SYMBOL, 3), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 3), + (1, HUFFMAN_EMIT_SYMBOL, 4), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 4), + (1, HUFFMAN_EMIT_SYMBOL, 5), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 5), + (1, HUFFMAN_EMIT_SYMBOL, 6), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 6), + (1, HUFFMAN_EMIT_SYMBOL, 7), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 7), + + # Node 226 + (3, HUFFMAN_EMIT_SYMBOL, 254), + (6, HUFFMAN_EMIT_SYMBOL, 254), + (10, HUFFMAN_EMIT_SYMBOL, 254), + (15, HUFFMAN_EMIT_SYMBOL, 254), + (24, HUFFMAN_EMIT_SYMBOL, 254), + (31, HUFFMAN_EMIT_SYMBOL, 254), + (41, HUFFMAN_EMIT_SYMBOL, 254), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 254), + (2, HUFFMAN_EMIT_SYMBOL, 2), + (9, HUFFMAN_EMIT_SYMBOL, 2), + (23, HUFFMAN_EMIT_SYMBOL, 2), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 2), + (2, HUFFMAN_EMIT_SYMBOL, 3), + (9, HUFFMAN_EMIT_SYMBOL, 3), + (23, HUFFMAN_EMIT_SYMBOL, 3), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 3), + + # Node 227 + (3, HUFFMAN_EMIT_SYMBOL, 2), + (6, HUFFMAN_EMIT_SYMBOL, 2), + (10, HUFFMAN_EMIT_SYMBOL, 2), + (15, HUFFMAN_EMIT_SYMBOL, 2), + (24, HUFFMAN_EMIT_SYMBOL, 2), + (31, HUFFMAN_EMIT_SYMBOL, 2), + (41, HUFFMAN_EMIT_SYMBOL, 2), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 2), + (3, HUFFMAN_EMIT_SYMBOL, 3), + (6, HUFFMAN_EMIT_SYMBOL, 3), + (10, HUFFMAN_EMIT_SYMBOL, 3), + (15, HUFFMAN_EMIT_SYMBOL, 3), + (24, HUFFMAN_EMIT_SYMBOL, 3), + (31, HUFFMAN_EMIT_SYMBOL, 3), + (41, HUFFMAN_EMIT_SYMBOL, 3), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 3), + + # Node 228 + (2, HUFFMAN_EMIT_SYMBOL, 4), + (9, HUFFMAN_EMIT_SYMBOL, 4), + (23, HUFFMAN_EMIT_SYMBOL, 4), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 4), + (2, HUFFMAN_EMIT_SYMBOL, 5), + (9, HUFFMAN_EMIT_SYMBOL, 5), + (23, HUFFMAN_EMIT_SYMBOL, 5), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 5), + (2, HUFFMAN_EMIT_SYMBOL, 6), + (9, HUFFMAN_EMIT_SYMBOL, 6), + (23, HUFFMAN_EMIT_SYMBOL, 6), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 6), + (2, HUFFMAN_EMIT_SYMBOL, 7), + (9, HUFFMAN_EMIT_SYMBOL, 7), + (23, HUFFMAN_EMIT_SYMBOL, 7), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 7), + + # Node 229 + (3, HUFFMAN_EMIT_SYMBOL, 4), + (6, HUFFMAN_EMIT_SYMBOL, 4), + (10, HUFFMAN_EMIT_SYMBOL, 4), + (15, HUFFMAN_EMIT_SYMBOL, 4), + (24, HUFFMAN_EMIT_SYMBOL, 4), + (31, HUFFMAN_EMIT_SYMBOL, 4), + (41, HUFFMAN_EMIT_SYMBOL, 4), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 4), + (3, HUFFMAN_EMIT_SYMBOL, 5), + (6, HUFFMAN_EMIT_SYMBOL, 5), + (10, HUFFMAN_EMIT_SYMBOL, 5), + (15, HUFFMAN_EMIT_SYMBOL, 5), + (24, HUFFMAN_EMIT_SYMBOL, 5), + (31, HUFFMAN_EMIT_SYMBOL, 5), + (41, HUFFMAN_EMIT_SYMBOL, 5), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 5), + + # Node 230 + (3, HUFFMAN_EMIT_SYMBOL, 6), + (6, HUFFMAN_EMIT_SYMBOL, 6), + (10, HUFFMAN_EMIT_SYMBOL, 6), + (15, HUFFMAN_EMIT_SYMBOL, 6), + (24, HUFFMAN_EMIT_SYMBOL, 6), + (31, HUFFMAN_EMIT_SYMBOL, 6), + (41, HUFFMAN_EMIT_SYMBOL, 6), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 6), + (3, HUFFMAN_EMIT_SYMBOL, 7), + (6, HUFFMAN_EMIT_SYMBOL, 7), + (10, HUFFMAN_EMIT_SYMBOL, 7), + (15, HUFFMAN_EMIT_SYMBOL, 7), + (24, HUFFMAN_EMIT_SYMBOL, 7), + (31, HUFFMAN_EMIT_SYMBOL, 7), + (41, HUFFMAN_EMIT_SYMBOL, 7), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 7), + + # Node 231 + (1, HUFFMAN_EMIT_SYMBOL, 8), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 8), + (1, HUFFMAN_EMIT_SYMBOL, 11), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 11), + (1, HUFFMAN_EMIT_SYMBOL, 12), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 12), + (1, HUFFMAN_EMIT_SYMBOL, 14), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 14), + (1, HUFFMAN_EMIT_SYMBOL, 15), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 15), + (1, HUFFMAN_EMIT_SYMBOL, 16), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 16), + (1, HUFFMAN_EMIT_SYMBOL, 17), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 17), + (1, HUFFMAN_EMIT_SYMBOL, 18), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 18), + + # Node 232 + (2, HUFFMAN_EMIT_SYMBOL, 8), + (9, HUFFMAN_EMIT_SYMBOL, 8), + (23, HUFFMAN_EMIT_SYMBOL, 8), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 8), + (2, HUFFMAN_EMIT_SYMBOL, 11), + (9, HUFFMAN_EMIT_SYMBOL, 11), + (23, HUFFMAN_EMIT_SYMBOL, 11), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 11), + (2, HUFFMAN_EMIT_SYMBOL, 12), + (9, HUFFMAN_EMIT_SYMBOL, 12), + (23, HUFFMAN_EMIT_SYMBOL, 12), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 12), + (2, HUFFMAN_EMIT_SYMBOL, 14), + (9, HUFFMAN_EMIT_SYMBOL, 14), + (23, HUFFMAN_EMIT_SYMBOL, 14), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 14), + + # Node 233 + (3, HUFFMAN_EMIT_SYMBOL, 8), + (6, HUFFMAN_EMIT_SYMBOL, 8), + (10, HUFFMAN_EMIT_SYMBOL, 8), + (15, HUFFMAN_EMIT_SYMBOL, 8), + (24, HUFFMAN_EMIT_SYMBOL, 8), + (31, HUFFMAN_EMIT_SYMBOL, 8), + (41, HUFFMAN_EMIT_SYMBOL, 8), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 8), + (3, HUFFMAN_EMIT_SYMBOL, 11), + (6, HUFFMAN_EMIT_SYMBOL, 11), + (10, HUFFMAN_EMIT_SYMBOL, 11), + (15, HUFFMAN_EMIT_SYMBOL, 11), + (24, HUFFMAN_EMIT_SYMBOL, 11), + (31, HUFFMAN_EMIT_SYMBOL, 11), + (41, HUFFMAN_EMIT_SYMBOL, 11), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 11), + + # Node 234 + (3, HUFFMAN_EMIT_SYMBOL, 12), + (6, HUFFMAN_EMIT_SYMBOL, 12), + (10, HUFFMAN_EMIT_SYMBOL, 12), + (15, HUFFMAN_EMIT_SYMBOL, 12), + (24, HUFFMAN_EMIT_SYMBOL, 12), + (31, HUFFMAN_EMIT_SYMBOL, 12), + (41, HUFFMAN_EMIT_SYMBOL, 12), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 12), + (3, HUFFMAN_EMIT_SYMBOL, 14), + (6, HUFFMAN_EMIT_SYMBOL, 14), + (10, HUFFMAN_EMIT_SYMBOL, 14), + (15, HUFFMAN_EMIT_SYMBOL, 14), + (24, HUFFMAN_EMIT_SYMBOL, 14), + (31, HUFFMAN_EMIT_SYMBOL, 14), + (41, HUFFMAN_EMIT_SYMBOL, 14), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 14), + + # Node 235 + (2, HUFFMAN_EMIT_SYMBOL, 15), + (9, HUFFMAN_EMIT_SYMBOL, 15), + (23, HUFFMAN_EMIT_SYMBOL, 15), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 15), + (2, HUFFMAN_EMIT_SYMBOL, 16), + (9, HUFFMAN_EMIT_SYMBOL, 16), + (23, HUFFMAN_EMIT_SYMBOL, 16), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 16), + (2, HUFFMAN_EMIT_SYMBOL, 17), + (9, HUFFMAN_EMIT_SYMBOL, 17), + (23, HUFFMAN_EMIT_SYMBOL, 17), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 17), + (2, HUFFMAN_EMIT_SYMBOL, 18), + (9, HUFFMAN_EMIT_SYMBOL, 18), + (23, HUFFMAN_EMIT_SYMBOL, 18), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 18), + + # Node 236 + (3, HUFFMAN_EMIT_SYMBOL, 15), + (6, HUFFMAN_EMIT_SYMBOL, 15), + (10, HUFFMAN_EMIT_SYMBOL, 15), + (15, HUFFMAN_EMIT_SYMBOL, 15), + (24, HUFFMAN_EMIT_SYMBOL, 15), + (31, HUFFMAN_EMIT_SYMBOL, 15), + (41, HUFFMAN_EMIT_SYMBOL, 15), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 15), + (3, HUFFMAN_EMIT_SYMBOL, 16), + (6, HUFFMAN_EMIT_SYMBOL, 16), + (10, HUFFMAN_EMIT_SYMBOL, 16), + (15, HUFFMAN_EMIT_SYMBOL, 16), + (24, HUFFMAN_EMIT_SYMBOL, 16), + (31, HUFFMAN_EMIT_SYMBOL, 16), + (41, HUFFMAN_EMIT_SYMBOL, 16), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 16), + + # Node 237 + (3, HUFFMAN_EMIT_SYMBOL, 17), + (6, HUFFMAN_EMIT_SYMBOL, 17), + (10, HUFFMAN_EMIT_SYMBOL, 17), + (15, HUFFMAN_EMIT_SYMBOL, 17), + (24, HUFFMAN_EMIT_SYMBOL, 17), + (31, HUFFMAN_EMIT_SYMBOL, 17), + (41, HUFFMAN_EMIT_SYMBOL, 17), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 17), + (3, HUFFMAN_EMIT_SYMBOL, 18), + (6, HUFFMAN_EMIT_SYMBOL, 18), + (10, HUFFMAN_EMIT_SYMBOL, 18), + (15, HUFFMAN_EMIT_SYMBOL, 18), + (24, HUFFMAN_EMIT_SYMBOL, 18), + (31, HUFFMAN_EMIT_SYMBOL, 18), + (41, HUFFMAN_EMIT_SYMBOL, 18), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 18), + + # Node 238 + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 19), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 20), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 21), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 23), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 24), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 25), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 26), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 27), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 28), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 29), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 30), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 31), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 127), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 220), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 249), + (253, 0, 0), + + # Node 239 + (1, HUFFMAN_EMIT_SYMBOL, 19), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 19), + (1, HUFFMAN_EMIT_SYMBOL, 20), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 20), + (1, HUFFMAN_EMIT_SYMBOL, 21), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 21), + (1, HUFFMAN_EMIT_SYMBOL, 23), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 23), + (1, HUFFMAN_EMIT_SYMBOL, 24), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 24), + (1, HUFFMAN_EMIT_SYMBOL, 25), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 25), + (1, HUFFMAN_EMIT_SYMBOL, 26), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 26), + (1, HUFFMAN_EMIT_SYMBOL, 27), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 27), + + # Node 240 + (2, HUFFMAN_EMIT_SYMBOL, 19), + (9, HUFFMAN_EMIT_SYMBOL, 19), + (23, HUFFMAN_EMIT_SYMBOL, 19), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 19), + (2, HUFFMAN_EMIT_SYMBOL, 20), + (9, HUFFMAN_EMIT_SYMBOL, 20), + (23, HUFFMAN_EMIT_SYMBOL, 20), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 20), + (2, HUFFMAN_EMIT_SYMBOL, 21), + (9, HUFFMAN_EMIT_SYMBOL, 21), + (23, HUFFMAN_EMIT_SYMBOL, 21), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 21), + (2, HUFFMAN_EMIT_SYMBOL, 23), + (9, HUFFMAN_EMIT_SYMBOL, 23), + (23, HUFFMAN_EMIT_SYMBOL, 23), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 23), + + # Node 241 + (3, HUFFMAN_EMIT_SYMBOL, 19), + (6, HUFFMAN_EMIT_SYMBOL, 19), + (10, HUFFMAN_EMIT_SYMBOL, 19), + (15, HUFFMAN_EMIT_SYMBOL, 19), + (24, HUFFMAN_EMIT_SYMBOL, 19), + (31, HUFFMAN_EMIT_SYMBOL, 19), + (41, HUFFMAN_EMIT_SYMBOL, 19), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 19), + (3, HUFFMAN_EMIT_SYMBOL, 20), + (6, HUFFMAN_EMIT_SYMBOL, 20), + (10, HUFFMAN_EMIT_SYMBOL, 20), + (15, HUFFMAN_EMIT_SYMBOL, 20), + (24, HUFFMAN_EMIT_SYMBOL, 20), + (31, HUFFMAN_EMIT_SYMBOL, 20), + (41, HUFFMAN_EMIT_SYMBOL, 20), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 20), + + # Node 242 + (3, HUFFMAN_EMIT_SYMBOL, 21), + (6, HUFFMAN_EMIT_SYMBOL, 21), + (10, HUFFMAN_EMIT_SYMBOL, 21), + (15, HUFFMAN_EMIT_SYMBOL, 21), + (24, HUFFMAN_EMIT_SYMBOL, 21), + (31, HUFFMAN_EMIT_SYMBOL, 21), + (41, HUFFMAN_EMIT_SYMBOL, 21), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 21), + (3, HUFFMAN_EMIT_SYMBOL, 23), + (6, HUFFMAN_EMIT_SYMBOL, 23), + (10, HUFFMAN_EMIT_SYMBOL, 23), + (15, HUFFMAN_EMIT_SYMBOL, 23), + (24, HUFFMAN_EMIT_SYMBOL, 23), + (31, HUFFMAN_EMIT_SYMBOL, 23), + (41, HUFFMAN_EMIT_SYMBOL, 23), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 23), + + # Node 243 + (2, HUFFMAN_EMIT_SYMBOL, 24), + (9, HUFFMAN_EMIT_SYMBOL, 24), + (23, HUFFMAN_EMIT_SYMBOL, 24), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 24), + (2, HUFFMAN_EMIT_SYMBOL, 25), + (9, HUFFMAN_EMIT_SYMBOL, 25), + (23, HUFFMAN_EMIT_SYMBOL, 25), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 25), + (2, HUFFMAN_EMIT_SYMBOL, 26), + (9, HUFFMAN_EMIT_SYMBOL, 26), + (23, HUFFMAN_EMIT_SYMBOL, 26), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 26), + (2, HUFFMAN_EMIT_SYMBOL, 27), + (9, HUFFMAN_EMIT_SYMBOL, 27), + (23, HUFFMAN_EMIT_SYMBOL, 27), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 27), + + # Node 244 + (3, HUFFMAN_EMIT_SYMBOL, 24), + (6, HUFFMAN_EMIT_SYMBOL, 24), + (10, HUFFMAN_EMIT_SYMBOL, 24), + (15, HUFFMAN_EMIT_SYMBOL, 24), + (24, HUFFMAN_EMIT_SYMBOL, 24), + (31, HUFFMAN_EMIT_SYMBOL, 24), + (41, HUFFMAN_EMIT_SYMBOL, 24), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 24), + (3, HUFFMAN_EMIT_SYMBOL, 25), + (6, HUFFMAN_EMIT_SYMBOL, 25), + (10, HUFFMAN_EMIT_SYMBOL, 25), + (15, HUFFMAN_EMIT_SYMBOL, 25), + (24, HUFFMAN_EMIT_SYMBOL, 25), + (31, HUFFMAN_EMIT_SYMBOL, 25), + (41, HUFFMAN_EMIT_SYMBOL, 25), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 25), + + # Node 245 + (3, HUFFMAN_EMIT_SYMBOL, 26), + (6, HUFFMAN_EMIT_SYMBOL, 26), + (10, HUFFMAN_EMIT_SYMBOL, 26), + (15, HUFFMAN_EMIT_SYMBOL, 26), + (24, HUFFMAN_EMIT_SYMBOL, 26), + (31, HUFFMAN_EMIT_SYMBOL, 26), + (41, HUFFMAN_EMIT_SYMBOL, 26), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 26), + (3, HUFFMAN_EMIT_SYMBOL, 27), + (6, HUFFMAN_EMIT_SYMBOL, 27), + (10, HUFFMAN_EMIT_SYMBOL, 27), + (15, HUFFMAN_EMIT_SYMBOL, 27), + (24, HUFFMAN_EMIT_SYMBOL, 27), + (31, HUFFMAN_EMIT_SYMBOL, 27), + (41, HUFFMAN_EMIT_SYMBOL, 27), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 27), + + # Node 246 + (1, HUFFMAN_EMIT_SYMBOL, 28), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 28), + (1, HUFFMAN_EMIT_SYMBOL, 29), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 29), + (1, HUFFMAN_EMIT_SYMBOL, 30), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 30), + (1, HUFFMAN_EMIT_SYMBOL, 31), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 31), + (1, HUFFMAN_EMIT_SYMBOL, 127), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 127), + (1, HUFFMAN_EMIT_SYMBOL, 220), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 220), + (1, HUFFMAN_EMIT_SYMBOL, 249), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 249), + (254, 0, 0), + (255, 0, 0), + + # Node 247 + (2, HUFFMAN_EMIT_SYMBOL, 28), + (9, HUFFMAN_EMIT_SYMBOL, 28), + (23, HUFFMAN_EMIT_SYMBOL, 28), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 28), + (2, HUFFMAN_EMIT_SYMBOL, 29), + (9, HUFFMAN_EMIT_SYMBOL, 29), + (23, HUFFMAN_EMIT_SYMBOL, 29), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 29), + (2, HUFFMAN_EMIT_SYMBOL, 30), + (9, HUFFMAN_EMIT_SYMBOL, 30), + (23, HUFFMAN_EMIT_SYMBOL, 30), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 30), + (2, HUFFMAN_EMIT_SYMBOL, 31), + (9, HUFFMAN_EMIT_SYMBOL, 31), + (23, HUFFMAN_EMIT_SYMBOL, 31), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 31), + + # Node 248 + (3, HUFFMAN_EMIT_SYMBOL, 28), + (6, HUFFMAN_EMIT_SYMBOL, 28), + (10, HUFFMAN_EMIT_SYMBOL, 28), + (15, HUFFMAN_EMIT_SYMBOL, 28), + (24, HUFFMAN_EMIT_SYMBOL, 28), + (31, HUFFMAN_EMIT_SYMBOL, 28), + (41, HUFFMAN_EMIT_SYMBOL, 28), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 28), + (3, HUFFMAN_EMIT_SYMBOL, 29), + (6, HUFFMAN_EMIT_SYMBOL, 29), + (10, HUFFMAN_EMIT_SYMBOL, 29), + (15, HUFFMAN_EMIT_SYMBOL, 29), + (24, HUFFMAN_EMIT_SYMBOL, 29), + (31, HUFFMAN_EMIT_SYMBOL, 29), + (41, HUFFMAN_EMIT_SYMBOL, 29), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 29), + + # Node 249 + (3, HUFFMAN_EMIT_SYMBOL, 30), + (6, HUFFMAN_EMIT_SYMBOL, 30), + (10, HUFFMAN_EMIT_SYMBOL, 30), + (15, HUFFMAN_EMIT_SYMBOL, 30), + (24, HUFFMAN_EMIT_SYMBOL, 30), + (31, HUFFMAN_EMIT_SYMBOL, 30), + (41, HUFFMAN_EMIT_SYMBOL, 30), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 30), + (3, HUFFMAN_EMIT_SYMBOL, 31), + (6, HUFFMAN_EMIT_SYMBOL, 31), + (10, HUFFMAN_EMIT_SYMBOL, 31), + (15, HUFFMAN_EMIT_SYMBOL, 31), + (24, HUFFMAN_EMIT_SYMBOL, 31), + (31, HUFFMAN_EMIT_SYMBOL, 31), + (41, HUFFMAN_EMIT_SYMBOL, 31), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 31), + + # Node 250 + (2, HUFFMAN_EMIT_SYMBOL, 127), + (9, HUFFMAN_EMIT_SYMBOL, 127), + (23, HUFFMAN_EMIT_SYMBOL, 127), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 127), + (2, HUFFMAN_EMIT_SYMBOL, 220), + (9, HUFFMAN_EMIT_SYMBOL, 220), + (23, HUFFMAN_EMIT_SYMBOL, 220), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 220), + (2, HUFFMAN_EMIT_SYMBOL, 249), + (9, HUFFMAN_EMIT_SYMBOL, 249), + (23, HUFFMAN_EMIT_SYMBOL, 249), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 249), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 10), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 13), + (0, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 22), + (0, HUFFMAN_FAIL, 0), + + # Node 251 + (3, HUFFMAN_EMIT_SYMBOL, 127), + (6, HUFFMAN_EMIT_SYMBOL, 127), + (10, HUFFMAN_EMIT_SYMBOL, 127), + (15, HUFFMAN_EMIT_SYMBOL, 127), + (24, HUFFMAN_EMIT_SYMBOL, 127), + (31, HUFFMAN_EMIT_SYMBOL, 127), + (41, HUFFMAN_EMIT_SYMBOL, 127), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 127), + (3, HUFFMAN_EMIT_SYMBOL, 220), + (6, HUFFMAN_EMIT_SYMBOL, 220), + (10, HUFFMAN_EMIT_SYMBOL, 220), + (15, HUFFMAN_EMIT_SYMBOL, 220), + (24, HUFFMAN_EMIT_SYMBOL, 220), + (31, HUFFMAN_EMIT_SYMBOL, 220), + (41, HUFFMAN_EMIT_SYMBOL, 220), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 220), + + # Node 252 + (3, HUFFMAN_EMIT_SYMBOL, 249), + (6, HUFFMAN_EMIT_SYMBOL, 249), + (10, HUFFMAN_EMIT_SYMBOL, 249), + (15, HUFFMAN_EMIT_SYMBOL, 249), + (24, HUFFMAN_EMIT_SYMBOL, 249), + (31, HUFFMAN_EMIT_SYMBOL, 249), + (41, HUFFMAN_EMIT_SYMBOL, 249), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 249), + (1, HUFFMAN_EMIT_SYMBOL, 10), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 10), + (1, HUFFMAN_EMIT_SYMBOL, 13), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 13), + (1, HUFFMAN_EMIT_SYMBOL, 22), + (22, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 22), + (0, HUFFMAN_FAIL, 0), + (0, HUFFMAN_FAIL, 0), + + # Node 253 + (2, HUFFMAN_EMIT_SYMBOL, 10), + (9, HUFFMAN_EMIT_SYMBOL, 10), + (23, HUFFMAN_EMIT_SYMBOL, 10), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 10), + (2, HUFFMAN_EMIT_SYMBOL, 13), + (9, HUFFMAN_EMIT_SYMBOL, 13), + (23, HUFFMAN_EMIT_SYMBOL, 13), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 13), + (2, HUFFMAN_EMIT_SYMBOL, 22), + (9, HUFFMAN_EMIT_SYMBOL, 22), + (23, HUFFMAN_EMIT_SYMBOL, 22), + (40, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 22), + (0, HUFFMAN_FAIL, 0), + (0, HUFFMAN_FAIL, 0), + (0, HUFFMAN_FAIL, 0), + (0, HUFFMAN_FAIL, 0), + + # Node 254 + (3, HUFFMAN_EMIT_SYMBOL, 10), + (6, HUFFMAN_EMIT_SYMBOL, 10), + (10, HUFFMAN_EMIT_SYMBOL, 10), + (15, HUFFMAN_EMIT_SYMBOL, 10), + (24, HUFFMAN_EMIT_SYMBOL, 10), + (31, HUFFMAN_EMIT_SYMBOL, 10), + (41, HUFFMAN_EMIT_SYMBOL, 10), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 10), + (3, HUFFMAN_EMIT_SYMBOL, 13), + (6, HUFFMAN_EMIT_SYMBOL, 13), + (10, HUFFMAN_EMIT_SYMBOL, 13), + (15, HUFFMAN_EMIT_SYMBOL, 13), + (24, HUFFMAN_EMIT_SYMBOL, 13), + (31, HUFFMAN_EMIT_SYMBOL, 13), + (41, HUFFMAN_EMIT_SYMBOL, 13), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 13), + + # Node 255 + (3, HUFFMAN_EMIT_SYMBOL, 22), + (6, HUFFMAN_EMIT_SYMBOL, 22), + (10, HUFFMAN_EMIT_SYMBOL, 22), + (15, HUFFMAN_EMIT_SYMBOL, 22), + (24, HUFFMAN_EMIT_SYMBOL, 22), + (31, HUFFMAN_EMIT_SYMBOL, 22), + (41, HUFFMAN_EMIT_SYMBOL, 22), + (56, HUFFMAN_COMPLETE | HUFFMAN_EMIT_SYMBOL, 22), + (0, HUFFMAN_FAIL, 0), + (0, HUFFMAN_FAIL, 0), + (0, HUFFMAN_FAIL, 0), + (0, HUFFMAN_FAIL, 0), + (0, HUFFMAN_FAIL, 0), + (0, HUFFMAN_FAIL, 0), + (0, HUFFMAN_FAIL, 0), + (0, HUFFMAN_FAIL, 0), +] diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/hpack/src/hpack/struct.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/hpack/src/hpack/struct.py new file mode 100644 index 0000000000..fcab929560 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/hpack/src/hpack/struct.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +""" +hpack/struct +~~~~~~~~~~~~ + +Contains structures for representing header fields with associated metadata. +""" + + +class HeaderTuple(tuple): + """ + A data structure that stores a single header field. + + HTTP headers can be thought of as tuples of ``(field name, field value)``. + A single header block is a sequence of such tuples. + + In HTTP/2, however, certain bits of additional information are required for + compressing these headers: in particular, whether the header field can be + safely added to the HPACK compression context. + + This class stores a header that can be added to the compression context. In + all other ways it behaves exactly like a tuple. + """ + __slots__ = () + + indexable = True + + def __new__(cls, *args): + return tuple.__new__(cls, args) + + +class NeverIndexedHeaderTuple(HeaderTuple): + """ + A data structure that stores a single header field that cannot be added to + a HTTP/2 header compression context. + """ + __slots__ = () + + indexable = False diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/hpack/src/hpack/table.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/hpack/src/hpack/table.py new file mode 100644 index 0000000000..2b656f32a1 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/hpack/src/hpack/table.py @@ -0,0 +1,235 @@ +# -*- coding: utf-8 -*- +# flake8: noqa +from collections import deque +import logging + +from .exceptions import InvalidTableIndex + +log = logging.getLogger(__name__) + + +def table_entry_size(name, value): + """ + Calculates the size of a single entry + + This size is mostly irrelevant to us and defined + specifically to accommodate memory management for + lower level implementations. The 32 extra bytes are + considered the "maximum" overhead that would be + required to represent each entry in the table. + + See RFC7541 Section 4.1 + """ + return 32 + len(name) + len(value) + + +class HeaderTable: + """ + Implements the combined static and dynamic header table + + The name and value arguments for all the functions + should ONLY be byte strings (b'') however this is not + strictly enforced in the interface. + + See RFC7541 Section 2.3 + """ + #: Default maximum size of the dynamic table. See + #: RFC7540 Section 6.5.2. + DEFAULT_SIZE = 4096 + + #: Constant list of static headers. See RFC7541 Section + #: 2.3.1 and Appendix A + STATIC_TABLE = ( + (b':authority' , b'' ), # noqa + (b':method' , b'GET' ), # noqa + (b':method' , b'POST' ), # noqa + (b':path' , b'/' ), # noqa + (b':path' , b'/index.html' ), # noqa + (b':scheme' , b'http' ), # noqa + (b':scheme' , b'https' ), # noqa + (b':status' , b'200' ), # noqa + (b':status' , b'204' ), # noqa + (b':status' , b'206' ), # noqa + (b':status' , b'304' ), # noqa + (b':status' , b'400' ), # noqa + (b':status' , b'404' ), # noqa + (b':status' , b'500' ), # noqa + (b'accept-charset' , b'' ), # noqa + (b'accept-encoding' , b'gzip, deflate'), # noqa + (b'accept-language' , b'' ), # noqa + (b'accept-ranges' , b'' ), # noqa + (b'accept' , b'' ), # noqa + (b'access-control-allow-origin' , b'' ), # noqa + (b'age' , b'' ), # noqa + (b'allow' , b'' ), # noqa + (b'authorization' , b'' ), # noqa + (b'cache-control' , b'' ), # noqa + (b'content-disposition' , b'' ), # noqa + (b'content-encoding' , b'' ), # noqa + (b'content-language' , b'' ), # noqa + (b'content-length' , b'' ), # noqa + (b'content-location' , b'' ), # noqa + (b'content-range' , b'' ), # noqa + (b'content-type' , b'' ), # noqa + (b'cookie' , b'' ), # noqa + (b'date' , b'' ), # noqa + (b'etag' , b'' ), # noqa + (b'expect' , b'' ), # noqa + (b'expires' , b'' ), # noqa + (b'from' , b'' ), # noqa + (b'host' , b'' ), # noqa + (b'if-match' , b'' ), # noqa + (b'if-modified-since' , b'' ), # noqa + (b'if-none-match' , b'' ), # noqa + (b'if-range' , b'' ), # noqa + (b'if-unmodified-since' , b'' ), # noqa + (b'last-modified' , b'' ), # noqa + (b'link' , b'' ), # noqa + (b'location' , b'' ), # noqa + (b'max-forwards' , b'' ), # noqa + (b'proxy-authenticate' , b'' ), # noqa + (b'proxy-authorization' , b'' ), # noqa + (b'range' , b'' ), # noqa + (b'referer' , b'' ), # noqa + (b'refresh' , b'' ), # noqa + (b'retry-after' , b'' ), # noqa + (b'server' , b'' ), # noqa + (b'set-cookie' , b'' ), # noqa + (b'strict-transport-security' , b'' ), # noqa + (b'transfer-encoding' , b'' ), # noqa + (b'user-agent' , b'' ), # noqa + (b'vary' , b'' ), # noqa + (b'via' , b'' ), # noqa + (b'www-authenticate' , b'' ), # noqa + ) # noqa + + STATIC_TABLE_LENGTH = len(STATIC_TABLE) + + def __init__(self): + self._maxsize = HeaderTable.DEFAULT_SIZE + self._current_size = 0 + self.resized = False + self.dynamic_entries = deque() + + def get_by_index(self, index): + """ + Returns the entry specified by index + + Note that the table is 1-based ie an index of 0 is + invalid. This is due to the fact that a zero value + index signals that a completely unindexed header + follows. + + The entry will either be from the static table or + the dynamic table depending on the value of index. + """ + original_index = index + index -= 1 + if 0 <= index: + if index < HeaderTable.STATIC_TABLE_LENGTH: + return HeaderTable.STATIC_TABLE[index] + + index -= HeaderTable.STATIC_TABLE_LENGTH + if index < len(self.dynamic_entries): + return self.dynamic_entries[index] + + raise InvalidTableIndex("Invalid table index %d" % original_index) + + def __repr__(self): + return "HeaderTable(%d, %s, %r)" % ( + self._maxsize, + self.resized, + self.dynamic_entries + ) + + def add(self, name, value): + """ + Adds a new entry to the table + + We reduce the table size if the entry will make the + table size greater than maxsize. + """ + # We just clear the table if the entry is too big + size = table_entry_size(name, value) + if size > self._maxsize: + self.dynamic_entries.clear() + self._current_size = 0 + else: + # Add new entry + self.dynamic_entries.appendleft((name, value)) + self._current_size += size + self._shrink() + + def search(self, name, value): + """ + Searches the table for the entry specified by name + and value + + Returns one of the following: + - ``None``, no match at all + - ``(index, name, None)`` for partial matches on name only. + - ``(index, name, value)`` for perfect matches. + """ + partial = None + + header_name_search_result = HeaderTable.STATIC_TABLE_MAPPING.get(name) + if header_name_search_result: + index = header_name_search_result[1].get(value) + if index is not None: + return index, name, value + else: + partial = (header_name_search_result[0], name, None) + + offset = HeaderTable.STATIC_TABLE_LENGTH + 1 + for (i, (n, v)) in enumerate(self.dynamic_entries): + if n == name: + if v == value: + return i + offset, n, v + elif partial is None: + partial = (i + offset, n, None) + return partial + + @property + def maxsize(self): + return self._maxsize + + @maxsize.setter + def maxsize(self, newmax): + newmax = int(newmax) + log.debug("Resizing header table to %d from %d", newmax, self._maxsize) + oldmax = self._maxsize + self._maxsize = newmax + self.resized = (newmax != oldmax) + if newmax <= 0: + self.dynamic_entries.clear() + self._current_size = 0 + elif oldmax > newmax: + self._shrink() + + def _shrink(self): + """ + Shrinks the dynamic table to be at or below maxsize + """ + cursize = self._current_size + while cursize > self._maxsize: + name, value = self.dynamic_entries.pop() + cursize -= table_entry_size(name, value) + log.debug("Evicting %s: %s from the header table", name, value) + self._current_size = cursize + + +def _build_static_table_mapping(): + """ + Build static table mapping from header name to tuple with next structure: + (, ). + + static_table_mapping used for hash searching. + """ + static_table_mapping = {} + for index, (name, value) in enumerate(HeaderTable.STATIC_TABLE, 1): + header_name_search_result = static_table_mapping.setdefault(name, (index, {})) + header_name_search_result[1][value] = index + return static_table_mapping + + +HeaderTable.STATIC_TABLE_MAPPING = _build_static_table_mapping() diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/hyperframe/src/hyperframe/__init__.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/hyperframe/src/hyperframe/__init__.py new file mode 100644 index 0000000000..bf5ba1cb22 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/hyperframe/src/hyperframe/__init__.py @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- +""" +hyperframe +~~~~~~~~~~ + +A module for providing a pure-Python HTTP/2 framing layer. +""" +__version__ = '6.0.1' diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/hyperframe/src/hyperframe/exceptions.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/hyperframe/src/hyperframe/exceptions.py new file mode 100644 index 0000000000..d6adab25c8 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/hyperframe/src/hyperframe/exceptions.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- +""" +hyperframe/exceptions +~~~~~~~~~~~~~~~~~~~~~ + +Defines the exceptions that can be thrown by hyperframe. +""" + + +class HyperframeError(Exception): + """ + The base class for all exceptions for the hyperframe module. + + .. versionadded:: 6.0.0 + """ + + +class UnknownFrameError(HyperframeError): + """ + A frame of unknown type was received. + + .. versionchanged:: 6.0.0 + Changed base class from `ValueError` to :class:`HyperframeError` + """ + def __init__(self, frame_type: int, length: int) -> None: + #: The type byte of the unknown frame that was received. + self.frame_type = frame_type + + #: The length of the data portion of the unknown frame. + self.length = length + + def __str__(self) -> str: + return ( + "UnknownFrameError: Unknown frame type 0x%X received, " + "length %d bytes" % (self.frame_type, self.length) + ) + + +class InvalidPaddingError(HyperframeError): + """ + A frame with invalid padding was received. + + .. versionchanged:: 6.0.0 + Changed base class from `ValueError` to :class:`HyperframeError` + """ + pass + + +class InvalidFrameError(HyperframeError): + """ + Parsing a frame failed because the data was not laid out appropriately. + + .. versionadded:: 3.0.2 + + .. versionchanged:: 6.0.0 + Changed base class from `ValueError` to :class:`HyperframeError` + """ + pass + + +class InvalidDataError(HyperframeError): + """ + Content or data of a frame was is invalid or violates the specification. + + .. versionadded:: 6.0.0 + """ + pass diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/hyperframe/src/hyperframe/flags.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/hyperframe/src/hyperframe/flags.py new file mode 100644 index 0000000000..f00cb99727 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/hyperframe/src/hyperframe/flags.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +""" +hyperframe/flags +~~~~~~~~~~~~~~~~ + +Defines basic Flag and Flags data structures. +""" +from collections.abc import MutableSet +from typing import NamedTuple, Iterable, Set, Iterator + + +class Flag(NamedTuple): + name: str + bit: int + + +class Flags(MutableSet): # type: ignore + """ + A simple MutableSet implementation that will only accept known flags as + elements. + + Will behave like a regular set(), except that a ValueError will be thrown + when .add()ing unexpected flags. + """ + def __init__(self, defined_flags: Iterable[Flag]): + self._valid_flags = set(flag.name for flag in defined_flags) + self._flags: Set[str] = set() + + def __repr__(self) -> str: + return repr(sorted(list(self._flags))) + + def __contains__(self, x: object) -> bool: + return self._flags.__contains__(x) + + def __iter__(self) -> Iterator[str]: + return self._flags.__iter__() + + def __len__(self) -> int: + return self._flags.__len__() + + def discard(self, value: str) -> None: + return self._flags.discard(value) + + def add(self, value: str) -> None: + if value not in self._valid_flags: + raise ValueError( + "Unexpected flag: {}. Valid flags are: {}".format( + value, self._valid_flags + ) + ) + return self._flags.add(value) diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/hyperframe/src/hyperframe/frame.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/hyperframe/src/hyperframe/frame.py new file mode 100644 index 0000000000..f4e75e35e3 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/hyperframe/src/hyperframe/frame.py @@ -0,0 +1,969 @@ +# -*- coding: utf-8 -*- +""" +hyperframe/frame +~~~~~~~~~~~~~~~~ + +Defines framing logic for HTTP/2. Provides both classes to represent framed +data and logic for aiding the connection when it comes to reading from the +socket. +""" +import struct +import binascii + +from .exceptions import ( + UnknownFrameError, InvalidPaddingError, InvalidFrameError, InvalidDataError +) +from .flags import Flag, Flags +from typing import Optional, Tuple, List, Iterable, Any, Dict, Type + + +# The maximum initial length of a frame. Some frames have shorter maximum +# lengths. +FRAME_MAX_LEN = (2 ** 14) + +# The maximum allowed length of a frame. +FRAME_MAX_ALLOWED_LEN = (2 ** 24) - 1 + +# Stream association enumerations. +_STREAM_ASSOC_HAS_STREAM = "has-stream" +_STREAM_ASSOC_NO_STREAM = "no-stream" +_STREAM_ASSOC_EITHER = "either" + +# Structs for packing and unpacking +_STRUCT_HBBBL = struct.Struct(">HBBBL") +_STRUCT_LL = struct.Struct(">LL") +_STRUCT_HL = struct.Struct(">HL") +_STRUCT_LB = struct.Struct(">LB") +_STRUCT_L = struct.Struct(">L") +_STRUCT_H = struct.Struct(">H") +_STRUCT_B = struct.Struct(">B") + + +class Frame: + """ + The base class for all HTTP/2 frames. + """ + #: The flags defined on this type of frame. + defined_flags: List[Flag] = [] + + #: The byte used to define the type of the frame. + type: Optional[int] = None + + # If 'has-stream', the frame's stream_id must be non-zero. If 'no-stream', + # it must be zero. If 'either', it's not checked. + stream_association: Optional[str] = None + + def __init__(self, stream_id: int, flags: Iterable[str] = ()) -> None: + #: The stream identifier for the stream this frame was received on. + #: Set to 0 for frames sent on the connection (stream-id 0). + self.stream_id = stream_id + + #: The flags set for this frame. + self.flags = Flags(self.defined_flags) + + #: The frame length, excluding the nine-byte header. + self.body_len = 0 + + for flag in flags: + self.flags.add(flag) + + if (not self.stream_id and + self.stream_association == _STREAM_ASSOC_HAS_STREAM): + raise InvalidDataError( + 'Stream ID must be non-zero for {}'.format( + type(self).__name__, + ) + ) + if (self.stream_id and + self.stream_association == _STREAM_ASSOC_NO_STREAM): + raise InvalidDataError( + 'Stream ID must be zero for {} with stream_id={}'.format( + type(self).__name__, + self.stream_id, + ) + ) + + def __repr__(self) -> str: + return ( + "{}(stream_id={}, flags={}): {}" + ).format( + type(self).__name__, + self.stream_id, + repr(self.flags), + self._body_repr(), + ) + + def _body_repr(self) -> str: + # More specific implementation may be provided by subclasses of Frame. + # This fallback shows the serialized (and truncated) body content. + return _raw_data_repr(self.serialize_body()) + + @staticmethod + def explain(data: memoryview) -> Tuple["Frame", int]: + """ + Takes a bytestring and tries to parse a single frame and print it. + + This function is only provided for debugging purposes. + + :param data: A memoryview object containing the raw data of at least + one complete frame (header and body). + + .. versionadded:: 6.0.0 + """ + frame, length = Frame.parse_frame_header(data[:9]) + frame.parse_body(data[9:9 + length]) + print(frame) + return frame, length + + @staticmethod + def parse_frame_header(header: memoryview, strict: bool = False) -> Tuple["Frame", int]: + """ + Takes a 9-byte frame header and returns a tuple of the appropriate + Frame object and the length that needs to be read from the socket. + + This populates the flags field, and determines how long the body is. + + :param header: A memoryview object containing the 9-byte frame header + data of a frame. Must not contain more or less. + + :param strict: Whether to raise an exception when encountering a frame + not defined by spec and implemented by hyperframe. + + :raises hyperframe.exceptions.UnknownFrameError: If a frame of unknown + type is received. + + .. versionchanged:: 5.0.0 + Added :param:`strict` to accommodate :class:`ExtensionFrame` + """ + try: + fields = _STRUCT_HBBBL.unpack(header) + except struct.error: + raise InvalidFrameError("Invalid frame header") + + # First 24 bits are frame length. + length = (fields[0] << 8) + fields[1] + type = fields[2] + flags = fields[3] + stream_id = fields[4] & 0x7FFFFFFF + + try: + frame = FRAMES[type](stream_id) + except KeyError: + if strict: + raise UnknownFrameError(type, length) + frame = ExtensionFrame(type=type, stream_id=stream_id) + + frame.parse_flags(flags) + return (frame, length) + + def parse_flags(self, flag_byte: int) -> Flags: + for flag, flag_bit in self.defined_flags: + if flag_byte & flag_bit: + self.flags.add(flag) + + return self.flags + + def serialize(self) -> bytes: + """ + Convert a frame into a bytestring, representing the serialized form of + the frame. + """ + body = self.serialize_body() + self.body_len = len(body) + + # Build the common frame header. + # First, get the flags. + flags = 0 + + for flag, flag_bit in self.defined_flags: + if flag in self.flags: + flags |= flag_bit + + header = _STRUCT_HBBBL.pack( + (self.body_len >> 8) & 0xFFFF, # Length spread over top 24 bits + self.body_len & 0xFF, + self.type, + flags, + self.stream_id & 0x7FFFFFFF # Stream ID is 32 bits. + ) + + return header + body + + def serialize_body(self) -> bytes: + raise NotImplementedError() + + def parse_body(self, data: memoryview) -> None: + """ + Given the body of a frame, parses it into frame data. This populates + the non-header parts of the frame: that is, it does not populate the + stream ID or flags. + + :param data: A memoryview object containing the body data of the frame. + Must not contain *more* data than the length returned by + :meth:`parse_frame_header + `. + """ + raise NotImplementedError() + + +class Padding: + """ + Mixin for frames that contain padding. Defines extra fields that can be + used and set by frames that can be padded. + """ + def __init__(self, stream_id: int, pad_length: int = 0, **kwargs: Any) -> None: + super().__init__(stream_id, **kwargs) # type: ignore + + #: The length of the padding to use. + self.pad_length = pad_length + + def serialize_padding_data(self) -> bytes: + if 'PADDED' in self.flags: # type: ignore + return _STRUCT_B.pack(self.pad_length) + return b'' + + def parse_padding_data(self, data: memoryview) -> int: + if 'PADDED' in self.flags: # type: ignore + try: + self.pad_length = struct.unpack('!B', data[:1])[0] + except struct.error: + raise InvalidFrameError("Invalid Padding data") + return 1 + return 0 + + #: .. deprecated:: 5.2.1 + #: Use self.pad_length instead. + @property + def total_padding(self) -> int: # pragma: no cover + import warnings + warnings.warn( + "total_padding contains the same information as pad_length.", + DeprecationWarning + ) + return self.pad_length + + +class Priority: + """ + Mixin for frames that contain priority data. Defines extra fields that can + be used and set by frames that contain priority data. + """ + def __init__(self, + stream_id: int, + depends_on: int = 0x0, + stream_weight: int = 0x0, + exclusive: bool = False, + **kwargs: Any) -> None: + super().__init__(stream_id, **kwargs) # type: ignore + + #: The stream ID of the stream on which this stream depends. + self.depends_on = depends_on + + #: The weight of the stream. This is an integer between 0 and 256. + self.stream_weight = stream_weight + + #: Whether the exclusive bit was set. + self.exclusive = exclusive + + def serialize_priority_data(self) -> bytes: + return _STRUCT_LB.pack( + self.depends_on + (0x80000000 if self.exclusive else 0), + self.stream_weight + ) + + def parse_priority_data(self, data: memoryview) -> int: + try: + self.depends_on, self.stream_weight = _STRUCT_LB.unpack(data[:5]) + except struct.error: + raise InvalidFrameError("Invalid Priority data") + + self.exclusive = True if self.depends_on >> 31 else False + self.depends_on &= 0x7FFFFFFF + return 5 + + +class DataFrame(Padding, Frame): + """ + DATA frames convey arbitrary, variable-length sequences of octets + associated with a stream. One or more DATA frames are used, for instance, + to carry HTTP request or response payloads. + """ + #: The flags defined for DATA frames. + defined_flags = [ + Flag('END_STREAM', 0x01), + Flag('PADDED', 0x08), + ] + + #: The type byte for data frames. + type = 0x0 + + stream_association = _STREAM_ASSOC_HAS_STREAM + + def __init__(self, stream_id: int, data: bytes = b'', **kwargs: Any) -> None: + super().__init__(stream_id, **kwargs) + + #: The data contained on this frame. + self.data = data + + def serialize_body(self) -> bytes: + padding_data = self.serialize_padding_data() + padding = b'\0' * self.pad_length + if isinstance(self.data, memoryview): + self.data = self.data.tobytes() + return b''.join([padding_data, self.data, padding]) + + def parse_body(self, data: memoryview) -> None: + padding_data_length = self.parse_padding_data(data) + self.data = ( + data[padding_data_length:len(data)-self.pad_length].tobytes() + ) + self.body_len = len(data) + + if self.pad_length and self.pad_length >= self.body_len: + raise InvalidPaddingError("Padding is too long.") + + @property + def flow_controlled_length(self) -> int: + """ + The length of the frame that needs to be accounted for when considering + flow control. + """ + padding_len = 0 + if 'PADDED' in self.flags: + # Account for extra 1-byte padding length field, which is still + # present if possibly zero-valued. + padding_len = self.pad_length + 1 + return len(self.data) + padding_len + + +class PriorityFrame(Priority, Frame): + """ + The PRIORITY frame specifies the sender-advised priority of a stream. It + can be sent at any time for an existing stream. This enables + reprioritisation of existing streams. + """ + #: The flags defined for PRIORITY frames. + defined_flags: List[Flag] = [] + + #: The type byte defined for PRIORITY frames. + type = 0x02 + + stream_association = _STREAM_ASSOC_HAS_STREAM + + def _body_repr(self) -> str: + return "exclusive={}, depends_on={}, stream_weight={}".format( + self.exclusive, + self.depends_on, + self.stream_weight + ) + + def serialize_body(self) -> bytes: + return self.serialize_priority_data() + + def parse_body(self, data: memoryview) -> None: + if len(data) > 5: + raise InvalidFrameError( + "PRIORITY must have 5 byte body: actual length %s." % + len(data) + ) + + self.parse_priority_data(data) + self.body_len = 5 + + +class RstStreamFrame(Frame): + """ + The RST_STREAM frame allows for abnormal termination of a stream. When sent + by the initiator of a stream, it indicates that they wish to cancel the + stream or that an error condition has occurred. When sent by the receiver + of a stream, it indicates that either the receiver is rejecting the stream, + requesting that the stream be cancelled or that an error condition has + occurred. + """ + #: The flags defined for RST_STREAM frames. + defined_flags: List[Flag] = [] + + #: The type byte defined for RST_STREAM frames. + type = 0x03 + + stream_association = _STREAM_ASSOC_HAS_STREAM + + def __init__(self, stream_id: int, error_code: int = 0, **kwargs: Any) -> None: + super().__init__(stream_id, **kwargs) + + #: The error code used when resetting the stream. + self.error_code = error_code + + def _body_repr(self) -> str: + return "error_code={}".format( + self.error_code, + ) + + def serialize_body(self) -> bytes: + return _STRUCT_L.pack(self.error_code) + + def parse_body(self, data: memoryview) -> None: + if len(data) != 4: + raise InvalidFrameError( + "RST_STREAM must have 4 byte body: actual length %s." % + len(data) + ) + + try: + self.error_code = _STRUCT_L.unpack(data)[0] + except struct.error: # pragma: no cover + raise InvalidFrameError("Invalid RST_STREAM body") + + self.body_len = 4 + + +class SettingsFrame(Frame): + """ + The SETTINGS frame conveys configuration parameters that affect how + endpoints communicate. The parameters are either constraints on peer + behavior or preferences. + + Settings are not negotiated. Settings describe characteristics of the + sending peer, which are used by the receiving peer. Different values for + the same setting can be advertised by each peer. For example, a client + might set a high initial flow control window, whereas a server might set a + lower value to conserve resources. + """ + #: The flags defined for SETTINGS frames. + defined_flags = [Flag('ACK', 0x01)] + + #: The type byte defined for SETTINGS frames. + type = 0x04 + + stream_association = _STREAM_ASSOC_NO_STREAM + + # We need to define the known settings, they may as well be class + # attributes. + #: The byte that signals the SETTINGS_HEADER_TABLE_SIZE setting. + HEADER_TABLE_SIZE = 0x01 + #: The byte that signals the SETTINGS_ENABLE_PUSH setting. + ENABLE_PUSH = 0x02 + #: The byte that signals the SETTINGS_MAX_CONCURRENT_STREAMS setting. + MAX_CONCURRENT_STREAMS = 0x03 + #: The byte that signals the SETTINGS_INITIAL_WINDOW_SIZE setting. + INITIAL_WINDOW_SIZE = 0x04 + #: The byte that signals the SETTINGS_MAX_FRAME_SIZE setting. + MAX_FRAME_SIZE = 0x05 + #: The byte that signals the SETTINGS_MAX_HEADER_LIST_SIZE setting. + MAX_HEADER_LIST_SIZE = 0x06 + #: The byte that signals SETTINGS_ENABLE_CONNECT_PROTOCOL setting. + ENABLE_CONNECT_PROTOCOL = 0x08 + + def __init__(self, stream_id: int = 0, settings: Optional[Dict[int, int]] = None, **kwargs: Any) -> None: + super().__init__(stream_id, **kwargs) + + if settings and "ACK" in kwargs.get("flags", ()): + raise InvalidDataError( + "Settings must be empty if ACK flag is set." + ) + + #: A dictionary of the setting type byte to the value of the setting. + self.settings = settings or {} + + def _body_repr(self) -> str: + return "settings={}".format( + self.settings, + ) + + def serialize_body(self) -> bytes: + return b''.join([_STRUCT_HL.pack(setting & 0xFF, value) + for setting, value in self.settings.items()]) + + def parse_body(self, data: memoryview) -> None: + if 'ACK' in self.flags and len(data) > 0: + raise InvalidDataError( + "SETTINGS ack frame must not have payload: got %s bytes" % + len(data) + ) + + body_len = 0 + for i in range(0, len(data), 6): + try: + name, value = _STRUCT_HL.unpack(data[i:i+6]) + except struct.error: + raise InvalidFrameError("Invalid SETTINGS body") + + self.settings[name] = value + body_len += 6 + + self.body_len = body_len + + +class PushPromiseFrame(Padding, Frame): + """ + The PUSH_PROMISE frame is used to notify the peer endpoint in advance of + streams the sender intends to initiate. + """ + #: The flags defined for PUSH_PROMISE frames. + defined_flags = [ + Flag('END_HEADERS', 0x04), + Flag('PADDED', 0x08) + ] + + #: The type byte defined for PUSH_PROMISE frames. + type = 0x05 + + stream_association = _STREAM_ASSOC_HAS_STREAM + + def __init__(self, stream_id: int, promised_stream_id: int = 0, data: bytes = b'', **kwargs: Any) -> None: + super().__init__(stream_id, **kwargs) + + #: The stream ID that is promised by this frame. + self.promised_stream_id = promised_stream_id + + #: The HPACK-encoded header block for the simulated request on the new + #: stream. + self.data = data + + def _body_repr(self) -> str: + return "promised_stream_id={}, data={}".format( + self.promised_stream_id, + _raw_data_repr(self.data), + ) + + def serialize_body(self) -> bytes: + padding_data = self.serialize_padding_data() + padding = b'\0' * self.pad_length + data = _STRUCT_L.pack(self.promised_stream_id) + return b''.join([padding_data, data, self.data, padding]) + + def parse_body(self, data: memoryview) -> None: + padding_data_length = self.parse_padding_data(data) + + try: + self.promised_stream_id = _STRUCT_L.unpack( + data[padding_data_length:padding_data_length + 4] + )[0] + except struct.error: + raise InvalidFrameError("Invalid PUSH_PROMISE body") + + self.data = ( + data[padding_data_length + 4:len(data)-self.pad_length].tobytes() + ) + self.body_len = len(data) + + if self.promised_stream_id == 0 or self.promised_stream_id % 2 != 0: + raise InvalidDataError( + "Invalid PUSH_PROMISE promised stream id: %s" % + self.promised_stream_id + ) + + if self.pad_length and self.pad_length >= self.body_len: + raise InvalidPaddingError("Padding is too long.") + + +class PingFrame(Frame): + """ + The PING frame is a mechanism for measuring a minimal round-trip time from + the sender, as well as determining whether an idle connection is still + functional. PING frames can be sent from any endpoint. + """ + #: The flags defined for PING frames. + defined_flags = [Flag('ACK', 0x01)] + + #: The type byte defined for PING frames. + type = 0x06 + + stream_association = _STREAM_ASSOC_NO_STREAM + + def __init__(self, stream_id: int = 0, opaque_data: bytes = b'', **kwargs: Any) -> None: + super().__init__(stream_id, **kwargs) + + #: The opaque data sent in this PING frame, as a bytestring. + self.opaque_data = opaque_data + + def _body_repr(self) -> str: + return "opaque_data={!r}".format( + self.opaque_data, + ) + + def serialize_body(self) -> bytes: + if len(self.opaque_data) > 8: + raise InvalidFrameError( + "PING frame may not have more than 8 bytes of data, got %r" % + self.opaque_data + ) + + data = self.opaque_data + data += b'\x00' * (8 - len(self.opaque_data)) + return data + + def parse_body(self, data: memoryview) -> None: + if len(data) != 8: + raise InvalidFrameError( + "PING frame must have 8 byte length: got %s" % len(data) + ) + + self.opaque_data = data.tobytes() + self.body_len = 8 + + +class GoAwayFrame(Frame): + """ + The GOAWAY frame informs the remote peer to stop creating streams on this + connection. It can be sent from the client or the server. Once sent, the + sender will ignore frames sent on new streams for the remainder of the + connection. + """ + #: The flags defined for GOAWAY frames. + defined_flags: List[Flag] = [] + + #: The type byte defined for GOAWAY frames. + type = 0x07 + + stream_association = _STREAM_ASSOC_NO_STREAM + + def __init__(self, + stream_id: int = 0, + last_stream_id: int = 0, + error_code: int = 0, + additional_data: bytes = b'', + **kwargs: Any) -> None: + super().__init__(stream_id, **kwargs) + + #: The last stream ID definitely seen by the remote peer. + self.last_stream_id = last_stream_id + + #: The error code for connection teardown. + self.error_code = error_code + + #: Any additional data sent in the GOAWAY. + self.additional_data = additional_data + + def _body_repr(self) -> str: + return "last_stream_id={}, error_code={}, additional_data={!r}".format( + self.last_stream_id, + self.error_code, + self.additional_data, + ) + + def serialize_body(self) -> bytes: + data = _STRUCT_LL.pack( + self.last_stream_id & 0x7FFFFFFF, + self.error_code + ) + data += self.additional_data + + return data + + def parse_body(self, data: memoryview) -> None: + try: + self.last_stream_id, self.error_code = _STRUCT_LL.unpack( + data[:8] + ) + except struct.error: + raise InvalidFrameError("Invalid GOAWAY body.") + + self.body_len = len(data) + + if len(data) > 8: + self.additional_data = data[8:].tobytes() + + +class WindowUpdateFrame(Frame): + """ + The WINDOW_UPDATE frame is used to implement flow control. + + Flow control operates at two levels: on each individual stream and on the + entire connection. + + Both types of flow control are hop by hop; that is, only between the two + endpoints. Intermediaries do not forward WINDOW_UPDATE frames between + dependent connections. However, throttling of data transfer by any receiver + can indirectly cause the propagation of flow control information toward the + original sender. + """ + #: The flags defined for WINDOW_UPDATE frames. + defined_flags: List[Flag] = [] + + #: The type byte defined for WINDOW_UPDATE frames. + type = 0x08 + + stream_association = _STREAM_ASSOC_EITHER + + def __init__(self, stream_id: int, window_increment: int = 0, **kwargs: Any) -> None: + super().__init__(stream_id, **kwargs) + + #: The amount the flow control window is to be incremented. + self.window_increment = window_increment + + def _body_repr(self) -> str: + return "window_increment={}".format( + self.window_increment, + ) + + def serialize_body(self) -> bytes: + return _STRUCT_L.pack(self.window_increment & 0x7FFFFFFF) + + def parse_body(self, data: memoryview) -> None: + if len(data) > 4: + raise InvalidFrameError( + "WINDOW_UPDATE frame must have 4 byte length: got %s" % + len(data) + ) + + try: + self.window_increment = _STRUCT_L.unpack(data)[0] + except struct.error: + raise InvalidFrameError("Invalid WINDOW_UPDATE body") + + if not 1 <= self.window_increment <= 2**31-1: + raise InvalidDataError( + "WINDOW_UPDATE increment must be between 1 to 2^31-1" + ) + + self.body_len = 4 + + +class HeadersFrame(Padding, Priority, Frame): + """ + The HEADERS frame carries name-value pairs. It is used to open a stream. + HEADERS frames can be sent on a stream in the "open" or "half closed + (remote)" states. + + The HeadersFrame class is actually basically a data frame in this + implementation, because of the requirement to control the sizes of frames. + A header block fragment that doesn't fit in an entire HEADERS frame needs + to be followed with CONTINUATION frames. From the perspective of the frame + building code the header block is an opaque data segment. + """ + #: The flags defined for HEADERS frames. + defined_flags = [ + Flag('END_STREAM', 0x01), + Flag('END_HEADERS', 0x04), + Flag('PADDED', 0x08), + Flag('PRIORITY', 0x20), + ] + + #: The type byte defined for HEADERS frames. + type = 0x01 + + stream_association = _STREAM_ASSOC_HAS_STREAM + + def __init__(self, stream_id: int, data: bytes = b'', **kwargs: Any) -> None: + super().__init__(stream_id, **kwargs) + + #: The HPACK-encoded header block. + self.data = data + + def _body_repr(self) -> str: + return "exclusive={}, depends_on={}, stream_weight={}, data={}".format( + self.exclusive, + self.depends_on, + self.stream_weight, + _raw_data_repr(self.data), + ) + + def serialize_body(self) -> bytes: + padding_data = self.serialize_padding_data() + padding = b'\0' * self.pad_length + + if 'PRIORITY' in self.flags: + priority_data = self.serialize_priority_data() + else: + priority_data = b'' + + return b''.join([padding_data, priority_data, self.data, padding]) + + def parse_body(self, data: memoryview) -> None: + padding_data_length = self.parse_padding_data(data) + data = data[padding_data_length:] + + if 'PRIORITY' in self.flags: + priority_data_length = self.parse_priority_data(data) + else: + priority_data_length = 0 + + self.body_len = len(data) + self.data = ( + data[priority_data_length:len(data)-self.pad_length].tobytes() + ) + + if self.pad_length and self.pad_length >= self.body_len: + raise InvalidPaddingError("Padding is too long.") + + +class ContinuationFrame(Frame): + """ + The CONTINUATION frame is used to continue a sequence of header block + fragments. Any number of CONTINUATION frames can be sent on an existing + stream, as long as the preceding frame on the same stream is one of + HEADERS, PUSH_PROMISE or CONTINUATION without the END_HEADERS flag set. + + Much like the HEADERS frame, hyper treats this as an opaque data frame with + different flags and a different type. + """ + #: The flags defined for CONTINUATION frames. + defined_flags = [Flag('END_HEADERS', 0x04)] + + #: The type byte defined for CONTINUATION frames. + type = 0x09 + + stream_association = _STREAM_ASSOC_HAS_STREAM + + def __init__(self, stream_id: int, data: bytes = b'', **kwargs: Any) -> None: + super().__init__(stream_id, **kwargs) + + #: The HPACK-encoded header block. + self.data = data + + def _body_repr(self) -> str: + return "data={}".format( + _raw_data_repr(self.data), + ) + + def serialize_body(self) -> bytes: + return self.data + + def parse_body(self, data: memoryview) -> None: + self.data = data.tobytes() + self.body_len = len(data) + + +class AltSvcFrame(Frame): + """ + The ALTSVC frame is used to advertise alternate services that the current + host, or a different one, can understand. This frame is standardised as + part of RFC 7838. + + This frame does no work to validate that the ALTSVC field parameter is + acceptable per the rules of RFC 7838. + + .. note:: If the ``stream_id`` of this frame is nonzero, the origin field + must have zero length. Conversely, if the ``stream_id`` of this + frame is zero, the origin field must have nonzero length. Put + another way, a valid ALTSVC frame has ``stream_id != 0`` XOR + ``len(origin) != 0``. + """ + type = 0xA + + stream_association = _STREAM_ASSOC_EITHER + + def __init__(self, stream_id: int, origin: bytes = b'', field: bytes = b'', **kwargs: Any) -> None: + super().__init__(stream_id, **kwargs) + + if not isinstance(origin, bytes): + raise InvalidDataError("AltSvc origin must be bytestring.") + if not isinstance(field, bytes): + raise InvalidDataError("AltSvc field must be a bytestring.") + self.origin = origin + self.field = field + + def _body_repr(self) -> str: + return "origin={!r}, field={!r}".format( + self.origin, + self.field, + ) + + def serialize_body(self) -> bytes: + origin_len = _STRUCT_H.pack(len(self.origin)) + return b''.join([origin_len, self.origin, self.field]) + + def parse_body(self, data: memoryview) -> None: + try: + origin_len = _STRUCT_H.unpack(data[0:2])[0] + self.origin = data[2:2+origin_len].tobytes() + + if len(self.origin) != origin_len: + raise InvalidFrameError("Invalid ALTSVC frame body.") + + self.field = data[2+origin_len:].tobytes() + except (struct.error, ValueError): + raise InvalidFrameError("Invalid ALTSVC frame body.") + + self.body_len = len(data) + + +class ExtensionFrame(Frame): + """ + ExtensionFrame is used to wrap frames which are not natively interpretable + by hyperframe. + + Although certain byte prefixes are ordained by specification to have + certain contextual meanings, frames with other prefixes are not prohibited, + and may be used to communicate arbitrary meaning between HTTP/2 peers. + + Thus, hyperframe, rather than raising an exception when such a frame is + encountered, wraps it in a generic frame to be properly acted upon by + upstream consumers which might have additional context on how to use it. + + .. versionadded:: 5.0.0 + """ + + stream_association = _STREAM_ASSOC_EITHER + + def __init__(self, type: int, stream_id: int, flag_byte: int = 0x0, body: bytes = b'', **kwargs: Any) -> None: + super().__init__(stream_id, **kwargs) + self.type = type + self.flag_byte = flag_byte + self.body = body + + def _body_repr(self) -> str: + return "type={}, flag_byte={}, body={}".format( + self.type, + self.flag_byte, + _raw_data_repr(self.body), + ) + + def parse_flags(self, flag_byte: int) -> None: # type: ignore + """ + For extension frames, we parse the flags by just storing a flag byte. + """ + self.flag_byte = flag_byte + + def parse_body(self, data: memoryview) -> None: + self.body = data.tobytes() + self.body_len = len(data) + + def serialize(self) -> bytes: + """ + A broad override of the serialize method that ensures that the data + comes back out exactly as it came in. This should not be used in most + user code: it exists only as a helper method if frames need to be + reconstituted. + """ + # Build the frame header. + # First, get the flags. + flags = self.flag_byte + + header = _STRUCT_HBBBL.pack( + (self.body_len >> 8) & 0xFFFF, # Length spread over top 24 bits + self.body_len & 0xFF, + self.type, + flags, + self.stream_id & 0x7FFFFFFF # Stream ID is 32 bits. + ) + + return header + self.body + + +def _raw_data_repr(data: Optional[bytes]) -> str: + if not data: + return "None" + r = binascii.hexlify(data).decode('ascii') + if len(r) > 20: + r = r[:20] + "..." + return "" + + +_FRAME_CLASSES: List[Type[Frame]] = [ + DataFrame, + HeadersFrame, + PriorityFrame, + RstStreamFrame, + SettingsFrame, + PushPromiseFrame, + PingFrame, + GoAwayFrame, + WindowUpdateFrame, + ContinuationFrame, + AltSvcFrame, +] +#: FRAMES maps the type byte for each frame to the class used to represent that +#: frame. +FRAMES = {cls.type: cls for cls in _FRAME_CLASSES} diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/hyperframe/src/hyperframe/py.typed b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/hyperframe/src/hyperframe/py.typed new file mode 100644 index 0000000000..e69de29bb2 diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/__init__.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/__init__.py new file mode 100644 index 0000000000..8f4ade0582 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/__init__.py @@ -0,0 +1,172 @@ +# Copyright 2011, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +""" A Standalone WebSocket Server for testing purposes + +pywebsocket3 is an API that provides WebSocket functionalities with +a standalone WebSocket server. It is intended for testing or +experimental purposes. + +Installation +============ +1. Follow standalone server documentation to start running the +standalone server. It can be read by running the following command: + + $ pydoc pywebsocket3.standalone + +2. Once the standalone server is launched verify it by accessing +http://localhost[:port]/console.html. Include the port number when +specified on launch. If everything is working correctly, you +will see a simple echo console. + + +Writing WebSocket handlers +========================== + +When a WebSocket request comes in, the resource name +specified in the handshake is considered as if it is a file path under + and the handler defined in +/_wsh.py is invoked. + +For example, if the resource name is /example/chat, the handler defined in +/example/chat_wsh.py is invoked. + +A WebSocket handler is composed of the following three functions: + + web_socket_do_extra_handshake(request) + web_socket_transfer_data(request) + web_socket_passive_closing_handshake(request) + +where: + request: mod_python request. + +web_socket_do_extra_handshake is called during the handshake after the +headers are successfully parsed and WebSocket properties (ws_origin, +and ws_resource) are added to request. A handler +can reject the request by raising an exception. + +A request object has the following properties that you can use during the +extra handshake (web_socket_do_extra_handshake): +- ws_resource +- ws_origin +- ws_version +- ws_extensions +- ws_deflate +- ws_protocol +- ws_requested_protocols + +The last two are a bit tricky. See the next subsection. + + +Subprotocol Negotiation +----------------------- + +ws_protocol is always set to None when +web_socket_do_extra_handshake is called. If ws_requested_protocols is not +None, you must choose one subprotocol from this list and set it to +ws_protocol. + +Data Transfer +------------- + +web_socket_transfer_data is called after the handshake completed +successfully. A handler can receive/send messages from/to the client +using request. pywebsocket3.msgutil module provides utilities +for data transfer. + +You can receive a message by the following statement. + + message = request.ws_stream.receive_message() + +This call blocks until any complete text frame arrives, and the payload data +of the incoming frame will be stored into message. When you're using IETF +HyBi 00 or later protocol, receive_message() will return None on receiving +client-initiated closing handshake. When any error occurs, receive_message() +will raise some exception. + +You can send a message by the following statement. + + request.ws_stream.send_message(message) + + +Closing Connection +------------------ + +Executing the following statement or just return-ing from +web_socket_transfer_data cause connection close. + + request.ws_stream.close_connection() + +close_connection will wait +for closing handshake acknowledgement coming from the client. When it +couldn't receive a valid acknowledgement, raises an exception. + +web_socket_passive_closing_handshake is called after the server receives +incoming closing frame from the client peer immediately. You can specify +code and reason by return values. They are sent as a outgoing closing frame +from the server. A request object has the following properties that you can +use in web_socket_passive_closing_handshake. +- ws_close_code +- ws_close_reason + + +Threading +--------- + +A WebSocket handler must be thread-safe. The standalone +server uses threads by default. + + +Configuring WebSocket Extension Processors +------------------------------------------ + +See extensions.py for supported WebSocket extensions. Note that they are +unstable and their APIs are subject to change substantially. + +A request object has these extension processing related attributes. + +- ws_requested_extensions: + + A list of common.ExtensionParameter instances representing extension + parameters received from the client in the client's opening handshake. + You shouldn't modify it manually. + +- ws_extensions: + + A list of common.ExtensionParameter instances representing extension + parameters to send back to the client in the server's opening handshake. + You shouldn't touch it directly. Instead, call methods on extension + processors. + +- ws_extension_processors: + + A list of loaded extension processors. Find the processor for the + extension you want to configure from it, and call its methods. +""" + +# vi:sts=4 sw=4 et tw=72 diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/_stream_exceptions.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/_stream_exceptions.py new file mode 100644 index 0000000000..b47878bc4a --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/_stream_exceptions.py @@ -0,0 +1,82 @@ +# Copyright 2020, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +"""Stream Exceptions. +""" + +# Note: request.connection.write/read are used in this module, even though +# mod_python document says that they should be used only in connection +# handlers. Unfortunately, we have no other options. For example, +# request.write/read are not suitable because they don't allow direct raw bytes +# writing/reading. + + +# Exceptions +class ConnectionTerminatedException(Exception): + """This exception will be raised when a connection is terminated + unexpectedly. + """ + + pass + + +class InvalidFrameException(ConnectionTerminatedException): + """This exception will be raised when we received an invalid frame we + cannot parse. + """ + + pass + + +class BadOperationException(Exception): + """This exception will be raised when send_message() is called on + server-terminated connection or receive_message() is called on + client-terminated connection. + """ + + pass + + +class UnsupportedFrameException(Exception): + """This exception will be raised when we receive a frame with flag, opcode + we cannot handle. Handlers can just catch and ignore this exception and + call receive_message() again to continue processing the next frame. + """ + + pass + + +class InvalidUTF8Exception(Exception): + """This exception will be raised when we receive a text frame which + contains invalid UTF-8 strings. + """ + + pass + + +# vi:sts=4 sw=4 et diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/common.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/common.py new file mode 100644 index 0000000000..a3321e4c68 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/common.py @@ -0,0 +1,275 @@ +# Copyright 2012, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +"""This file must not depend on any module specific to the WebSocket protocol. +""" + +from __future__ import absolute_import + +from pywebsocket3 import http_header_util + + +# Additional log level definitions. +LOGLEVEL_FINE = 9 + +# Constants indicating WebSocket protocol version. +VERSION_HYBI13 = 13 +VERSION_HYBI14 = 13 +VERSION_HYBI15 = 13 +VERSION_HYBI16 = 13 +VERSION_HYBI17 = 13 + +# Constants indicating WebSocket protocol latest version. +VERSION_HYBI_LATEST = VERSION_HYBI13 + +# Port numbers +DEFAULT_WEB_SOCKET_PORT = 80 +DEFAULT_WEB_SOCKET_SECURE_PORT = 443 + +# Schemes +WEB_SOCKET_SCHEME = 'ws' +WEB_SOCKET_SECURE_SCHEME = 'wss' + +# Frame opcodes defined in the spec. +OPCODE_CONTINUATION = 0x0 +OPCODE_TEXT = 0x1 +OPCODE_BINARY = 0x2 +OPCODE_CLOSE = 0x8 +OPCODE_PING = 0x9 +OPCODE_PONG = 0xa + +# UUID for the opening handshake and frame masking. +WEBSOCKET_ACCEPT_UUID = b'258EAFA5-E914-47DA-95CA-C5AB0DC85B11' + +# Opening handshake header names and expected values. +UPGRADE_HEADER = 'Upgrade' +WEBSOCKET_UPGRADE_TYPE = 'websocket' +CONNECTION_HEADER = 'Connection' +UPGRADE_CONNECTION_TYPE = 'Upgrade' +HOST_HEADER = 'Host' +ORIGIN_HEADER = 'Origin' +SEC_WEBSOCKET_KEY_HEADER = 'Sec-WebSocket-Key' +SEC_WEBSOCKET_ACCEPT_HEADER = 'Sec-WebSocket-Accept' +SEC_WEBSOCKET_VERSION_HEADER = 'Sec-WebSocket-Version' +SEC_WEBSOCKET_PROTOCOL_HEADER = 'Sec-WebSocket-Protocol' +SEC_WEBSOCKET_EXTENSIONS_HEADER = 'Sec-WebSocket-Extensions' + +# Extensions +PERMESSAGE_DEFLATE_EXTENSION = 'permessage-deflate' + +# Status codes +# Code STATUS_NO_STATUS_RECEIVED, STATUS_ABNORMAL_CLOSURE, and +# STATUS_TLS_HANDSHAKE are pseudo codes to indicate specific error cases. +# Could not be used for codes in actual closing frames. +# Application level errors must use codes in the range +# STATUS_USER_REGISTERED_BASE to STATUS_USER_PRIVATE_MAX. The codes in the +# range STATUS_USER_REGISTERED_BASE to STATUS_USER_REGISTERED_MAX are managed +# by IANA. Usually application must define user protocol level errors in the +# range STATUS_USER_PRIVATE_BASE to STATUS_USER_PRIVATE_MAX. +STATUS_NORMAL_CLOSURE = 1000 +STATUS_GOING_AWAY = 1001 +STATUS_PROTOCOL_ERROR = 1002 +STATUS_UNSUPPORTED_DATA = 1003 +STATUS_NO_STATUS_RECEIVED = 1005 +STATUS_ABNORMAL_CLOSURE = 1006 +STATUS_INVALID_FRAME_PAYLOAD_DATA = 1007 +STATUS_POLICY_VIOLATION = 1008 +STATUS_MESSAGE_TOO_BIG = 1009 +STATUS_MANDATORY_EXTENSION = 1010 +STATUS_INTERNAL_ENDPOINT_ERROR = 1011 +STATUS_TLS_HANDSHAKE = 1015 +STATUS_USER_REGISTERED_BASE = 3000 +STATUS_USER_REGISTERED_MAX = 3999 +STATUS_USER_PRIVATE_BASE = 4000 +STATUS_USER_PRIVATE_MAX = 4999 +# Following definitions are aliases to keep compatibility. Applications must +# not use these obsoleted definitions anymore. +STATUS_NORMAL = STATUS_NORMAL_CLOSURE +STATUS_UNSUPPORTED = STATUS_UNSUPPORTED_DATA +STATUS_CODE_NOT_AVAILABLE = STATUS_NO_STATUS_RECEIVED +STATUS_ABNORMAL_CLOSE = STATUS_ABNORMAL_CLOSURE +STATUS_INVALID_FRAME_PAYLOAD = STATUS_INVALID_FRAME_PAYLOAD_DATA +STATUS_MANDATORY_EXT = STATUS_MANDATORY_EXTENSION + +# HTTP status codes +HTTP_STATUS_BAD_REQUEST = 400 +HTTP_STATUS_FORBIDDEN = 403 +HTTP_STATUS_NOT_FOUND = 404 + + +def is_control_opcode(opcode): + return (opcode >> 3) == 1 + + +class ExtensionParameter(object): + """This is exchanged on extension negotiation in opening handshake.""" + def __init__(self, name): + self._name = name + # TODO(tyoshino): Change the data structure to more efficient one such + # as dict when the spec changes to say like + # - Parameter names must be unique + # - The order of parameters is not significant + self._parameters = [] + + def name(self): + """Return the extension name.""" + return self._name + + def add_parameter(self, name, value): + """Add a parameter.""" + self._parameters.append((name, value)) + + def get_parameters(self): + """Return the parameters.""" + return self._parameters + + def get_parameter_names(self): + """Return the names of the parameters.""" + return [name for name, unused_value in self._parameters] + + def has_parameter(self, name): + """Test if a parameter exists.""" + for param_name, param_value in self._parameters: + if param_name == name: + return True + return False + + def get_parameter_value(self, name): + """Get the value of a specific parameter.""" + for param_name, param_value in self._parameters: + if param_name == name: + return param_value + + +class ExtensionParsingException(Exception): + """Exception to handle errors in extension parsing.""" + def __init__(self, name): + super(ExtensionParsingException, self).__init__(name) + + +def _parse_extension_param(state, definition): + param_name = http_header_util.consume_token(state) + + if param_name is None: + raise ExtensionParsingException('No valid parameter name found') + + http_header_util.consume_lwses(state) + + if not http_header_util.consume_string(state, '='): + definition.add_parameter(param_name, None) + return + + http_header_util.consume_lwses(state) + + # TODO(tyoshino): Add code to validate that parsed param_value is token + param_value = http_header_util.consume_token_or_quoted_string(state) + if param_value is None: + raise ExtensionParsingException( + 'No valid parameter value found on the right-hand side of ' + 'parameter %r' % param_name) + + definition.add_parameter(param_name, param_value) + + +def _parse_extension(state): + extension_token = http_header_util.consume_token(state) + if extension_token is None: + return None + + extension = ExtensionParameter(extension_token) + + while True: + http_header_util.consume_lwses(state) + + if not http_header_util.consume_string(state, ';'): + break + + http_header_util.consume_lwses(state) + + try: + _parse_extension_param(state, extension) + except ExtensionParsingException as e: + raise ExtensionParsingException( + 'Failed to parse parameter for %r (%r)' % (extension_token, e)) + + return extension + + +def parse_extensions(data): + """Parse Sec-WebSocket-Extensions header value. + + Returns a list of ExtensionParameter objects. + Leading LWSes must be trimmed. + """ + state = http_header_util.ParsingState(data) + + extension_list = [] + while True: + extension = _parse_extension(state) + if extension is not None: + extension_list.append(extension) + + http_header_util.consume_lwses(state) + + if http_header_util.peek(state) is None: + break + + if not http_header_util.consume_string(state, ','): + raise ExtensionParsingException( + 'Failed to parse Sec-WebSocket-Extensions header: ' + 'Expected a comma but found %r' % http_header_util.peek(state)) + + http_header_util.consume_lwses(state) + + if len(extension_list) == 0: + raise ExtensionParsingException('No valid extension entry found') + + return extension_list + + +def format_extension(extension): + """Format an ExtensionParameter object.""" + formatted_params = [extension.name()] + for param_name, param_value in extension.get_parameters(): + if param_value is None: + formatted_params.append(param_name) + else: + quoted_value = http_header_util.quote_if_necessary(param_value) + formatted_params.append('%s=%s' % (param_name, quoted_value)) + return '; '.join(formatted_params) + + +def format_extensions(extension_list): + """Format a list of ExtensionParameter objects.""" + formatted_extension_list = [] + for extension in extension_list: + formatted_extension_list.append(format_extension(extension)) + return ', '.join(formatted_extension_list) + + +# vi:sts=4 sw=4 et diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/dispatch.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/dispatch.py new file mode 100644 index 0000000000..fd35ceab29 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/dispatch.py @@ -0,0 +1,391 @@ +# Copyright 2012, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +"""Dispatch WebSocket request. +""" + +from __future__ import absolute_import + +import io +import os +import re +import traceback + +from pywebsocket3 import ( + common, + handshake, + msgutil, + stream, + util +) + +_SOURCE_PATH_PATTERN = re.compile(r'(?i)_wsh\.py$') +_SOURCE_SUFFIX = '_wsh.py' +_DO_EXTRA_HANDSHAKE_HANDLER_NAME = 'web_socket_do_extra_handshake' +_TRANSFER_DATA_HANDLER_NAME = 'web_socket_transfer_data' +_PASSIVE_CLOSING_HANDSHAKE_HANDLER_NAME = ( + 'web_socket_passive_closing_handshake') + + +class DispatchException(Exception): + """Exception in dispatching WebSocket request.""" + def __init__(self, name, status=common.HTTP_STATUS_NOT_FOUND): + super(DispatchException, self).__init__(name) + self.status = status + + +def _default_passive_closing_handshake_handler(request): + """Default web_socket_passive_closing_handshake handler.""" + + return common.STATUS_NORMAL_CLOSURE, '' + + +def _normalize_path(path): + """Normalize path. + + Args: + path: the path to normalize. + + Path is converted to the absolute path. + The input path can use either '\\' or '/' as the separator. + The normalized path always uses '/' regardless of the platform. + """ + + path = path.replace('\\', os.path.sep) + path = os.path.realpath(path) + path = path.replace('\\', '/') + return path + + +def _create_path_to_resource_converter(base_dir): + """Returns a function that converts the path of a WebSocket handler source + file to a resource string by removing the path to the base directory from + its head, removing _SOURCE_SUFFIX from its tail, and replacing path + separators in it with '/'. + + Args: + base_dir: the path to the base directory. + """ + + base_dir = _normalize_path(base_dir) + + base_len = len(base_dir) + suffix_len = len(_SOURCE_SUFFIX) + + def converter(path): + if not path.endswith(_SOURCE_SUFFIX): + return None + # _normalize_path must not be used because resolving symlink breaks + # following path check. + path = path.replace('\\', '/') + if not path.startswith(base_dir): + return None + return path[base_len:-suffix_len] + + return converter + + +def _enumerate_handler_file_paths(directory): + """Returns a generator that enumerates WebSocket Handler source file names + in the given directory. + """ + + for root, unused_dirs, files in os.walk(directory): + for base in files: + path = os.path.join(root, base) + if _SOURCE_PATH_PATTERN.search(path): + yield path + + +class _HandlerSuite(object): + """A handler suite holder class.""" + def __init__(self, do_extra_handshake, transfer_data, + passive_closing_handshake): + self.do_extra_handshake = do_extra_handshake + self.transfer_data = transfer_data + self.passive_closing_handshake = passive_closing_handshake + + +def _source_handler_file(handler_definition): + """Source a handler definition string. + + Args: + handler_definition: a string containing Python statements that define + handler functions. + """ + + global_dic = {} + try: + # This statement is gramatically different in python 2 and 3. + # Hence, yapf will complain about this. To overcome this, we disable + # yapf for this line. + exec(handler_definition, global_dic) # yapf: disable + except Exception: + raise DispatchException('Error in sourcing handler:' + + traceback.format_exc()) + passive_closing_handshake_handler = None + try: + passive_closing_handshake_handler = _extract_handler( + global_dic, _PASSIVE_CLOSING_HANDSHAKE_HANDLER_NAME) + except Exception: + passive_closing_handshake_handler = ( + _default_passive_closing_handshake_handler) + return _HandlerSuite( + _extract_handler(global_dic, _DO_EXTRA_HANDSHAKE_HANDLER_NAME), + _extract_handler(global_dic, _TRANSFER_DATA_HANDLER_NAME), + passive_closing_handshake_handler) + + +def _extract_handler(dic, name): + """Extracts a callable with the specified name from the given dictionary + dic. + """ + + if name not in dic: + raise DispatchException('%s is not defined.' % name) + handler = dic[name] + if not callable(handler): + raise DispatchException('%s is not callable.' % name) + return handler + + +class Dispatcher(object): + """Dispatches WebSocket requests. + + This class maintains a map from resource name to handlers. + """ + def __init__(self, + root_dir, + scan_dir=None, + allow_handlers_outside_root_dir=True, + handler_encoding=None): + """Construct an instance. + + Args: + root_dir: The directory where handler definition files are + placed. + scan_dir: The directory where handler definition files are + searched. scan_dir must be a directory under root_dir, + including root_dir itself. If scan_dir is None, + root_dir is used as scan_dir. scan_dir can be useful + in saving scan time when root_dir contains many + subdirectories. + allow_handlers_outside_root_dir: Scans handler files even if their + canonical path is not under root_dir. + """ + + self._logger = util.get_class_logger(self) + + self._handler_suite_map = {} + self._source_warnings = [] + if scan_dir is None: + scan_dir = root_dir + if not os.path.realpath(scan_dir).startswith( + os.path.realpath(root_dir)): + raise DispatchException('scan_dir:%s must be a directory under ' + 'root_dir:%s.' % (scan_dir, root_dir)) + self._source_handler_files_in_dir(root_dir, scan_dir, + allow_handlers_outside_root_dir, + handler_encoding) + + def add_resource_path_alias(self, alias_resource_path, + existing_resource_path): + """Add resource path alias. + + Once added, request to alias_resource_path would be handled by + handler registered for existing_resource_path. + + Args: + alias_resource_path: alias resource path + existing_resource_path: existing resource path + """ + try: + handler_suite = self._handler_suite_map[existing_resource_path] + self._handler_suite_map[alias_resource_path] = handler_suite + except KeyError: + raise DispatchException('No handler for: %r' % + existing_resource_path) + + def source_warnings(self): + """Return warnings in sourcing handlers.""" + + return self._source_warnings + + def do_extra_handshake(self, request): + """Do extra checking in WebSocket handshake. + + Select a handler based on request.uri and call its + web_socket_do_extra_handshake function. + + Args: + request: mod_python request. + + Raises: + DispatchException: when handler was not found + AbortedByUserException: when user handler abort connection + HandshakeException: when opening handshake failed + """ + + handler_suite = self.get_handler_suite(request.ws_resource) + if handler_suite is None: + raise DispatchException('No handler for: %r' % request.ws_resource) + do_extra_handshake_ = handler_suite.do_extra_handshake + try: + do_extra_handshake_(request) + except handshake.AbortedByUserException as e: + # Re-raise to tell the caller of this function to finish this + # connection without sending any error. + self._logger.debug('%s', traceback.format_exc()) + raise + except Exception as e: + util.prepend_message_to_exception( + '%s raised exception for %s: ' % + (_DO_EXTRA_HANDSHAKE_HANDLER_NAME, request.ws_resource), e) + raise handshake.HandshakeException(e, common.HTTP_STATUS_FORBIDDEN) + + def transfer_data(self, request): + """Let a handler transfer_data with a WebSocket client. + + Select a handler based on request.ws_resource and call its + web_socket_transfer_data function. + + Args: + request: mod_python request. + + Raises: + DispatchException: when handler was not found + AbortedByUserException: when user handler abort connection + """ + + # TODO(tyoshino): Terminate underlying TCP connection if possible. + try: + handler_suite = self.get_handler_suite(request.ws_resource) + if handler_suite is None: + raise DispatchException('No handler for: %r' % + request.ws_resource) + transfer_data_ = handler_suite.transfer_data + transfer_data_(request) + + if not request.server_terminated: + request.ws_stream.close_connection() + # Catch non-critical exceptions the handler didn't handle. + except handshake.AbortedByUserException as e: + self._logger.debug('%s', traceback.format_exc()) + raise + except msgutil.BadOperationException as e: + self._logger.debug('%s', e) + request.ws_stream.close_connection( + common.STATUS_INTERNAL_ENDPOINT_ERROR) + except msgutil.InvalidFrameException as e: + # InvalidFrameException must be caught before + # ConnectionTerminatedException that catches InvalidFrameException. + self._logger.debug('%s', e) + request.ws_stream.close_connection(common.STATUS_PROTOCOL_ERROR) + except msgutil.UnsupportedFrameException as e: + self._logger.debug('%s', e) + request.ws_stream.close_connection(common.STATUS_UNSUPPORTED_DATA) + except stream.InvalidUTF8Exception as e: + self._logger.debug('%s', e) + request.ws_stream.close_connection( + common.STATUS_INVALID_FRAME_PAYLOAD_DATA) + except msgutil.ConnectionTerminatedException as e: + self._logger.debug('%s', e) + except Exception as e: + # Any other exceptions are forwarded to the caller of this + # function. + util.prepend_message_to_exception( + '%s raised exception for %s: ' % + (_TRANSFER_DATA_HANDLER_NAME, request.ws_resource), e) + raise + + def passive_closing_handshake(self, request): + """Prepare code and reason for responding client initiated closing + handshake. + """ + + handler_suite = self.get_handler_suite(request.ws_resource) + if handler_suite is None: + return _default_passive_closing_handshake_handler(request) + return handler_suite.passive_closing_handshake(request) + + def get_handler_suite(self, resource): + """Retrieves two handlers (one for extra handshake processing, and one + for data transfer) for the given request as a HandlerSuite object. + """ + + fragment = None + if '#' in resource: + resource, fragment = resource.split('#', 1) + if '?' in resource: + resource = resource.split('?', 1)[0] + handler_suite = self._handler_suite_map.get(resource) + if handler_suite and fragment: + raise DispatchException( + 'Fragment identifiers MUST NOT be used on WebSocket URIs', + common.HTTP_STATUS_BAD_REQUEST) + return handler_suite + + def _source_handler_files_in_dir(self, root_dir, scan_dir, + allow_handlers_outside_root_dir, + handler_encoding): + """Source all the handler source files in the scan_dir directory. + + The resource path is determined relative to root_dir. + """ + + # We build a map from resource to handler code assuming that there's + # only one path from root_dir to scan_dir and it can be obtained by + # comparing realpath of them. + + # Here we cannot use abspath. See + # https://bugs.webkit.org/show_bug.cgi?id=31603 + + convert = _create_path_to_resource_converter(root_dir) + scan_realpath = os.path.realpath(scan_dir) + root_realpath = os.path.realpath(root_dir) + for path in _enumerate_handler_file_paths(scan_realpath): + if (not allow_handlers_outside_root_dir and + (not os.path.realpath(path).startswith(root_realpath))): + self._logger.debug( + 'Canonical path of %s is not under root directory' % path) + continue + try: + with io.open(path, encoding=handler_encoding) as handler_file: + handler_suite = _source_handler_file(handler_file.read()) + except DispatchException as e: + self._source_warnings.append('%s: %s' % (path, e)) + continue + resource = convert(path) + if resource is None: + self._logger.debug('Path to resource conversion on %s failed' % + path) + else: + self._handler_suite_map[convert(path)] = handler_suite + + +# vi:sts=4 sw=4 et diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/extensions.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/extensions.py new file mode 100644 index 0000000000..4b5b9e8fb2 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/extensions.py @@ -0,0 +1,474 @@ +# Copyright 2012, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import + +from pywebsocket3 import common, util +from pywebsocket3.http_header_util import quote_if_necessary + +# The list of available server side extension processor classes. +_available_processors = {} + + +class ExtensionProcessorInterface(object): + def __init__(self, request): + self._logger = util.get_class_logger(self) + + self._request = request + self._active = True + + def request(self): + return self._request + + def name(self): + return None + + def check_consistency_with_other_processors(self, processors): + pass + + def set_active(self, active): + self._active = active + + def is_active(self): + return self._active + + def _get_extension_response_internal(self): + return None + + def get_extension_response(self): + if not self._active: + self._logger.debug('Extension %s is deactivated', self.name()) + return None + + response = self._get_extension_response_internal() + if response is None: + self._active = False + return response + + def _setup_stream_options_internal(self, stream_options): + pass + + def setup_stream_options(self, stream_options): + if self._active: + self._setup_stream_options_internal(stream_options) + + +def _log_outgoing_compression_ratio(logger, original_bytes, filtered_bytes, + average_ratio): + # Print inf when ratio is not available. + ratio = float('inf') + if original_bytes != 0: + ratio = float(filtered_bytes) / original_bytes + + logger.debug('Outgoing compression ratio: %f (average: %f)' % + (ratio, average_ratio)) + + +def _log_incoming_compression_ratio(logger, received_bytes, filtered_bytes, + average_ratio): + # Print inf when ratio is not available. + ratio = float('inf') + if filtered_bytes != 0: + ratio = float(received_bytes) / filtered_bytes + + logger.debug('Incoming compression ratio: %f (average: %f)' % + (ratio, average_ratio)) + + +def _parse_window_bits(bits): + """Return parsed integer value iff the given string conforms to the + grammar of the window bits extension parameters. + """ + + if bits is None: + raise ValueError('Value is required') + + # For non integer values such as "10.0", ValueError will be raised. + int_bits = int(bits) + + # First condition is to drop leading zero case e.g. "08". + if bits != str(int_bits) or int_bits < 8 or int_bits > 15: + raise ValueError('Invalid value: %r' % bits) + + return int_bits + + +class _AverageRatioCalculator(object): + """Stores total bytes of original and result data, and calculates average + result / original ratio. + """ + def __init__(self): + self._total_original_bytes = 0 + self._total_result_bytes = 0 + + def add_original_bytes(self, value): + self._total_original_bytes += value + + def add_result_bytes(self, value): + self._total_result_bytes += value + + def get_average_ratio(self): + if self._total_original_bytes != 0: + return (float(self._total_result_bytes) / + self._total_original_bytes) + else: + return float('inf') + + +class PerMessageDeflateExtensionProcessor(ExtensionProcessorInterface): + """permessage-deflate extension processor. + + Specification: + http://tools.ietf.org/html/draft-ietf-hybi-permessage-compression-08 + """ + + _SERVER_MAX_WINDOW_BITS_PARAM = 'server_max_window_bits' + _SERVER_NO_CONTEXT_TAKEOVER_PARAM = 'server_no_context_takeover' + _CLIENT_MAX_WINDOW_BITS_PARAM = 'client_max_window_bits' + _CLIENT_NO_CONTEXT_TAKEOVER_PARAM = 'client_no_context_takeover' + + def __init__(self, request): + """Construct PerMessageDeflateExtensionProcessor.""" + + ExtensionProcessorInterface.__init__(self, request) + self._logger = util.get_class_logger(self) + + self._preferred_client_max_window_bits = None + self._client_no_context_takeover = False + + def name(self): + # This method returns "deflate" (not "permessage-deflate") for + # compatibility. + return 'deflate' + + def _get_extension_response_internal(self): + for name in self._request.get_parameter_names(): + if name not in [ + self._SERVER_MAX_WINDOW_BITS_PARAM, + self._SERVER_NO_CONTEXT_TAKEOVER_PARAM, + self._CLIENT_MAX_WINDOW_BITS_PARAM + ]: + self._logger.debug('Unknown parameter: %r', name) + return None + + server_max_window_bits = None + if self._request.has_parameter(self._SERVER_MAX_WINDOW_BITS_PARAM): + server_max_window_bits = self._request.get_parameter_value( + self._SERVER_MAX_WINDOW_BITS_PARAM) + try: + server_max_window_bits = _parse_window_bits( + server_max_window_bits) + except ValueError as e: + self._logger.debug('Bad %s parameter: %r', + self._SERVER_MAX_WINDOW_BITS_PARAM, e) + return None + + server_no_context_takeover = self._request.has_parameter( + self._SERVER_NO_CONTEXT_TAKEOVER_PARAM) + if (server_no_context_takeover and self._request.get_parameter_value( + self._SERVER_NO_CONTEXT_TAKEOVER_PARAM) is not None): + self._logger.debug('%s parameter must not have a value: %r', + self._SERVER_NO_CONTEXT_TAKEOVER_PARAM, + server_no_context_takeover) + return None + + # client_max_window_bits from a client indicates whether the client can + # accept client_max_window_bits from a server or not. + client_client_max_window_bits = self._request.has_parameter( + self._CLIENT_MAX_WINDOW_BITS_PARAM) + if (client_client_max_window_bits + and self._request.get_parameter_value( + self._CLIENT_MAX_WINDOW_BITS_PARAM) is not None): + self._logger.debug( + '%s parameter must not have a value in a ' + 'client\'s opening handshake: %r', + self._CLIENT_MAX_WINDOW_BITS_PARAM, + client_client_max_window_bits) + return None + + self._rfc1979_deflater = util._RFC1979Deflater( + server_max_window_bits, server_no_context_takeover) + + # Note that we prepare for incoming messages compressed with window + # bits upto 15 regardless of the client_max_window_bits value to be + # sent to the client. + self._rfc1979_inflater = util._RFC1979Inflater() + + self._framer = _PerMessageDeflateFramer(server_max_window_bits, + server_no_context_takeover) + self._framer.set_bfinal(False) + self._framer.set_compress_outgoing_enabled(True) + + response = common.ExtensionParameter(self._request.name()) + + if server_max_window_bits is not None: + response.add_parameter(self._SERVER_MAX_WINDOW_BITS_PARAM, + str(server_max_window_bits)) + + if server_no_context_takeover: + response.add_parameter(self._SERVER_NO_CONTEXT_TAKEOVER_PARAM, + None) + + if self._preferred_client_max_window_bits is not None: + if not client_client_max_window_bits: + self._logger.debug( + 'Processor is configured to use %s but ' + 'the client cannot accept it', + self._CLIENT_MAX_WINDOW_BITS_PARAM) + return None + response.add_parameter(self._CLIENT_MAX_WINDOW_BITS_PARAM, + str(self._preferred_client_max_window_bits)) + + if self._client_no_context_takeover: + response.add_parameter(self._CLIENT_NO_CONTEXT_TAKEOVER_PARAM, + None) + + self._logger.debug('Enable %s extension (' + 'request: server_max_window_bits=%s; ' + 'server_no_context_takeover=%r, ' + 'response: client_max_window_bits=%s; ' + 'client_no_context_takeover=%r)' % + (self._request.name(), server_max_window_bits, + server_no_context_takeover, + self._preferred_client_max_window_bits, + self._client_no_context_takeover)) + + return response + + def _setup_stream_options_internal(self, stream_options): + self._framer.setup_stream_options(stream_options) + + def set_client_max_window_bits(self, value): + """If this option is specified, this class adds the + client_max_window_bits extension parameter to the handshake response, + but doesn't reduce the LZ77 sliding window size of its inflater. + I.e., you can use this for testing client implementation but cannot + reduce memory usage of this class. + + If this method has been called with True and an offer without the + client_max_window_bits extension parameter is received, + + - (When processing the permessage-deflate extension) this processor + declines the request. + - (When processing the permessage-compress extension) this processor + accepts the request. + """ + + self._preferred_client_max_window_bits = value + + def set_client_no_context_takeover(self, value): + """If this option is specified, this class adds the + client_no_context_takeover extension parameter to the handshake + response, but doesn't reset inflater for each message. I.e., you can + use this for testing client implementation but cannot reduce memory + usage of this class. + """ + + self._client_no_context_takeover = value + + def set_bfinal(self, value): + self._framer.set_bfinal(value) + + def enable_outgoing_compression(self): + self._framer.set_compress_outgoing_enabled(True) + + def disable_outgoing_compression(self): + self._framer.set_compress_outgoing_enabled(False) + + +class _PerMessageDeflateFramer(object): + """A framer for extensions with per-message DEFLATE feature.""" + def __init__(self, deflate_max_window_bits, deflate_no_context_takeover): + self._logger = util.get_class_logger(self) + + self._rfc1979_deflater = util._RFC1979Deflater( + deflate_max_window_bits, deflate_no_context_takeover) + + self._rfc1979_inflater = util._RFC1979Inflater() + + self._bfinal = False + + self._compress_outgoing_enabled = False + + # True if a message is fragmented and compression is ongoing. + self._compress_ongoing = False + + # Calculates + # (Total outgoing bytes supplied to this filter) / + # (Total bytes sent to the network after applying this filter) + self._outgoing_average_ratio_calculator = _AverageRatioCalculator() + + # Calculates + # (Total bytes received from the network) / + # (Total incoming bytes obtained after applying this filter) + self._incoming_average_ratio_calculator = _AverageRatioCalculator() + + def set_bfinal(self, value): + self._bfinal = value + + def set_compress_outgoing_enabled(self, value): + self._compress_outgoing_enabled = value + + def _process_incoming_message(self, message, decompress): + if not decompress: + return message + + received_payload_size = len(message) + self._incoming_average_ratio_calculator.add_result_bytes( + received_payload_size) + + message = self._rfc1979_inflater.filter(message) + + filtered_payload_size = len(message) + self._incoming_average_ratio_calculator.add_original_bytes( + filtered_payload_size) + + _log_incoming_compression_ratio( + self._logger, received_payload_size, filtered_payload_size, + self._incoming_average_ratio_calculator.get_average_ratio()) + + return message + + def _process_outgoing_message(self, message, end, binary): + if not binary: + message = message.encode('utf-8') + + if not self._compress_outgoing_enabled: + return message + + original_payload_size = len(message) + self._outgoing_average_ratio_calculator.add_original_bytes( + original_payload_size) + + message = self._rfc1979_deflater.filter(message, + end=end, + bfinal=self._bfinal) + + filtered_payload_size = len(message) + self._outgoing_average_ratio_calculator.add_result_bytes( + filtered_payload_size) + + _log_outgoing_compression_ratio( + self._logger, original_payload_size, filtered_payload_size, + self._outgoing_average_ratio_calculator.get_average_ratio()) + + if not self._compress_ongoing: + self._outgoing_frame_filter.set_compression_bit() + self._compress_ongoing = not end + return message + + def _process_incoming_frame(self, frame): + if frame.rsv1 == 1 and not common.is_control_opcode(frame.opcode): + self._incoming_message_filter.decompress_next_message() + frame.rsv1 = 0 + + def _process_outgoing_frame(self, frame, compression_bit): + if (not compression_bit or common.is_control_opcode(frame.opcode)): + return + + frame.rsv1 = 1 + + def setup_stream_options(self, stream_options): + """Creates filters and sets them to the StreamOptions.""" + class _OutgoingMessageFilter(object): + def __init__(self, parent): + self._parent = parent + + def filter(self, message, end=True, binary=False): + return self._parent._process_outgoing_message( + message, end, binary) + + class _IncomingMessageFilter(object): + def __init__(self, parent): + self._parent = parent + self._decompress_next_message = False + + def decompress_next_message(self): + self._decompress_next_message = True + + def filter(self, message): + message = self._parent._process_incoming_message( + message, self._decompress_next_message) + self._decompress_next_message = False + return message + + self._outgoing_message_filter = _OutgoingMessageFilter(self) + self._incoming_message_filter = _IncomingMessageFilter(self) + stream_options.outgoing_message_filters.append( + self._outgoing_message_filter) + stream_options.incoming_message_filters.append( + self._incoming_message_filter) + + class _OutgoingFrameFilter(object): + def __init__(self, parent): + self._parent = parent + self._set_compression_bit = False + + def set_compression_bit(self): + self._set_compression_bit = True + + def filter(self, frame): + self._parent._process_outgoing_frame(frame, + self._set_compression_bit) + self._set_compression_bit = False + + class _IncomingFrameFilter(object): + def __init__(self, parent): + self._parent = parent + + def filter(self, frame): + self._parent._process_incoming_frame(frame) + + self._outgoing_frame_filter = _OutgoingFrameFilter(self) + self._incoming_frame_filter = _IncomingFrameFilter(self) + stream_options.outgoing_frame_filters.append( + self._outgoing_frame_filter) + stream_options.incoming_frame_filters.append( + self._incoming_frame_filter) + + stream_options.encode_text_message_to_utf8 = False + + +_available_processors[common.PERMESSAGE_DEFLATE_EXTENSION] = ( + PerMessageDeflateExtensionProcessor) + + +def get_extension_processor(extension_request): + """Given an ExtensionParameter representing an extension offer received + from a client, configures and returns an instance of the corresponding + extension processor class. + """ + + processor_class = _available_processors.get(extension_request.name()) + if processor_class is None: + return None + return processor_class(extension_request) + + +# vi:sts=4 sw=4 et diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/handshake/__init__.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/handshake/__init__.py new file mode 100644 index 0000000000..275e447552 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/handshake/__init__.py @@ -0,0 +1,105 @@ +# Copyright 2011, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +"""WebSocket opening handshake processor. This class try to apply available +opening handshake processors for each protocol version until a connection is +successfully established. +""" + +from __future__ import absolute_import + +import logging + +from pywebsocket3 import common +from pywebsocket3.handshake import hybi +# Export AbortedByUserException, HandshakeException, and VersionException +# symbol from this module. +from pywebsocket3.handshake.base import ( + AbortedByUserException, + HandshakeException, + VersionException +) + + +_LOGGER = logging.getLogger(__name__) + + +def do_handshake(request, dispatcher): + """Performs WebSocket handshake. + + Args: + request: mod_python request. + dispatcher: Dispatcher (dispatch.Dispatcher). + + Handshaker will add attributes such as ws_resource in performing + handshake. + """ + + _LOGGER.debug('Client\'s opening handshake resource: %r', request.uri) + # To print mimetools.Message as escaped one-line string, we converts + # headers_in to dict object. Without conversion, if we use %r, it just + # prints the type and address, and if we use %s, it prints the original + # header string as multiple lines. + # + # Both mimetools.Message and MpTable_Type of mod_python can be + # converted to dict. + # + # mimetools.Message.__str__ returns the original header string. + # dict(mimetools.Message object) returns the map from header names to + # header values. While MpTable_Type doesn't have such __str__ but just + # __repr__ which formats itself as well as dictionary object. + _LOGGER.debug('Client\'s opening handshake headers: %r', + dict(request.headers_in)) + + handshakers = [] + handshakers.append(('RFC 6455', hybi.Handshaker(request, dispatcher))) + + for name, handshaker in handshakers: + _LOGGER.debug('Trying protocol version %s', name) + try: + handshaker.do_handshake() + _LOGGER.info('Established (%s protocol)', name) + return + except HandshakeException as e: + _LOGGER.debug( + 'Failed to complete opening handshake as %s protocol: %r', + name, e) + if e.status: + raise e + except AbortedByUserException as e: + raise + except VersionException as e: + raise + + # TODO(toyoshim): Add a test to cover the case all handshakers fail. + raise HandshakeException( + 'Failed to complete opening handshake for all available protocols', + status=common.HTTP_STATUS_BAD_REQUEST) + + +# vi:sts=4 sw=4 et diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/handshake/base.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/handshake/base.py new file mode 100644 index 0000000000..561f7b650a --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/handshake/base.py @@ -0,0 +1,393 @@ +# Copyright 2012, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +"""Common functions and exceptions used by WebSocket opening handshake +processors. +""" + +from __future__ import absolute_import + +from pywebsocket3 import common, http_header_util, util +from pywebsocket3.extensions import get_extension_processor +from pywebsocket3.stream import Stream, StreamOptions + +from six.moves import map, range + + +# Defining aliases for values used frequently. +_VERSION_LATEST = common.VERSION_HYBI_LATEST +_VERSION_LATEST_STRING = str(_VERSION_LATEST) +_SUPPORTED_VERSIONS = [ + _VERSION_LATEST, +] + + +class AbortedByUserException(Exception): + """Exception for aborting a connection intentionally. + + If this exception is raised in do_extra_handshake handler, the connection + will be abandoned. No other WebSocket or HTTP(S) handler will be invoked. + + If this exception is raised in transfer_data_handler, the connection will + be closed without closing handshake. No other WebSocket or HTTP(S) handler + will be invoked. + """ + + pass + + +class HandshakeException(Exception): + """This exception will be raised when an error occurred while processing + WebSocket initial handshake. + """ + def __init__(self, name, status=None): + super(HandshakeException, self).__init__(name) + self.status = status + + +class VersionException(Exception): + """This exception will be raised when a version of client request does not + match with version the server supports. + """ + def __init__(self, name, supported_versions=''): + """Construct an instance. + + Args: + supported_version: a str object to show supported hybi versions. + (e.g. '13') + """ + super(VersionException, self).__init__(name) + self.supported_versions = supported_versions + + +def get_default_port(is_secure): + if is_secure: + return common.DEFAULT_WEB_SOCKET_SECURE_PORT + else: + return common.DEFAULT_WEB_SOCKET_PORT + + +def validate_subprotocol(subprotocol): + """Validate a value in the Sec-WebSocket-Protocol field. + + See the Section 4.1., 4.2.2., and 4.3. of RFC 6455. + """ + + if not subprotocol: + raise HandshakeException('Invalid subprotocol name: empty') + + # Parameter should be encoded HTTP token. + state = http_header_util.ParsingState(subprotocol) + token = http_header_util.consume_token(state) + rest = http_header_util.peek(state) + # If |rest| is not None, |subprotocol| is not one token or invalid. If + # |rest| is None, |token| must not be None because |subprotocol| is + # concatenation of |token| and |rest| and is not None. + if rest is not None: + raise HandshakeException('Invalid non-token string in subprotocol ' + 'name: %r' % rest) + + +def parse_host_header(request): + fields = request.headers_in[common.HOST_HEADER].split(':', 1) + if len(fields) == 1: + return fields[0], get_default_port(request.is_https()) + try: + return fields[0], int(fields[1]) + except ValueError as e: + raise HandshakeException('Invalid port number format: %r' % e) + + +def get_mandatory_header(request, key): + value = request.headers_in.get(key) + if value is None: + raise HandshakeException('Header %s is not defined' % key) + return value + + +def validate_mandatory_header(request, key, expected_value, fail_status=None): + value = get_mandatory_header(request, key) + + if value.lower() != expected_value.lower(): + raise HandshakeException( + 'Expected %r for header %s but found %r (case-insensitive)' % + (expected_value, key, value), + status=fail_status) + + +def parse_token_list(data): + """Parses a header value which follows 1#token and returns parsed elements + as a list of strings. + + Leading LWSes must be trimmed. + """ + + state = http_header_util.ParsingState(data) + + token_list = [] + + while True: + token = http_header_util.consume_token(state) + if token is not None: + token_list.append(token) + + http_header_util.consume_lwses(state) + + if http_header_util.peek(state) is None: + break + + if not http_header_util.consume_string(state, ','): + raise HandshakeException('Expected a comma but found %r' % + http_header_util.peek(state)) + + http_header_util.consume_lwses(state) + + if len(token_list) == 0: + raise HandshakeException('No valid token found') + + return token_list + + +class HandshakerBase(object): + def __init__(self, request, dispatcher): + self._logger = util.get_class_logger(self) + self._request = request + self._dispatcher = dispatcher + + """ subclasses must implement the five following methods """ + + def _protocol_rfc(self): + """ Return the name of the RFC that the handshake class is implementing. + """ + + raise AssertionError("subclasses should implement this method") + + def _transform_header(self, header): + """ + :param header: header name + + transform the header name if needed. For example, HTTP/2 subclass will + return the name of the header in lower case. + """ + + raise AssertionError("subclasses should implement this method") + + def _validate_request(self): + """ validate that all the mandatory fields are set """ + + raise AssertionError("subclasses should implement this method") + + def _set_accept(self): + """ Computes accept value based on Sec-WebSocket-Accept if needed. """ + + raise AssertionError("subclasses should implement this method") + + def _send_handshake(self): + """ Prepare and send the response after it has been parsed and processed. + """ + + raise AssertionError("subclasses should implement this method") + + def do_handshake(self): + self._request.ws_close_code = None + self._request.ws_close_reason = None + + # Parsing. + self._validate_request() + self._request.ws_resource = self._request.uri + self._request.ws_version = self._check_version() + + try: + self._get_origin() + self._set_protocol() + self._parse_extensions() + + self._set_accept() + + self._logger.debug('Protocol version is ' + self._protocol_rfc()) + + # Setup extension processors. + self._request.ws_extension_processors = self._get_extension_processors_requested( + ) + + # List of extra headers. The extra handshake handler may add header + # data as name/value pairs to this list and pywebsocket appends + # them to the WebSocket handshake. + self._request.extra_headers = [] + + # Extra handshake handler may modify/remove processors. + self._dispatcher.do_extra_handshake(self._request) + + stream_options = StreamOptions() + self._process_extensions(stream_options) + + self._request.ws_stream = Stream(self._request, stream_options) + + if self._request.ws_requested_protocols is not None: + if self._request.ws_protocol is None: + raise HandshakeException( + 'do_extra_handshake must choose one subprotocol from ' + 'ws_requested_protocols and set it to ws_protocol') + validate_subprotocol(self._request.ws_protocol) + + self._logger.debug('Subprotocol accepted: %r', + self._request.ws_protocol) + else: + if self._request.ws_protocol is not None: + raise HandshakeException( + 'ws_protocol must be None when the client didn\'t ' + 'request any subprotocol') + + self._send_handshake() + except HandshakeException as e: + if not e.status: + # Fallback to 400 bad request by default. + e.status = common.HTTP_STATUS_BAD_REQUEST + raise e + + def _check_version(self): + sec_websocket_version_header = self._transform_header( + common.SEC_WEBSOCKET_VERSION_HEADER) + version = get_mandatory_header(self._request, + sec_websocket_version_header) + if version == _VERSION_LATEST_STRING: + return _VERSION_LATEST + + if version.find(',') >= 0: + raise HandshakeException( + 'Multiple versions (%r) are not allowed for header %s' % + (version, sec_websocket_version_header), + status=common.HTTP_STATUS_BAD_REQUEST) + raise VersionException('Unsupported version %r for header %s' % + (version, sec_websocket_version_header), + supported_versions=', '.join( + map(str, _SUPPORTED_VERSIONS))) + + def _get_origin(self): + origin_header = self._transform_header(common.ORIGIN_HEADER) + origin = self._request.headers_in.get(origin_header) + if origin is None: + self._logger.debug('Client request does not have origin header') + self._request.ws_origin = origin + + def _set_protocol(self): + self._request.ws_protocol = None + + sec_websocket_protocol_header = self._transform_header( + common.SEC_WEBSOCKET_PROTOCOL_HEADER) + protocol_header = self._request.headers_in.get( + sec_websocket_protocol_header) + + if protocol_header is None: + self._request.ws_requested_protocols = None + return + + self._request.ws_requested_protocols = parse_token_list( + protocol_header) + self._logger.debug('Subprotocols requested: %r', + self._request.ws_requested_protocols) + + def _parse_extensions(self): + sec_websocket_extensions_header = self._transform_header( + common.SEC_WEBSOCKET_EXTENSIONS_HEADER) + extensions_header = self._request.headers_in.get( + sec_websocket_extensions_header) + if not extensions_header: + self._request.ws_requested_extensions = None + return + + try: + self._request.ws_requested_extensions = common.parse_extensions( + extensions_header) + except common.ExtensionParsingException as e: + raise HandshakeException( + 'Failed to parse sec-websocket-extensions header: %r' % e) + + self._logger.debug( + 'Extensions requested: %r', + list( + map(common.ExtensionParameter.name, + self._request.ws_requested_extensions))) + + def _get_extension_processors_requested(self): + processors = [] + if self._request.ws_requested_extensions is not None: + for extension_request in self._request.ws_requested_extensions: + processor = get_extension_processor(extension_request) + # Unknown extension requests are just ignored. + if processor is not None: + processors.append(processor) + return processors + + def _process_extensions(self, stream_options): + processors = [ + processor for processor in self._request.ws_extension_processors + if processor is not None + ] + + # Ask each processor if there are extensions on the request which + # cannot co-exist. When processor decided other processors cannot + # co-exist with it, the processor marks them (or itself) as + # "inactive". The first extension processor has the right to + # make the final call. + for processor in reversed(processors): + if processor.is_active(): + processor.check_consistency_with_other_processors(processors) + processors = [ + processor for processor in processors if processor.is_active() + ] + + accepted_extensions = [] + + for index, processor in enumerate(processors): + if not processor.is_active(): + continue + + extension_response = processor.get_extension_response() + if extension_response is None: + # Rejected. + continue + + accepted_extensions.append(extension_response) + + processor.setup_stream_options(stream_options) + + # Inactivate all of the following compression extensions. + for j in range(index + 1, len(processors)): + processors[j].set_active(False) + + if len(accepted_extensions) > 0: + self._request.ws_extensions = accepted_extensions + self._logger.debug( + 'Extensions accepted: %r', + list(map(common.ExtensionParameter.name, accepted_extensions))) + else: + self._request.ws_extensions = None + + +# vi:sts=4 sw=4 et diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/handshake/hybi.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/handshake/hybi.py new file mode 100644 index 0000000000..2e26532c3f --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/handshake/hybi.py @@ -0,0 +1,226 @@ +# Copyright 2012, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +"""This file provides the opening handshake processor for the WebSocket +protocol (RFC 6455). + +Specification: +http://tools.ietf.org/html/rfc6455 +""" + +from __future__ import absolute_import + +import base64 +import re +from hashlib import sha1 + +from pywebsocket3 import common, util +from pywebsocket3.handshake.base import ( + get_mandatory_header, + HandshakeException, + parse_token_list, + validate_mandatory_header, + HandshakerBase +) + + +# Used to validate the value in the Sec-WebSocket-Key header strictly. RFC 4648 +# disallows non-zero padding, so the character right before == must be any of +# A, Q, g and w. +_SEC_WEBSOCKET_KEY_REGEX = re.compile('^[+/0-9A-Za-z]{21}[AQgw]==$') + + +def check_request_line(request): + # 5.1 1. The three character UTF-8 string "GET". + # 5.1 2. A UTF-8-encoded U+0020 SPACE character (0x20 byte). + if request.method != u'GET': + raise HandshakeException('Method is not GET: %r' % request.method) + + if request.protocol != u'HTTP/1.1': + raise HandshakeException('Version is not HTTP/1.1: %r' % + request.protocol) + + +def compute_accept(key): + """Computes value for the Sec-WebSocket-Accept header from value of the + Sec-WebSocket-Key header. + """ + + accept_binary = sha1(key + common.WEBSOCKET_ACCEPT_UUID).digest() + accept = base64.b64encode(accept_binary) + + return accept + + +def compute_accept_from_unicode(unicode_key): + """A wrapper function for compute_accept which takes a unicode string as an + argument, and encodes it to byte string. It then passes it on to + compute_accept. + """ + + key = unicode_key.encode('UTF-8') + return compute_accept(key) + + +def format_header(name, value): + return u'%s: %s\r\n' % (name, value) + + +class Handshaker(HandshakerBase): + """Opening handshake processor for the WebSocket protocol (RFC 6455).""" + def __init__(self, request, dispatcher): + """Construct an instance. + + Args: + request: mod_python request. + dispatcher: Dispatcher (dispatch.Dispatcher). + + Handshaker will add attributes such as ws_resource during handshake. + """ + super(Handshaker, self).__init__(request, dispatcher) + + def _transform_header(self, header): + return header + + def _protocol_rfc(self): + return 'RFC 6455' + + def _validate_connection_header(self): + connection = get_mandatory_header(self._request, + common.CONNECTION_HEADER) + + try: + connection_tokens = parse_token_list(connection) + except HandshakeException as e: + raise HandshakeException('Failed to parse %s: %s' % + (common.CONNECTION_HEADER, e)) + + connection_is_valid = False + for token in connection_tokens: + if token.lower() == common.UPGRADE_CONNECTION_TYPE.lower(): + connection_is_valid = True + break + if not connection_is_valid: + raise HandshakeException( + '%s header doesn\'t contain "%s"' % + (common.CONNECTION_HEADER, common.UPGRADE_CONNECTION_TYPE)) + + def _validate_request(self): + check_request_line(self._request) + validate_mandatory_header(self._request, common.UPGRADE_HEADER, + common.WEBSOCKET_UPGRADE_TYPE) + self._validate_connection_header() + unused_host = get_mandatory_header(self._request, common.HOST_HEADER) + + def _set_accept(self): + # Key validation, response generation. + key = self._get_key() + accept = compute_accept(key) + self._logger.debug('%s: %r (%s)', common.SEC_WEBSOCKET_ACCEPT_HEADER, + accept, util.hexify(base64.b64decode(accept))) + self._request._accept = accept + + def _validate_key(self, key): + if key.find(',') >= 0: + raise HandshakeException('Request has multiple %s header lines or ' + 'contains illegal character \',\': %r' % + (common.SEC_WEBSOCKET_KEY_HEADER, key)) + + # Validate + key_is_valid = False + try: + # Validate key by quick regex match before parsing by base64 + # module. Because base64 module skips invalid characters, we have + # to do this in advance to make this server strictly reject illegal + # keys. + if _SEC_WEBSOCKET_KEY_REGEX.match(key): + decoded_key = base64.b64decode(key) + if len(decoded_key) == 16: + key_is_valid = True + except TypeError as e: + pass + + if not key_is_valid: + raise HandshakeException('Illegal value for header %s: %r' % + (common.SEC_WEBSOCKET_KEY_HEADER, key)) + + return decoded_key + + def _get_key(self): + key = get_mandatory_header(self._request, + common.SEC_WEBSOCKET_KEY_HEADER) + + decoded_key = self._validate_key(key) + + self._logger.debug('%s: %r (%s)', common.SEC_WEBSOCKET_KEY_HEADER, key, + util.hexify(decoded_key)) + + return key.encode('UTF-8') + + def _create_handshake_response(self, accept): + response = [] + + response.append(u'HTTP/1.1 101 Switching Protocols\r\n') + + # WebSocket headers + response.append( + format_header(common.UPGRADE_HEADER, + common.WEBSOCKET_UPGRADE_TYPE)) + response.append( + format_header(common.CONNECTION_HEADER, + common.UPGRADE_CONNECTION_TYPE)) + response.append( + format_header(common.SEC_WEBSOCKET_ACCEPT_HEADER, + accept.decode('UTF-8'))) + if self._request.ws_protocol is not None: + response.append( + format_header(common.SEC_WEBSOCKET_PROTOCOL_HEADER, + self._request.ws_protocol)) + if (self._request.ws_extensions is not None + and len(self._request.ws_extensions) != 0): + response.append( + format_header( + common.SEC_WEBSOCKET_EXTENSIONS_HEADER, + common.format_extensions(self._request.ws_extensions))) + + # Headers not specific for WebSocket + for name, value in self._request.extra_headers: + response.append(format_header(name, value)) + + response.append(u'\r\n') + + return u''.join(response) + + def _send_handshake(self): + raw_response = self._create_handshake_response(self._request._accept) + self._request.connection.write(raw_response.encode('UTF-8')) + self._logger.debug('Sent server\'s opening handshake: %r', + raw_response) + + +# vi:sts=4 sw=4 et diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/http_header_util.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/http_header_util.py new file mode 100644 index 0000000000..63e698bc16 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/http_header_util.py @@ -0,0 +1,256 @@ +# Copyright 2011, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +"""Utilities for parsing and formatting headers that follow the grammar defined +in HTTP RFC http://www.ietf.org/rfc/rfc2616.txt. +""" + +from __future__ import absolute_import + +import six.moves.urllib.parse + + +_SEPARATORS = '()<>@,;:\\"/[]?={} \t' + + +def _is_char(c): + """Returns true iff c is in CHAR as specified in HTTP RFC.""" + + return ord(c) <= 127 + + +def _is_ctl(c): + """Returns true iff c is in CTL as specified in HTTP RFC.""" + + return ord(c) <= 31 or ord(c) == 127 + + +class ParsingState(object): + def __init__(self, data): + self.data = data + self.head = 0 + + +def peek(state, pos=0): + """Peeks the character at pos from the head of data.""" + + if state.head + pos >= len(state.data): + return None + + return state.data[state.head + pos] + + +def consume(state, amount=1): + """Consumes specified amount of bytes from the head and returns the + consumed bytes. If there's not enough bytes to consume, returns None. + """ + + if state.head + amount > len(state.data): + return None + + result = state.data[state.head:state.head + amount] + state.head = state.head + amount + return result + + +def consume_string(state, expected): + """Given a parsing state and a expected string, consumes the string from + the head. Returns True if consumed successfully. Otherwise, returns + False. + """ + + pos = 0 + + for c in expected: + if c != peek(state, pos): + return False + pos += 1 + + consume(state, pos) + return True + + +def consume_lws(state): + """Consumes a LWS from the head. Returns True if any LWS is consumed. + Otherwise, returns False. + + LWS = [CRLF] 1*( SP | HT ) + """ + + original_head = state.head + + consume_string(state, '\r\n') + + pos = 0 + + while True: + c = peek(state, pos) + if c == ' ' or c == '\t': + pos += 1 + else: + if pos == 0: + state.head = original_head + return False + else: + consume(state, pos) + return True + + +def consume_lwses(state): + r"""Consumes \*LWS from the head.""" + + while consume_lws(state): + pass + + +def consume_token(state): + """Consumes a token from the head. Returns the token or None if no token + was found. + """ + + pos = 0 + + while True: + c = peek(state, pos) + if c is None or c in _SEPARATORS or _is_ctl(c) or not _is_char(c): + if pos == 0: + return None + + return consume(state, pos) + else: + pos += 1 + + +def consume_token_or_quoted_string(state): + """Consumes a token or a quoted-string, and returns the token or unquoted + string. If no token or quoted-string was found, returns None. + """ + + original_head = state.head + + if not consume_string(state, '"'): + return consume_token(state) + + result = [] + + expect_quoted_pair = False + + while True: + if not expect_quoted_pair and consume_lws(state): + result.append(' ') + continue + + c = consume(state) + if c is None: + # quoted-string is not enclosed with double quotation + state.head = original_head + return None + elif expect_quoted_pair: + expect_quoted_pair = False + if _is_char(c): + result.append(c) + else: + # Non CHAR character found in quoted-pair + state.head = original_head + return None + elif c == '\\': + expect_quoted_pair = True + elif c == '"': + return ''.join(result) + elif _is_ctl(c): + # Invalid character %r found in qdtext + state.head = original_head + return None + else: + result.append(c) + + +def quote_if_necessary(s): + """Quotes arbitrary string into quoted-string.""" + + quote = False + if s == '': + return '""' + + result = [] + for c in s: + if c == '"' or c in _SEPARATORS or _is_ctl(c) or not _is_char(c): + quote = True + + if c == '"' or _is_ctl(c): + result.append('\\' + c) + else: + result.append(c) + + if quote: + return '"' + ''.join(result) + '"' + else: + return ''.join(result) + + +def parse_uri(uri): + """Parse absolute URI then return host, port and resource.""" + + parsed = six.moves.urllib.parse.urlsplit(uri) + if parsed.scheme != 'wss' and parsed.scheme != 'ws': + # |uri| must be a relative URI. + # TODO(toyoshim): Should validate |uri|. + return None, None, uri + + if parsed.hostname is None: + return None, None, None + + port = None + try: + port = parsed.port + except ValueError: + # The port property cause ValueError on invalid null port descriptions + # like 'ws://host:INVALID_PORT/path', where the assigned port is not + # *DIGIT. For python 3.6 and later, ValueError also raises when + # assigning invalid port numbers such as 'ws://host:-1/path'. Earlier + # versions simply return None and ignore invalid port attributes. + return None, None, None + + if port is None: + if parsed.scheme == 'ws': + port = 80 + else: + port = 443 + + path = parsed.path + if not path: + path += '/' + if parsed.query: + path += '?' + parsed.query + if parsed.fragment: + path += '#' + parsed.fragment + + return parsed.hostname, port, path + + +# vi:sts=4 sw=4 et diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/memorizingfile.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/memorizingfile.py new file mode 100644 index 0000000000..4ee132fae6 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/memorizingfile.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python +# +# Copyright 2011, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +"""Memorizing file. + +A memorizing file wraps a file and memorizes lines read by readline. +""" + +from __future__ import absolute_import + +import sys + + +class MemorizingFile(object): + """MemorizingFile wraps a file and memorizes lines read by readline. + + Note that data read by other methods are not memorized. This behavior + is good enough for memorizing lines SimpleHTTPServer reads before + the control reaches WebSocketRequestHandler. + """ + def __init__(self, file_, max_memorized_lines=sys.maxsize): + """Construct an instance. + + Args: + file_: the file object to wrap. + max_memorized_lines: the maximum number of lines to memorize. + Only the first max_memorized_lines are memorized. + Default: sys.maxint. + """ + self._file = file_ + self._memorized_lines = [] + self._max_memorized_lines = max_memorized_lines + self._buffered = False + self._buffered_line = None + + def __getattribute__(self, name): + """Return a file attribute. + + Returns the value overridden by this class for some attributes, + and forwards the call to _file for the other attributes. + """ + if name in ('_file', '_memorized_lines', '_max_memorized_lines', + '_buffered', '_buffered_line', 'readline', + 'get_memorized_lines'): + return object.__getattribute__(self, name) + return self._file.__getattribute__(name) + + def readline(self, size=-1): + """Override file.readline and memorize the line read. + + Note that even if size is specified and smaller than actual size, + the whole line will be read out from underlying file object by + subsequent readline calls. + """ + if self._buffered: + line = self._buffered_line + self._buffered = False + else: + line = self._file.readline() + if line and len(self._memorized_lines) < self._max_memorized_lines: + self._memorized_lines.append(line) + if size >= 0 and size < len(line): + self._buffered = True + self._buffered_line = line[size:] + return line[:size] + return line + + def get_memorized_lines(self): + """Get lines memorized so far.""" + return self._memorized_lines + + +# vi:sts=4 sw=4 et diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/msgutil.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/msgutil.py new file mode 100644 index 0000000000..dd6a6fc410 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/msgutil.py @@ -0,0 +1,218 @@ +# Copyright 2011, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +"""Message related utilities. + +Note: request.connection.write/read are used in this module, even though +mod_python document says that they should be used only in connection +handlers. Unfortunately, we have no other options. For example, +request.write/read are not suitable because they don't allow direct raw +bytes writing/reading. +""" + +from __future__ import absolute_import + +import threading + +import six.moves.queue + +# Export Exception symbols from msgutil for backward compatibility +from pywebsocket3._stream_exceptions import ( + ConnectionTerminatedException, + InvalidFrameException, + BadOperationException, + UnsupportedFrameException +) + + +# An API for handler to send/receive WebSocket messages. +def close_connection(request): + """Close connection. + + Args: + request: mod_python request. + """ + request.ws_stream.close_connection() + + +def send_message(request, payload_data, end=True, binary=False): + """Send a message (or part of a message). + + Args: + request: mod_python request. + payload_data: unicode text or str binary to send. + end: True to terminate a message. + False to send payload_data as part of a message that is to be + terminated by next or later send_message call with end=True. + binary: send payload_data as binary frame(s). + Raises: + BadOperationException: when server already terminated. + """ + request.ws_stream.send_message(payload_data, end, binary) + + +def receive_message(request): + """Receive a WebSocket frame and return its payload as a text in + unicode or a binary in str. + + Args: + request: mod_python request. + Raises: + InvalidFrameException: when client send invalid frame. + UnsupportedFrameException: when client send unsupported frame e.g. some + of reserved bit is set but no extension can + recognize it. + InvalidUTF8Exception: when client send a text frame containing any + invalid UTF-8 string. + ConnectionTerminatedException: when the connection is closed + unexpectedly. + BadOperationException: when client already terminated. + """ + return request.ws_stream.receive_message() + + +def send_ping(request, body): + request.ws_stream.send_ping(body) + + +class MessageReceiver(threading.Thread): + """This class receives messages from the client. + + This class provides three ways to receive messages: blocking, + non-blocking, and via callback. Callback has the highest precedence. + + Note: This class should not be used with the standalone server for wss + because pyOpenSSL used by the server raises a fatal error if the socket + is accessed from multiple threads. + """ + def __init__(self, request, onmessage=None): + """Construct an instance. + + Args: + request: mod_python request. + onmessage: a function to be called when a message is received. + May be None. If not None, the function is called on + another thread. In that case, MessageReceiver.receive + and MessageReceiver.receive_nowait are useless + because they will never return any messages. + """ + + threading.Thread.__init__(self) + self._request = request + self._queue = six.moves.queue.Queue() + self._onmessage = onmessage + self._stop_requested = False + self.setDaemon(True) + self.start() + + def run(self): + try: + while not self._stop_requested: + message = receive_message(self._request) + if self._onmessage: + self._onmessage(message) + else: + self._queue.put(message) + finally: + close_connection(self._request) + + def receive(self): + """ Receive a message from the channel, blocking. + + Returns: + message as a unicode string. + """ + return self._queue.get() + + def receive_nowait(self): + """ Receive a message from the channel, non-blocking. + + Returns: + message as a unicode string if available. None otherwise. + """ + try: + message = self._queue.get_nowait() + except six.moves.queue.Empty: + message = None + return message + + def stop(self): + """Request to stop this instance. + + The instance will be stopped after receiving the next message. + This method may not be very useful, but there is no clean way + in Python to forcefully stop a running thread. + """ + self._stop_requested = True + + +class MessageSender(threading.Thread): + """This class sends messages to the client. + + This class provides both synchronous and asynchronous ways to send + messages. + + Note: This class should not be used with the standalone server for wss + because pyOpenSSL used by the server raises a fatal error if the socket + is accessed from multiple threads. + """ + def __init__(self, request): + """Construct an instance. + + Args: + request: mod_python request. + """ + threading.Thread.__init__(self) + self._request = request + self._queue = six.moves.queue.Queue() + self.setDaemon(True) + self.start() + + def run(self): + while True: + message, condition = self._queue.get() + condition.acquire() + send_message(self._request, message) + condition.notify() + condition.release() + + def send(self, message): + """Send a message, blocking.""" + + condition = threading.Condition() + condition.acquire() + self._queue.put((message, condition)) + condition.wait() + + def send_nowait(self, message): + """Send a message, non-blocking.""" + + self._queue.put((message, threading.Condition())) + + +# vi:sts=4 sw=4 et diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/request_handler.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/request_handler.py new file mode 100644 index 0000000000..e02e2ccf28 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/request_handler.py @@ -0,0 +1,322 @@ +# Copyright 2020, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +"""Request Handler and Request/Connection classes for standalone server. +""" + +import os + +from six.moves import CGIHTTPServer +from six.moves import http_client + +from pywebsocket3 import ( + common, + dispatch, + handshake, + http_header_util, + memorizingfile, + util +) + +# 1024 is practically large enough to contain WebSocket handshake lines. +_MAX_MEMORIZED_LINES = 1024 + + +class _StandaloneConnection(object): + """Mimic mod_python mp_conn.""" + def __init__(self, request_handler): + """Construct an instance. + + Args: + request_handler: A WebSocketRequestHandler instance. + """ + + self._request_handler = request_handler + + def get_local_addr(self): + """Getter to mimic mp_conn.local_addr.""" + + return (self._request_handler.server.server_name, + self._request_handler.server.server_port) + + local_addr = property(get_local_addr) + + def get_remote_addr(self): + """Getter to mimic mp_conn.remote_addr. + + Setting the property in __init__ won't work because the request + handler is not initialized yet there.""" + + return self._request_handler.client_address + + remote_addr = property(get_remote_addr) + + def write(self, data): + """Mimic mp_conn.write().""" + + return self._request_handler.wfile.write(data) + + def read(self, length): + """Mimic mp_conn.read().""" + + return self._request_handler.rfile.read(length) + + def get_memorized_lines(self): + """Get memorized lines.""" + + return self._request_handler.rfile.get_memorized_lines() + + +class _StandaloneRequest(object): + """Mimic mod_python request.""" + def __init__(self, request_handler, use_tls): + """Construct an instance. + + Args: + request_handler: A WebSocketRequestHandler instance. + """ + + self._logger = util.get_class_logger(self) + + self._request_handler = request_handler + self.connection = _StandaloneConnection(request_handler) + self._use_tls = use_tls + self.headers_in = request_handler.headers + + def get_uri(self): + """Getter to mimic request.uri. + + This method returns the raw data at the Request-URI part of the + Request-Line, while the uri method on the request object of mod_python + returns the path portion after parsing the raw data. This behavior is + kept for compatibility. + """ + + return self._request_handler.path + + uri = property(get_uri) + + def get_unparsed_uri(self): + """Getter to mimic request.unparsed_uri.""" + + return self._request_handler.path + + unparsed_uri = property(get_unparsed_uri) + + def get_method(self): + """Getter to mimic request.method.""" + + return self._request_handler.command + + method = property(get_method) + + def get_protocol(self): + """Getter to mimic request.protocol.""" + + return self._request_handler.request_version + + protocol = property(get_protocol) + + def is_https(self): + """Mimic request.is_https().""" + + return self._use_tls + + +class WebSocketRequestHandler(CGIHTTPServer.CGIHTTPRequestHandler): + """CGIHTTPRequestHandler specialized for WebSocket.""" + + # Use httplib.HTTPMessage instead of mimetools.Message. + MessageClass = http_client.HTTPMessage + + def setup(self): + """Override SocketServer.StreamRequestHandler.setup to wrap rfile + with MemorizingFile. + + This method will be called by BaseRequestHandler's constructor + before calling BaseHTTPRequestHandler.handle. + BaseHTTPRequestHandler.handle will call + BaseHTTPRequestHandler.handle_one_request and it will call + WebSocketRequestHandler.parse_request. + """ + + # Call superclass's setup to prepare rfile, wfile, etc. See setup + # definition on the root class SocketServer.StreamRequestHandler to + # understand what this does. + CGIHTTPServer.CGIHTTPRequestHandler.setup(self) + + self.rfile = memorizingfile.MemorizingFile( + self.rfile, max_memorized_lines=_MAX_MEMORIZED_LINES) + + def __init__(self, request, client_address, server): + self._logger = util.get_class_logger(self) + + self._options = server.websocket_server_options + + # Overrides CGIHTTPServerRequestHandler.cgi_directories. + self.cgi_directories = self._options.cgi_directories + # Replace CGIHTTPRequestHandler.is_executable method. + if self._options.is_executable_method is not None: + self.is_executable = self._options.is_executable_method + + # This actually calls BaseRequestHandler.__init__. + CGIHTTPServer.CGIHTTPRequestHandler.__init__(self, request, + client_address, server) + + def parse_request(self): + """Override BaseHTTPServer.BaseHTTPRequestHandler.parse_request. + + Return True to continue processing for HTTP(S), False otherwise. + + See BaseHTTPRequestHandler.handle_one_request method which calls + this method to understand how the return value will be handled. + """ + + # We hook parse_request method, but also call the original + # CGIHTTPRequestHandler.parse_request since when we return False, + # CGIHTTPRequestHandler.handle_one_request continues processing and + # it needs variables set by CGIHTTPRequestHandler.parse_request. + # + # Variables set by this method will be also used by WebSocket request + # handling (self.path, self.command, self.requestline, etc. See also + # how _StandaloneRequest's members are implemented using these + # attributes). + if not CGIHTTPServer.CGIHTTPRequestHandler.parse_request(self): + return False + + if self._options.use_basic_auth: + auth = self.headers.get('Authorization') + if auth != self._options.basic_auth_credential: + self.send_response(401) + self.send_header('WWW-Authenticate', + 'Basic realm="Pywebsocket"') + self.end_headers() + self._logger.info('Request basic authentication') + return False + + whole_path = self.path + host, port, resource = http_header_util.parse_uri(self.path) + if resource is None: + self._logger.info('Invalid URI: %r', self.path) + self._logger.info('Fallback to CGIHTTPRequestHandler') + return True + server_options = self.server.websocket_server_options + if host is not None: + validation_host = server_options.validation_host + if validation_host is not None and host != validation_host: + self._logger.info('Invalid host: %r (expected: %r)', host, + validation_host) + self._logger.info('Fallback to CGIHTTPRequestHandler') + return True + if port is not None: + validation_port = server_options.validation_port + if validation_port is not None and port != validation_port: + self._logger.info('Invalid port: %r (expected: %r)', port, + validation_port) + self._logger.info('Fallback to CGIHTTPRequestHandler') + return True + self.path = resource + + request = _StandaloneRequest(self, self._options.use_tls) + + try: + # Fallback to default http handler for request paths for which + # we don't have request handlers. + if not self._options.dispatcher.get_handler_suite(self.path): + self._logger.info('No handler for resource: %r', whole_path) + self._logger.info('Fallback to CGIHTTPRequestHandler') + return True + except dispatch.DispatchException as e: + self._logger.info('Dispatch failed for error: %s', e) + self.send_error(e.status) + return False + + # If any Exceptions without except clause setup (including + # DispatchException) is raised below this point, it will be caught + # and logged by WebSocketServer. + + try: + try: + handshake.do_handshake(request, self._options.dispatcher) + except handshake.VersionException as e: + self._logger.info('Handshake failed for version error: %s', e) + self.send_response(common.HTTP_STATUS_BAD_REQUEST) + self.send_header(common.SEC_WEBSOCKET_VERSION_HEADER, + e.supported_versions) + self.end_headers() + return False + except handshake.HandshakeException as e: + # Handshake for ws(s) failed. + self._logger.info('Handshake failed for error: %s', e) + self.send_error(e.status) + return False + + request._dispatcher = self._options.dispatcher + self._options.dispatcher.transfer_data(request) + except handshake.AbortedByUserException as e: + self._logger.info('Aborted: %s', e) + return False + + def log_request(self, code='-', size='-'): + """Override BaseHTTPServer.log_request.""" + + self._logger.info('"%s" %s %s', self.requestline, str(code), str(size)) + + def log_error(self, *args): + """Override BaseHTTPServer.log_error.""" + + # Despite the name, this method is for warnings than for errors. + # For example, HTTP status code is logged by this method. + self._logger.warning('%s - %s', self.address_string(), + args[0] % args[1:]) + + def is_cgi(self): + """Test whether self.path corresponds to a CGI script. + + Add extra check that self.path doesn't contains .. + Also check if the file is a executable file or not. + If the file is not executable, it is handled as static file or dir + rather than a CGI script. + """ + + if CGIHTTPServer.CGIHTTPRequestHandler.is_cgi(self): + if '..' in self.path: + return False + # strip query parameter from request path + resource_name = self.path.split('?', 2)[0] + # convert resource_name into real path name in filesystem. + scriptfile = self.translate_path(resource_name) + if not os.path.isfile(scriptfile): + return False + if not self.is_executable(scriptfile): + return False + return True + return False + + +# vi:sts=4 sw=4 et diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/server_util.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/server_util.py new file mode 100644 index 0000000000..3bf07f885b --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/server_util.py @@ -0,0 +1,86 @@ +# Copyright 2020, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +"""Server related utilities.""" + +import logging +import logging.handlers +import threading +import time + +from pywebsocket3 import common, util + + +def _get_logger_from_class(c): + return logging.getLogger('%s.%s' % (c.__module__, c.__name__)) + + +def configure_logging(options): + logging.addLevelName(common.LOGLEVEL_FINE, 'FINE') + + logger = logging.getLogger() + logger.setLevel(logging.getLevelName(options.log_level.upper())) + if options.log_file: + handler = logging.handlers.RotatingFileHandler(options.log_file, 'a', + options.log_max, + options.log_count) + else: + handler = logging.StreamHandler() + formatter = logging.Formatter( + '[%(asctime)s] [%(levelname)s] %(name)s: %(message)s') + handler.setFormatter(formatter) + logger.addHandler(handler) + + deflate_log_level_name = logging.getLevelName( + options.deflate_log_level.upper()) + _get_logger_from_class(util._Deflater).setLevel(deflate_log_level_name) + _get_logger_from_class(util._Inflater).setLevel(deflate_log_level_name) + + +class ThreadMonitor(threading.Thread): + daemon = True + + def __init__(self, interval_in_sec): + threading.Thread.__init__(self, name='ThreadMonitor') + + self._logger = util.get_class_logger(self) + + self._interval_in_sec = interval_in_sec + + def run(self): + while True: + thread_name_list = [] + for thread in threading.enumerate(): + thread_name_list.append(thread.name) + self._logger.info("%d active threads: %s", + threading.active_count(), + ', '.join(thread_name_list)) + time.sleep(self._interval_in_sec) + + +# vi:sts=4 sw=4 et diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/standalone.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/standalone.py new file mode 100755 index 0000000000..0c324c4221 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/standalone.py @@ -0,0 +1,492 @@ +#!/usr/bin/env python +# +# Copyright 2012, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +"""Standalone WebSocket server. + +Use this file to launch pywebsocket as a standalone server. + + +BASIC USAGE +=========== + +Go to the src directory and run + + $ python pywebsocket3/standalone.py [-p ] + [-w ] + [-d ] + + is the port number to use for ws:// connection. + + is the path to the root directory of HTML files. + + is the path to the root directory of WebSocket handlers. +If not specified, will be used. See __init__.py (or +run $ pydoc pywebsocket3) for how to write WebSocket handlers. + +For more detail and other options, run + + $ python pywebsocket3/standalone.py --help + +or see _build_option_parser method below. + +For trouble shooting, adding "--log_level debug" might help you. + + +TRY DEMO +======== + +Go to the src directory and run standalone.py with -d option to set the +document root to the directory containing example HTMLs and handlers like this: + + $ cd src + $ PYTHONPATH=. python pywebsocket3/standalone.py -d example + +to launch pywebsocket with the sample handler and html on port 80. Open +http://localhost/console.html, click the connect button, type something into +the text box next to the send button and click the send button. If everything +is working, you'll see the message you typed echoed by the server. + + +USING TLS +========= + +To run the standalone server with TLS support, run it with -t, -k, and -c +options. When TLS is enabled, the standalone server accepts only TLS connection. + +Note that when ssl module is used and the key/cert location is incorrect, +TLS connection silently fails while pyOpenSSL fails on startup. + +Example: + + $ PYTHONPATH=. python pywebsocket3/standalone.py \ + -d example \ + -p 10443 \ + -t \ + -c ../test/cert/cert.pem \ + -k ../test/cert/key.pem \ + +Note that when passing a relative path to -c and -k option, it will be resolved +using the document root directory as the base. + + +USING CLIENT AUTHENTICATION +=========================== + +To run the standalone server with TLS client authentication support, run it with +--tls-client-auth and --tls-client-ca options in addition to ones required for +TLS support. + +Example: + + $ PYTHONPATH=. python pywebsocket3/standalone.py -d example -p 10443 -t \ + -c ../test/cert/cert.pem -k ../test/cert/key.pem \ + --tls-client-auth \ + --tls-client-ca=../test/cert/cacert.pem + +Note that when passing a relative path to --tls-client-ca option, it will be +resolved using the document root directory as the base. + + +CONFIGURATION FILE +================== + +You can also write a configuration file and use it by specifying the path to +the configuration file by --config option. Please write a configuration file +following the documentation of the Python ConfigParser library. Name of each +entry must be the long version argument name. E.g. to set log level to debug, +add the following line: + +log_level=debug + +For options which doesn't take value, please add some fake value. E.g. for +--tls option, add the following line: + +tls=True + +Note that tls will be enabled even if you write tls=False as the value part is +fake. + +When both a command line argument and a configuration file entry are set for +the same configuration item, the command line value will override one in the +configuration file. + + +THREADING +========= + +This server is derived from SocketServer.ThreadingMixIn. Hence a thread is +used for each request. + + +SECURITY WARNING +================ + +This uses CGIHTTPServer and CGIHTTPServer is not secure. +It may execute arbitrary Python code or external programs. It should not be +used outside a firewall. +""" + +from __future__ import absolute_import + +import argparse +import base64 +import logging +import os +import sys +import traceback + +import six +from six.moves import configparser + +from pywebsocket3 import common, server_util, util +from pywebsocket3.websocket_server import WebSocketServer + + +_DEFAULT_LOG_MAX_BYTES = 1024 * 256 +_DEFAULT_LOG_BACKUP_COUNT = 5 + +_DEFAULT_REQUEST_QUEUE_SIZE = 128 + + +def _build_option_parser(): + parser = argparse.ArgumentParser() + + parser.add_argument( + '--config', + dest='config_file', + type=six.text_type, + default=None, + help=('Path to configuration file. See the file comment ' + 'at the top of this file for the configuration ' + 'file format')) + parser.add_argument('-H', + '--server-host', + '--server_host', + dest='server_host', + default='', + help='server hostname to listen to') + parser.add_argument('-V', + '--validation-host', + '--validation_host', + dest='validation_host', + default=None, + help='server hostname to validate in absolute path.') + parser.add_argument('-p', + '--port', + dest='port', + type=int, + default=common.DEFAULT_WEB_SOCKET_PORT, + help='port to listen to') + parser.add_argument('-P', + '--validation-port', + '--validation_port', + dest='validation_port', + type=int, + default=None, + help='server port to validate in absolute path.') + parser.add_argument( + '-w', + '--websock-handlers', + '--websock_handlers', + dest='websock_handlers', + default='.', + help=('The root directory of WebSocket handler files. ' + 'If the path is relative, --document-root is used ' + 'as the base.')) + parser.add_argument('-m', + '--websock-handlers-map-file', + '--websock_handlers_map_file', + dest='websock_handlers_map_file', + default=None, + help=('WebSocket handlers map file. ' + 'Each line consists of alias_resource_path and ' + 'existing_resource_path, separated by spaces.')) + parser.add_argument('-s', + '--scan-dir', + '--scan_dir', + dest='scan_dir', + default=None, + help=('Must be a directory under --websock-handlers. ' + 'Only handlers under this directory are scanned ' + 'and registered to the server. ' + 'Useful for saving scan time when the handler ' + 'root directory contains lots of files that are ' + 'not handler file or are handler files but you ' + 'don\'t want them to be registered. ')) + parser.add_argument( + '--allow-handlers-outside-root-dir', + '--allow_handlers_outside_root_dir', + dest='allow_handlers_outside_root_dir', + action='store_true', + default=False, + help=('Scans WebSocket handlers even if their canonical ' + 'path is not under --websock-handlers.')) + parser.add_argument('-d', + '--document-root', + '--document_root', + dest='document_root', + default='.', + help='Document root directory.') + parser.add_argument('-x', + '--cgi-paths', + '--cgi_paths', + dest='cgi_paths', + default=None, + help=('CGI paths relative to document_root.' + 'Comma-separated. (e.g -x /cgi,/htbin) ' + 'Files under document_root/cgi_path are handled ' + 'as CGI programs. Must be executable.')) + parser.add_argument('-t', + '--tls', + dest='use_tls', + action='store_true', + default=False, + help='use TLS (wss://)') + parser.add_argument('-k', + '--private-key', + '--private_key', + dest='private_key', + default='', + help='TLS private key file.') + parser.add_argument('-c', + '--certificate', + dest='certificate', + default='', + help='TLS certificate file.') + parser.add_argument('--tls-client-auth', + dest='tls_client_auth', + action='store_true', + default=False, + help='Requests TLS client auth on every connection.') + parser.add_argument('--tls-client-cert-optional', + dest='tls_client_cert_optional', + action='store_true', + default=False, + help=('Makes client certificate optional even though ' + 'TLS client auth is enabled.')) + parser.add_argument('--tls-client-ca', + dest='tls_client_ca', + default='', + help=('Specifies a pem file which contains a set of ' + 'concatenated CA certificates which are used to ' + 'validate certificates passed from clients')) + parser.add_argument('--basic-auth', + dest='use_basic_auth', + action='store_true', + default=False, + help='Requires Basic authentication.') + parser.add_argument( + '--basic-auth-credential', + dest='basic_auth_credential', + default='test:test', + help='Specifies the credential of basic authentication ' + 'by username:password pair (e.g. test:test).') + parser.add_argument('-l', + '--log-file', + '--log_file', + dest='log_file', + default='', + help='Log file.') + # Custom log level: + # - FINE: Prints status of each frame processing step + parser.add_argument('--log-level', + '--log_level', + type=six.text_type, + dest='log_level', + default='warn', + choices=[ + 'fine', 'debug', 'info', 'warning', 'warn', + 'error', 'critical' + ], + help='Log level.') + parser.add_argument( + '--deflate-log-level', + '--deflate_log_level', + type=six.text_type, + dest='deflate_log_level', + default='warn', + choices=['debug', 'info', 'warning', 'warn', 'error', 'critical'], + help='Log level for _Deflater and _Inflater.') + parser.add_argument('--thread-monitor-interval-in-sec', + '--thread_monitor_interval_in_sec', + dest='thread_monitor_interval_in_sec', + type=int, + default=-1, + help=('If positive integer is specified, run a thread ' + 'monitor to show the status of server threads ' + 'periodically in the specified inteval in ' + 'second. If non-positive integer is specified, ' + 'disable the thread monitor.')) + parser.add_argument('--log-max', + '--log_max', + dest='log_max', + type=int, + default=_DEFAULT_LOG_MAX_BYTES, + help='Log maximum bytes') + parser.add_argument('--log-count', + '--log_count', + dest='log_count', + type=int, + default=_DEFAULT_LOG_BACKUP_COUNT, + help='Log backup count') + parser.add_argument('-q', + '--queue', + dest='request_queue_size', + type=int, + default=_DEFAULT_REQUEST_QUEUE_SIZE, + help='request queue size') + parser.add_argument( + '--handler-encoding', + '--handler_encoding', + dest='handler_encoding', + type=six.text_type, + default=None, + help=('Text encoding used for loading handlers. ' + 'By default, the encoding from the locale is used when ' + 'reading handler files, but this option can override it. ' + 'Any encoding supported by the codecs module may be used.')) + + return parser + + +def _parse_args_and_config(args): + parser = _build_option_parser() + + # First, parse options without configuration file. + temporary_options, temporary_args = parser.parse_known_args(args=args) + if temporary_args: + logging.critical('Unrecognized positional arguments: %r', + temporary_args) + sys.exit(1) + + if temporary_options.config_file: + try: + config_fp = open(temporary_options.config_file, 'r') + except IOError as e: + logging.critical('Failed to open configuration file %r: %r', + temporary_options.config_file, e) + sys.exit(1) + + config_parser = configparser.SafeConfigParser() + config_parser.readfp(config_fp) + config_fp.close() + + args_from_config = [] + for name, value in config_parser.items('pywebsocket'): + args_from_config.append('--' + name) + args_from_config.append(value) + if args is None: + args = args_from_config + else: + args = args_from_config + args + return parser.parse_known_args(args=args) + else: + return temporary_options, temporary_args + + +def _main(args=None): + """You can call this function from your own program, but please note that + this function has some side-effects that might affect your program. For + example, it changes the current directory. + """ + + options, args = _parse_args_and_config(args=args) + + os.chdir(options.document_root) + + server_util.configure_logging(options) + + # TODO(tyoshino): Clean up initialization of CGI related values. Move some + # of code here to WebSocketRequestHandler class if it's better. + options.cgi_directories = [] + options.is_executable_method = None + if options.cgi_paths: + options.cgi_directories = options.cgi_paths.split(',') + if sys.platform in ('cygwin', 'win32'): + cygwin_path = None + # For Win32 Python, it is expected that CYGWIN_PATH + # is set to a directory of cygwin binaries. + # For example, websocket_server.py in Chromium sets CYGWIN_PATH to + # full path of third_party/cygwin/bin. + if 'CYGWIN_PATH' in os.environ: + cygwin_path = os.environ['CYGWIN_PATH'] + + def __check_script(scriptpath): + return util.get_script_interp(scriptpath, cygwin_path) + + options.is_executable_method = __check_script + + if options.use_tls: + logging.debug('Using ssl module') + + if not options.private_key or not options.certificate: + logging.critical( + 'To use TLS, specify private_key and certificate.') + sys.exit(1) + + if (options.tls_client_cert_optional and not options.tls_client_auth): + logging.critical('Client authentication must be enabled to ' + 'specify tls_client_cert_optional') + sys.exit(1) + else: + if options.tls_client_auth: + logging.critical('TLS must be enabled for client authentication.') + sys.exit(1) + + if options.tls_client_cert_optional: + logging.critical('TLS must be enabled for client authentication.') + sys.exit(1) + + if not options.scan_dir: + options.scan_dir = options.websock_handlers + + if options.use_basic_auth: + options.basic_auth_credential = 'Basic ' + base64.b64encode( + options.basic_auth_credential.encode('UTF-8')).decode() + + try: + if options.thread_monitor_interval_in_sec > 0: + # Run a thread monitor to show the status of server threads for + # debugging. + server_util.ThreadMonitor( + options.thread_monitor_interval_in_sec).start() + + server = WebSocketServer(options) + server.serve_forever() + except Exception as e: + logging.critical('pywebsocket3: %s' % e) + logging.critical('pywebsocket3: %s' % traceback.format_exc()) + sys.exit(1) + + +if __name__ == '__main__': + _main(sys.argv[1:]) + +# vi:sts=4 sw=4 et diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/stream.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/stream.py new file mode 100644 index 0000000000..dd41850dc4 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/stream.py @@ -0,0 +1,951 @@ +# Copyright 2011, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +"""This file provides classes and helper functions for parsing/building frames +of the WebSocket protocol (RFC 6455). + +Specification: +http://tools.ietf.org/html/rfc6455 +""" + +import logging +import os +import struct +import time +from collections import deque + +import six + +from pywebsocket3 import common, util +from pywebsocket3._stream_exceptions import ( + BadOperationException, + ConnectionTerminatedException, + InvalidFrameException, + InvalidUTF8Exception, + UnsupportedFrameException +) + +_NOOP_MASKER = util.NoopMasker() + + +class Frame(object): + def __init__(self, + fin=1, + rsv1=0, + rsv2=0, + rsv3=0, + opcode=None, + payload=b''): + self.fin = fin + self.rsv1 = rsv1 + self.rsv2 = rsv2 + self.rsv3 = rsv3 + self.opcode = opcode + self.payload = payload + + +# Helper functions made public to be used for writing unittests for WebSocket +# clients. + + +def create_length_header(length, mask): + """Creates a length header. + + Args: + length: Frame length. Must be less than 2^63. + mask: Mask bit. Must be boolean. + + Raises: + ValueError: when bad data is given. + """ + + if mask: + mask_bit = 1 << 7 + else: + mask_bit = 0 + + if length < 0: + raise ValueError('length must be non negative integer') + elif length <= 125: + return util.pack_byte(mask_bit | length) + elif length < (1 << 16): + return util.pack_byte(mask_bit | 126) + struct.pack('!H', length) + elif length < (1 << 63): + return util.pack_byte(mask_bit | 127) + struct.pack('!Q', length) + else: + raise ValueError('Payload is too big for one frame') + + +def create_header(opcode, payload_length, fin, rsv1, rsv2, rsv3, mask): + """Creates a frame header. + + Raises: + Exception: when bad data is given. + """ + + if opcode < 0 or 0xf < opcode: + raise ValueError('Opcode out of range') + + if payload_length < 0 or (1 << 63) <= payload_length: + raise ValueError('payload_length out of range') + + if (fin | rsv1 | rsv2 | rsv3) & ~1: + raise ValueError('FIN bit and Reserved bit parameter must be 0 or 1') + + header = b'' + + first_byte = ((fin << 7) + | (rsv1 << 6) | (rsv2 << 5) | (rsv3 << 4) + | opcode) + header += util.pack_byte(first_byte) + header += create_length_header(payload_length, mask) + + return header + + +def _build_frame(header, body, mask): + if not mask: + return header + body + + masking_nonce = os.urandom(4) + masker = util.RepeatedXorMasker(masking_nonce) + + return header + masking_nonce + masker.mask(body) + + +def _filter_and_format_frame_object(frame, mask, frame_filters): + for frame_filter in frame_filters: + frame_filter.filter(frame) + + header = create_header(frame.opcode, len(frame.payload), frame.fin, + frame.rsv1, frame.rsv2, frame.rsv3, mask) + return _build_frame(header, frame.payload, mask) + + +def create_binary_frame(message, + opcode=common.OPCODE_BINARY, + fin=1, + mask=False, + frame_filters=[]): + """Creates a simple binary frame with no extension, reserved bit.""" + + frame = Frame(fin=fin, opcode=opcode, payload=message) + return _filter_and_format_frame_object(frame, mask, frame_filters) + + +def create_text_frame(message, + opcode=common.OPCODE_TEXT, + fin=1, + mask=False, + frame_filters=[]): + """Creates a simple text frame with no extension, reserved bit.""" + + encoded_message = message.encode('utf-8') + return create_binary_frame(encoded_message, opcode, fin, mask, + frame_filters) + + +def parse_frame(receive_bytes, + logger=None, + ws_version=common.VERSION_HYBI_LATEST, + unmask_receive=True): + """Parses a frame. Returns a tuple containing each header field and + payload. + + Args: + receive_bytes: a function that reads frame data from a stream or + something similar. The function takes length of the bytes to be + read. The function must raise ConnectionTerminatedException if + there is not enough data to be read. + logger: a logging object. + ws_version: the version of WebSocket protocol. + unmask_receive: unmask received frames. When received unmasked + frame, raises InvalidFrameException. + + Raises: + ConnectionTerminatedException: when receive_bytes raises it. + InvalidFrameException: when the frame contains invalid data. + """ + + if not logger: + logger = logging.getLogger() + + logger.log(common.LOGLEVEL_FINE, 'Receive the first 2 octets of a frame') + + first_byte = ord(receive_bytes(1)) + fin = (first_byte >> 7) & 1 + rsv1 = (first_byte >> 6) & 1 + rsv2 = (first_byte >> 5) & 1 + rsv3 = (first_byte >> 4) & 1 + opcode = first_byte & 0xf + + second_byte = ord(receive_bytes(1)) + mask = (second_byte >> 7) & 1 + payload_length = second_byte & 0x7f + + logger.log( + common.LOGLEVEL_FINE, 'FIN=%s, RSV1=%s, RSV2=%s, RSV3=%s, opcode=%s, ' + 'Mask=%s, Payload_length=%s', fin, rsv1, rsv2, rsv3, opcode, mask, + payload_length) + + if (mask == 1) != unmask_receive: + raise InvalidFrameException( + 'Mask bit on the received frame did\'nt match masking ' + 'configuration for received frames') + + # The HyBi and later specs disallow putting a value in 0x0-0xFFFF + # into the 8-octet extended payload length field (or 0x0-0xFD in + # 2-octet field). + valid_length_encoding = True + length_encoding_bytes = 1 + if payload_length == 127: + logger.log(common.LOGLEVEL_FINE, + 'Receive 8-octet extended payload length') + + extended_payload_length = receive_bytes(8) + payload_length = struct.unpack('!Q', extended_payload_length)[0] + if payload_length > 0x7FFFFFFFFFFFFFFF: + raise InvalidFrameException('Extended payload length >= 2^63') + if ws_version >= 13 and payload_length < 0x10000: + valid_length_encoding = False + length_encoding_bytes = 8 + + logger.log(common.LOGLEVEL_FINE, 'Decoded_payload_length=%s', + payload_length) + elif payload_length == 126: + logger.log(common.LOGLEVEL_FINE, + 'Receive 2-octet extended payload length') + + extended_payload_length = receive_bytes(2) + payload_length = struct.unpack('!H', extended_payload_length)[0] + if ws_version >= 13 and payload_length < 126: + valid_length_encoding = False + length_encoding_bytes = 2 + + logger.log(common.LOGLEVEL_FINE, 'Decoded_payload_length=%s', + payload_length) + + if not valid_length_encoding: + logger.warning( + 'Payload length is not encoded using the minimal number of ' + 'bytes (%d is encoded using %d bytes)', payload_length, + length_encoding_bytes) + + if mask == 1: + logger.log(common.LOGLEVEL_FINE, 'Receive mask') + + masking_nonce = receive_bytes(4) + masker = util.RepeatedXorMasker(masking_nonce) + + logger.log(common.LOGLEVEL_FINE, 'Mask=%r', masking_nonce) + else: + masker = _NOOP_MASKER + + logger.log(common.LOGLEVEL_FINE, 'Receive payload data') + if logger.isEnabledFor(common.LOGLEVEL_FINE): + receive_start = time.time() + + raw_payload_bytes = receive_bytes(payload_length) + + if logger.isEnabledFor(common.LOGLEVEL_FINE): + logger.log( + common.LOGLEVEL_FINE, 'Done receiving payload data at %s MB/s', + payload_length / (time.time() - receive_start) / 1000 / 1000) + logger.log(common.LOGLEVEL_FINE, 'Unmask payload data') + + if logger.isEnabledFor(common.LOGLEVEL_FINE): + unmask_start = time.time() + + unmasked_bytes = masker.mask(raw_payload_bytes) + + if logger.isEnabledFor(common.LOGLEVEL_FINE): + logger.log(common.LOGLEVEL_FINE, + 'Done unmasking payload data at %s MB/s', + payload_length / (time.time() - unmask_start) / 1000 / 1000) + + return opcode, unmasked_bytes, fin, rsv1, rsv2, rsv3 + + +class FragmentedFrameBuilder(object): + """A stateful class to send a message as fragments.""" + def __init__(self, mask, frame_filters=[], encode_utf8=True): + """Constructs an instance.""" + + self._mask = mask + self._frame_filters = frame_filters + # This is for skipping UTF-8 encoding when building text type frames + # from compressed data. + self._encode_utf8 = encode_utf8 + + self._started = False + + # Hold opcode of the first frame in messages to verify types of other + # frames in the message are all the same. + self._opcode = common.OPCODE_TEXT + + def build(self, payload_data, end, binary): + if binary: + frame_type = common.OPCODE_BINARY + else: + frame_type = common.OPCODE_TEXT + if self._started: + if self._opcode != frame_type: + raise ValueError('Message types are different in frames for ' + 'the same message') + opcode = common.OPCODE_CONTINUATION + else: + opcode = frame_type + self._opcode = frame_type + + if end: + self._started = False + fin = 1 + else: + self._started = True + fin = 0 + + if binary or not self._encode_utf8: + return create_binary_frame(payload_data, opcode, fin, self._mask, + self._frame_filters) + else: + return create_text_frame(payload_data, opcode, fin, self._mask, + self._frame_filters) + + +def _create_control_frame(opcode, body, mask, frame_filters): + frame = Frame(opcode=opcode, payload=body) + + for frame_filter in frame_filters: + frame_filter.filter(frame) + + if len(frame.payload) > 125: + raise BadOperationException( + 'Payload data size of control frames must be 125 bytes or less') + + header = create_header(frame.opcode, len(frame.payload), frame.fin, + frame.rsv1, frame.rsv2, frame.rsv3, mask) + return _build_frame(header, frame.payload, mask) + + +def create_ping_frame(body, mask=False, frame_filters=[]): + return _create_control_frame(common.OPCODE_PING, body, mask, frame_filters) + + +def create_pong_frame(body, mask=False, frame_filters=[]): + return _create_control_frame(common.OPCODE_PONG, body, mask, frame_filters) + + +def create_close_frame(body, mask=False, frame_filters=[]): + return _create_control_frame(common.OPCODE_CLOSE, body, mask, + frame_filters) + + +def create_closing_handshake_body(code, reason): + body = b'' + if code is not None: + if (code > common.STATUS_USER_PRIVATE_MAX + or code < common.STATUS_NORMAL_CLOSURE): + raise BadOperationException('Status code is out of range') + if (code == common.STATUS_NO_STATUS_RECEIVED + or code == common.STATUS_ABNORMAL_CLOSURE + or code == common.STATUS_TLS_HANDSHAKE): + raise BadOperationException('Status code is reserved pseudo ' + 'code') + encoded_reason = reason.encode('utf-8') + body = struct.pack('!H', code) + encoded_reason + return body + + +class StreamOptions(object): + """Holds option values to configure Stream objects.""" + def __init__(self): + """Constructs StreamOptions.""" + + # Filters applied to frames. + self.outgoing_frame_filters = [] + self.incoming_frame_filters = [] + + # Filters applied to messages. Control frames are not affected by them. + self.outgoing_message_filters = [] + self.incoming_message_filters = [] + + self.encode_text_message_to_utf8 = True + self.mask_send = False + self.unmask_receive = True + + +class Stream(object): + """A class for parsing/building frames of the WebSocket protocol + (RFC 6455). + """ + def __init__(self, request, options): + """Constructs an instance. + + Args: + request: mod_python request. + """ + + self._logger = util.get_class_logger(self) + + self._options = options + self._request = request + + self._request.client_terminated = False + self._request.server_terminated = False + + # Holds body of received fragments. + self._received_fragments = [] + # Holds the opcode of the first fragment. + self._original_opcode = None + + self._writer = FragmentedFrameBuilder( + self._options.mask_send, self._options.outgoing_frame_filters, + self._options.encode_text_message_to_utf8) + + self._ping_queue = deque() + + def _read(self, length): + """Reads length bytes from connection. In case we catch any exception, + prepends remote address to the exception message and raise again. + + Raises: + ConnectionTerminatedException: when read returns empty string. + """ + + try: + read_bytes = self._request.connection.read(length) + if not read_bytes: + raise ConnectionTerminatedException( + 'Receiving %d byte failed. Peer (%r) closed connection' % + (length, (self._request.connection.remote_addr, ))) + return read_bytes + except IOError as e: + # Also catch an IOError because mod_python throws it. + raise ConnectionTerminatedException( + 'Receiving %d byte failed. IOError (%s) occurred' % + (length, e)) + + def _write(self, bytes_to_write): + """Writes given bytes to connection. In case we catch any exception, + prepends remote address to the exception message and raise again. + """ + + try: + self._request.connection.write(bytes_to_write) + except Exception as e: + util.prepend_message_to_exception( + 'Failed to send message to %r: ' % + (self._request.connection.remote_addr, ), e) + raise + + def receive_bytes(self, length): + """Receives multiple bytes. Retries read when we couldn't receive the + specified amount. This method returns byte strings. + + Raises: + ConnectionTerminatedException: when read returns empty string. + """ + + read_bytes = [] + while length > 0: + new_read_bytes = self._read(length) + read_bytes.append(new_read_bytes) + length -= len(new_read_bytes) + return b''.join(read_bytes) + + def _read_until(self, delim_char): + """Reads bytes until we encounter delim_char. The result will not + contain delim_char. + + Raises: + ConnectionTerminatedException: when read returns empty string. + """ + + read_bytes = [] + while True: + ch = self._read(1) + if ch == delim_char: + break + read_bytes.append(ch) + return b''.join(read_bytes) + + def _receive_frame(self): + """Receives a frame and return data in the frame as a tuple containing + each header field and payload separately. + + Raises: + ConnectionTerminatedException: when read returns empty + string. + InvalidFrameException: when the frame contains invalid data. + """ + def _receive_bytes(length): + return self.receive_bytes(length) + + return parse_frame(receive_bytes=_receive_bytes, + logger=self._logger, + ws_version=self._request.ws_version, + unmask_receive=self._options.unmask_receive) + + def _receive_frame_as_frame_object(self): + opcode, unmasked_bytes, fin, rsv1, rsv2, rsv3 = self._receive_frame() + + return Frame(fin=fin, + rsv1=rsv1, + rsv2=rsv2, + rsv3=rsv3, + opcode=opcode, + payload=unmasked_bytes) + + def receive_filtered_frame(self): + """Receives a frame and applies frame filters and message filters. + The frame to be received must satisfy following conditions: + - The frame is not fragmented. + - The opcode of the frame is TEXT or BINARY. + + DO NOT USE this method except for testing purpose. + """ + + frame = self._receive_frame_as_frame_object() + if not frame.fin: + raise InvalidFrameException( + 'Segmented frames must not be received via ' + 'receive_filtered_frame()') + if (frame.opcode != common.OPCODE_TEXT + and frame.opcode != common.OPCODE_BINARY): + raise InvalidFrameException( + 'Control frames must not be received via ' + 'receive_filtered_frame()') + + for frame_filter in self._options.incoming_frame_filters: + frame_filter.filter(frame) + for message_filter in self._options.incoming_message_filters: + frame.payload = message_filter.filter(frame.payload) + return frame + + def send_message(self, message, end=True, binary=False): + """Send message. + + Args: + message: text in unicode or binary in str to send. + binary: send message as binary frame. + + Raises: + BadOperationException: when called on a server-terminated + connection or called with inconsistent message type or + binary parameter. + """ + + if self._request.server_terminated: + raise BadOperationException( + 'Requested send_message after sending out a closing handshake') + + if binary and isinstance(message, six.text_type): + raise BadOperationException( + 'Message for binary frame must not be instance of Unicode') + + for message_filter in self._options.outgoing_message_filters: + message = message_filter.filter(message, end, binary) + + try: + # Set this to any positive integer to limit maximum size of data in + # payload data of each frame. + MAX_PAYLOAD_DATA_SIZE = -1 + + if MAX_PAYLOAD_DATA_SIZE <= 0: + self._write(self._writer.build(message, end, binary)) + return + + bytes_written = 0 + while True: + end_for_this_frame = end + bytes_to_write = len(message) - bytes_written + if (MAX_PAYLOAD_DATA_SIZE > 0 + and bytes_to_write > MAX_PAYLOAD_DATA_SIZE): + end_for_this_frame = False + bytes_to_write = MAX_PAYLOAD_DATA_SIZE + + frame = self._writer.build( + message[bytes_written:bytes_written + bytes_to_write], + end_for_this_frame, binary) + self._write(frame) + + bytes_written += bytes_to_write + + # This if must be placed here (the end of while block) so that + # at least one frame is sent. + if len(message) <= bytes_written: + break + except ValueError as e: + raise BadOperationException(e) + + def _get_message_from_frame(self, frame): + """Gets a message from frame. If the message is composed of fragmented + frames and the frame is not the last fragmented frame, this method + returns None. The whole message will be returned when the last + fragmented frame is passed to this method. + + Raises: + InvalidFrameException: when the frame doesn't match defragmentation + context, or the frame contains invalid data. + """ + + if frame.opcode == common.OPCODE_CONTINUATION: + if not self._received_fragments: + if frame.fin: + raise InvalidFrameException( + 'Received a termination frame but fragmentation ' + 'not started') + else: + raise InvalidFrameException( + 'Received an intermediate frame but ' + 'fragmentation not started') + + if frame.fin: + # End of fragmentation frame + self._received_fragments.append(frame.payload) + message = b''.join(self._received_fragments) + self._received_fragments = [] + return message + else: + # Intermediate frame + self._received_fragments.append(frame.payload) + return None + else: + if self._received_fragments: + if frame.fin: + raise InvalidFrameException( + 'Received an unfragmented frame without ' + 'terminating existing fragmentation') + else: + raise InvalidFrameException( + 'New fragmentation started without terminating ' + 'existing fragmentation') + + if frame.fin: + # Unfragmented frame + + self._original_opcode = frame.opcode + return frame.payload + else: + # Start of fragmentation frame + + if common.is_control_opcode(frame.opcode): + raise InvalidFrameException( + 'Control frames must not be fragmented') + + self._original_opcode = frame.opcode + self._received_fragments.append(frame.payload) + return None + + def _process_close_message(self, message): + """Processes close message. + + Args: + message: close message. + + Raises: + InvalidFrameException: when the message is invalid. + """ + + self._request.client_terminated = True + + # Status code is optional. We can have status reason only if we + # have status code. Status reason can be empty string. So, + # allowed cases are + # - no application data: no code no reason + # - 2 octet of application data: has code but no reason + # - 3 or more octet of application data: both code and reason + if len(message) == 0: + self._logger.debug('Received close frame (empty body)') + self._request.ws_close_code = common.STATUS_NO_STATUS_RECEIVED + elif len(message) == 1: + raise InvalidFrameException( + 'If a close frame has status code, the length of ' + 'status code must be 2 octet') + elif len(message) >= 2: + self._request.ws_close_code = struct.unpack('!H', message[0:2])[0] + self._request.ws_close_reason = message[2:].decode( + 'utf-8', 'replace') + self._logger.debug('Received close frame (code=%d, reason=%r)', + self._request.ws_close_code, + self._request.ws_close_reason) + + # As we've received a close frame, no more data is coming over the + # socket. We can now safely close the socket without worrying about + # RST sending. + + if self._request.server_terminated: + self._logger.debug( + 'Received ack for server-initiated closing handshake') + return + + self._logger.debug('Received client-initiated closing handshake') + + code = common.STATUS_NORMAL_CLOSURE + reason = '' + if hasattr(self._request, '_dispatcher'): + dispatcher = self._request._dispatcher + code, reason = dispatcher.passive_closing_handshake(self._request) + if code is None and reason is not None and len(reason) > 0: + self._logger.warning( + 'Handler specified reason despite code being None') + reason = '' + if reason is None: + reason = '' + self._send_closing_handshake(code, reason) + self._logger.debug( + 'Acknowledged closing handshake initiated by the peer ' + '(code=%r, reason=%r)', code, reason) + + def _process_ping_message(self, message): + """Processes ping message. + + Args: + message: ping message. + """ + + try: + handler = self._request.on_ping_handler + if handler: + handler(self._request, message) + return + except AttributeError: + pass + self._send_pong(message) + + def _process_pong_message(self, message): + """Processes pong message. + + Args: + message: pong message. + """ + + # TODO(tyoshino): Add ping timeout handling. + + inflight_pings = deque() + + while True: + try: + expected_body = self._ping_queue.popleft() + if expected_body == message: + # inflight_pings contains pings ignored by the + # other peer. Just forget them. + self._logger.debug( + 'Ping %r is acked (%d pings were ignored)', + expected_body, len(inflight_pings)) + break + else: + inflight_pings.append(expected_body) + except IndexError: + # The received pong was unsolicited pong. Keep the + # ping queue as is. + self._ping_queue = inflight_pings + self._logger.debug('Received a unsolicited pong') + break + + try: + handler = self._request.on_pong_handler + if handler: + handler(self._request, message) + except AttributeError: + pass + + def receive_message(self): + """Receive a WebSocket frame and return its payload as a text in + unicode or a binary in str. + + Returns: + payload data of the frame + - as unicode instance if received text frame + - as str instance if received binary frame + or None iff received closing handshake. + Raises: + BadOperationException: when called on a client-terminated + connection. + ConnectionTerminatedException: when read returns empty + string. + InvalidFrameException: when the frame contains invalid + data. + UnsupportedFrameException: when the received frame has + flags, opcode we cannot handle. You can ignore this + exception and continue receiving the next frame. + """ + + if self._request.client_terminated: + raise BadOperationException( + 'Requested receive_message after receiving a closing ' + 'handshake') + + while True: + # mp_conn.read will block if no bytes are available. + + frame = self._receive_frame_as_frame_object() + + # Check the constraint on the payload size for control frames + # before extension processes the frame. + # See also http://tools.ietf.org/html/rfc6455#section-5.5 + if (common.is_control_opcode(frame.opcode) + and len(frame.payload) > 125): + raise InvalidFrameException( + 'Payload data size of control frames must be 125 bytes or ' + 'less') + + for frame_filter in self._options.incoming_frame_filters: + frame_filter.filter(frame) + + if frame.rsv1 or frame.rsv2 or frame.rsv3: + raise UnsupportedFrameException( + 'Unsupported flag is set (rsv = %d%d%d)' % + (frame.rsv1, frame.rsv2, frame.rsv3)) + + message = self._get_message_from_frame(frame) + if message is None: + continue + + for message_filter in self._options.incoming_message_filters: + message = message_filter.filter(message) + + if self._original_opcode == common.OPCODE_TEXT: + # The WebSocket protocol section 4.4 specifies that invalid + # characters must be replaced with U+fffd REPLACEMENT + # CHARACTER. + try: + return message.decode('utf-8') + except UnicodeDecodeError as e: + raise InvalidUTF8Exception(e) + elif self._original_opcode == common.OPCODE_BINARY: + return message + elif self._original_opcode == common.OPCODE_CLOSE: + self._process_close_message(message) + return None + elif self._original_opcode == common.OPCODE_PING: + self._process_ping_message(message) + elif self._original_opcode == common.OPCODE_PONG: + self._process_pong_message(message) + else: + raise UnsupportedFrameException('Opcode %d is not supported' % + self._original_opcode) + + def _send_closing_handshake(self, code, reason): + body = create_closing_handshake_body(code, reason) + frame = create_close_frame( + body, + mask=self._options.mask_send, + frame_filters=self._options.outgoing_frame_filters) + + self._request.server_terminated = True + + self._write(frame) + + def close_connection(self, + code=common.STATUS_NORMAL_CLOSURE, + reason='', + wait_response=True): + """Closes a WebSocket connection. Note that this method blocks until + it receives acknowledgement to the closing handshake. + + Args: + code: Status code for close frame. If code is None, a close + frame with empty body will be sent. + reason: string representing close reason. + wait_response: True when caller want to wait the response. + Raises: + BadOperationException: when reason is specified with code None + or reason is not an instance of both str and unicode. + """ + + if self._request.server_terminated: + self._logger.debug( + 'Requested close_connection but server is already terminated') + return + + # When we receive a close frame, we call _process_close_message(). + # _process_close_message() immediately acknowledges to the + # server-initiated closing handshake and sets server_terminated to + # True. So, here we can assume that we haven't received any close + # frame. We're initiating a closing handshake. + + if code is None: + if reason is not None and len(reason) > 0: + raise BadOperationException( + 'close reason must not be specified if code is None') + reason = '' + else: + if not isinstance(reason, bytes) and not isinstance( + reason, six.text_type): + raise BadOperationException( + 'close reason must be an instance of bytes or unicode') + + self._send_closing_handshake(code, reason) + self._logger.debug('Initiated closing handshake (code=%r, reason=%r)', + code, reason) + + if (code == common.STATUS_GOING_AWAY + or code == common.STATUS_PROTOCOL_ERROR) or not wait_response: + # It doesn't make sense to wait for a close frame if the reason is + # protocol error or that the server is going away. For some of + # other reasons, it might not make sense to wait for a close frame, + # but it's not clear, yet. + return + + # TODO(ukai): 2. wait until the /client terminated/ flag has been set, + # or until a server-defined timeout expires. + # + # For now, we expect receiving closing handshake right after sending + # out closing handshake. + message = self.receive_message() + if message is not None: + raise ConnectionTerminatedException( + 'Didn\'t receive valid ack for closing handshake') + # TODO: 3. close the WebSocket connection. + # note: mod_python Connection (mp_conn) doesn't have close method. + + def send_ping(self, body, binary=False): + if not binary and isinstance(body, six.text_type): + body = body.encode('UTF-8') + frame = create_ping_frame(body, self._options.mask_send, + self._options.outgoing_frame_filters) + self._write(frame) + + self._ping_queue.append(body) + + def _send_pong(self, body): + frame = create_pong_frame(body, self._options.mask_send, + self._options.outgoing_frame_filters) + self._write(frame) + + def get_last_received_opcode(self): + """Returns the opcode of the WebSocket message which the last received + frame belongs to. The return value is valid iff immediately after + receive_message call. + """ + + return self._original_opcode + + +# vi:sts=4 sw=4 et diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/util.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/util.py new file mode 100644 index 0000000000..9c25ab8315 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/util.py @@ -0,0 +1,384 @@ +# Copyright 2011, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +"""WebSocket utilities.""" + +from __future__ import absolute_import + +import logging +import os +import re +import struct +import zlib + +import six +from six.moves import map, range + +try: + from pywebsocket3 import fast_masking +except ImportError: + pass + + +def prepend_message_to_exception(message, exc): + """Prepend message to the exception.""" + exc.args = (message + str(exc), ) + return + + +def __translate_interp(interp, cygwin_path): + """Translate interp program path for Win32 python to run cygwin program + (e.g. perl). Note that it doesn't support path that contains space, + which is typically true for Unix, where #!-script is written. + For Win32 python, cygwin_path is a directory of cygwin binaries. + + Args: + interp: interp command line + cygwin_path: directory name of cygwin binary, or None + Returns: + translated interp command line. + """ + if not cygwin_path: + return interp + m = re.match('^[^ ]*/([^ ]+)( .*)?', interp) + if m: + cmd = os.path.join(cygwin_path, m.group(1)) + return cmd + m.group(2) + return interp + + +def get_script_interp(script_path, cygwin_path=None): + r"""Get #!-interpreter command line from the script. + + It also fixes command path. When Cygwin Python is used, e.g. in WebKit, + it could run "/usr/bin/perl -wT hello.pl". + When Win32 Python is used, e.g. in Chromium, it couldn't. So, fix + "/usr/bin/perl" to "\perl.exe". + + Args: + script_path: pathname of the script + cygwin_path: directory name of cygwin binary, or None + Returns: + #!-interpreter command line, or None if it is not #!-script. + """ + fp = open(script_path) + line = fp.readline() + fp.close() + m = re.match('^#!(.*)', line) + if m: + return __translate_interp(m.group(1), cygwin_path) + return None + + +def hexify(s): + return ' '.join(['%02x' % x for x in six.iterbytes(s)]) + + +def get_class_logger(o): + """Return the logging class information.""" + return logging.getLogger('%s.%s' % + (o.__class__.__module__, o.__class__.__name__)) + + +def pack_byte(b): + """Pack an integer to network-ordered byte""" + return struct.pack('!B', b) + + +class NoopMasker(object): + """A NoOp masking object. + + This has the same interface as RepeatedXorMasker but just returns + the string passed in without making any change. + """ + def __init__(self): + """NoOp.""" + pass + + def mask(self, s): + """NoOp.""" + return s + + +class RepeatedXorMasker(object): + """A masking object that applies XOR on the string. + + Applies XOR on the byte string given to mask method with the masking bytes + given to the constructor repeatedly. This object remembers the position + in the masking bytes the last mask method call ended and resumes from + that point on the next mask method call. + """ + def __init__(self, masking_key): + self._masking_key = masking_key + self._masking_key_index = 0 + + def _mask_using_swig(self, s): + """Perform the mask via SWIG.""" + masked_data = fast_masking.mask(s, self._masking_key, + self._masking_key_index) + self._masking_key_index = ((self._masking_key_index + len(s)) % + len(self._masking_key)) + return masked_data + + def _mask_using_array(self, s): + """Perform the mask via python.""" + if isinstance(s, six.text_type): + raise Exception( + 'Masking Operation should not process unicode strings') + + result = bytearray(s) + + # Use temporary local variables to eliminate the cost to access + # attributes + masking_key = [c for c in six.iterbytes(self._masking_key)] + masking_key_size = len(masking_key) + masking_key_index = self._masking_key_index + + for i in range(len(result)): + result[i] ^= masking_key[masking_key_index] + masking_key_index = (masking_key_index + 1) % masking_key_size + + self._masking_key_index = masking_key_index + + return bytes(result) + + if 'fast_masking' in globals(): + mask = _mask_using_swig + else: + mask = _mask_using_array + + +# By making wbits option negative, we can suppress CMF/FLG (2 octet) and +# ADLER32 (4 octet) fields of zlib so that we can use zlib module just as +# deflate library. DICTID won't be added as far as we don't set dictionary. +# LZ77 window of 32K will be used for both compression and decompression. +# For decompression, we can just use 32K to cover any windows size. For +# compression, we use 32K so receivers must use 32K. +# +# Compression level is Z_DEFAULT_COMPRESSION. We don't have to match level +# to decode. +# +# See zconf.h, deflate.cc, inflate.cc of zlib library, and zlibmodule.c of +# Python. See also RFC1950 (ZLIB 3.3). + + +class _Deflater(object): + def __init__(self, window_bits): + self._logger = get_class_logger(self) + + # Using the smallest window bits of 9 for generating input frames. + # On WebSocket spec, the smallest window bit is 8. However, zlib does + # not accept window_bit = 8. + # + # Because of a zlib deflate quirk, back-references will not use the + # entire range of 1 << window_bits, but will instead use a restricted + # range of (1 << window_bits) - 262. With an increased window_bits = 9, + # back-references will be within a range of 250. These can still be + # decompressed with window_bits = 8 and the 256-byte window used there. + # + # Similar disscussions can be found in https://crbug.com/691074 + window_bits = max(window_bits, 9) + + self._compress = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, + zlib.DEFLATED, -window_bits) + + def compress(self, bytes): + compressed_bytes = self._compress.compress(bytes) + self._logger.debug('Compress input %r', bytes) + self._logger.debug('Compress result %r', compressed_bytes) + return compressed_bytes + + def compress_and_flush(self, bytes): + compressed_bytes = self._compress.compress(bytes) + compressed_bytes += self._compress.flush(zlib.Z_SYNC_FLUSH) + self._logger.debug('Compress input %r', bytes) + self._logger.debug('Compress result %r', compressed_bytes) + return compressed_bytes + + def compress_and_finish(self, bytes): + compressed_bytes = self._compress.compress(bytes) + compressed_bytes += self._compress.flush(zlib.Z_FINISH) + self._logger.debug('Compress input %r', bytes) + self._logger.debug('Compress result %r', compressed_bytes) + return compressed_bytes + + +class _Inflater(object): + def __init__(self, window_bits): + self._logger = get_class_logger(self) + self._window_bits = window_bits + + self._unconsumed = b'' + + self.reset() + + def decompress(self, size): + if not (size == -1 or size > 0): + raise Exception('size must be -1 or positive') + + data = b'' + + while True: + data += self._decompress.decompress(self._unconsumed, + max(0, size - len(data))) + self._unconsumed = self._decompress.unconsumed_tail + if self._decompress.unused_data: + # Encountered a last block (i.e. a block with BFINAL = 1) and + # found a new stream (unused_data). We cannot use the same + # zlib.Decompress object for the new stream. Create a new + # Decompress object to decompress the new one. + # + # It's fine to ignore unconsumed_tail if unused_data is not + # empty. + self._unconsumed = self._decompress.unused_data + self.reset() + if size >= 0 and len(data) == size: + # data is filled. Don't call decompress again. + break + else: + # Re-invoke Decompress.decompress to try to decompress all + # available bytes before invoking read which blocks until + # any new byte is available. + continue + else: + # Here, since unused_data is empty, even if unconsumed_tail is + # not empty, bytes of requested length are already in data. We + # don't have to "continue" here. + break + + if data: + self._logger.debug('Decompressed %r', data) + return data + + def append(self, data): + self._logger.debug('Appended %r', data) + self._unconsumed += data + + def reset(self): + self._logger.debug('Reset') + self._decompress = zlib.decompressobj(-self._window_bits) + + +# Compresses/decompresses given octets using the method introduced in RFC1979. + + +class _RFC1979Deflater(object): + """A compressor class that applies DEFLATE to given byte sequence and + flushes using the algorithm described in the RFC1979 section 2.1. + """ + def __init__(self, window_bits, no_context_takeover): + self._deflater = None + if window_bits is None: + window_bits = zlib.MAX_WBITS + self._window_bits = window_bits + self._no_context_takeover = no_context_takeover + + def filter(self, bytes, end=True, bfinal=False): + if self._deflater is None: + self._deflater = _Deflater(self._window_bits) + + if bfinal: + result = self._deflater.compress_and_finish(bytes) + # Add a padding block with BFINAL = 0 and BTYPE = 0. + result = result + pack_byte(0) + self._deflater = None + return result + + result = self._deflater.compress_and_flush(bytes) + if end: + # Strip last 4 octets which is LEN and NLEN field of a + # non-compressed block added for Z_SYNC_FLUSH. + result = result[:-4] + + if self._no_context_takeover and end: + self._deflater = None + + return result + + +class _RFC1979Inflater(object): + """A decompressor class a la RFC1979. + + A decompressor class for byte sequence compressed and flushed following + the algorithm described in the RFC1979 section 2.1. + """ + def __init__(self, window_bits=zlib.MAX_WBITS): + self._inflater = _Inflater(window_bits) + + def filter(self, bytes): + # Restore stripped LEN and NLEN field of a non-compressed block added + # for Z_SYNC_FLUSH. + self._inflater.append(bytes + b'\x00\x00\xff\xff') + return self._inflater.decompress(-1) + + +class DeflateSocket(object): + """A wrapper class for socket object to intercept send and recv to perform + deflate compression and decompression transparently. + """ + + # Size of the buffer passed to recv to receive compressed data. + _RECV_SIZE = 4096 + + def __init__(self, socket): + self._socket = socket + + self._logger = get_class_logger(self) + + self._deflater = _Deflater(zlib.MAX_WBITS) + self._inflater = _Inflater(zlib.MAX_WBITS) + + def recv(self, size): + """Receives data from the socket specified on the construction up + to the specified size. Once any data is available, returns it even + if it's smaller than the specified size. + """ + + # TODO(tyoshino): Allow call with size=0. It should block until any + # decompressed data is available. + if size <= 0: + raise Exception('Non-positive size passed') + while True: + data = self._inflater.decompress(size) + if len(data) != 0: + return data + + read_data = self._socket.recv(DeflateSocket._RECV_SIZE) + if not read_data: + return b'' + self._inflater.append(read_data) + + def sendall(self, bytes): + self.send(bytes) + + def send(self, bytes): + self._socket.sendall(self._deflater.compress_and_flush(bytes)) + return len(bytes) + + +# vi:sts=4 sw=4 et diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/websocket_server.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/websocket_server.py new file mode 100644 index 0000000000..dab2f079ff --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/pywebsocket3/pywebsocket3/websocket_server.py @@ -0,0 +1,290 @@ +# Copyright 2020, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +"""Standalone WebsocketServer + +This file deals with the main module of standalone server. Although it is fine +to import this file directly to use WebSocketServer, it is strongly recommended +to use standalone.py, since it is intended to act as a skeleton of this module. +""" + +from __future__ import absolute_import + +import logging +import re +import select +import socket +import ssl +import threading +import traceback + +from six.moves import BaseHTTPServer, socketserver + +from pywebsocket3 import dispatch, util +from pywebsocket3.request_handler import WebSocketRequestHandler + + +def _alias_handlers(dispatcher, websock_handlers_map_file): + """Set aliases specified in websock_handler_map_file in dispatcher. + + Args: + dispatcher: dispatch.Dispatcher instance + websock_handler_map_file: alias map file + """ + + with open(websock_handlers_map_file) as f: + for line in f: + if line[0] == '#' or line.isspace(): + continue + m = re.match(r'(\S+)\s+(\S+)$', line) + if not m: + logging.warning('Wrong format in map file:' + line) + continue + try: + dispatcher.add_resource_path_alias(m.group(1), m.group(2)) + except dispatch.DispatchException as e: + logging.error(str(e)) + + +class WebSocketServer(socketserver.ThreadingMixIn, BaseHTTPServer.HTTPServer): + """HTTPServer specialized for WebSocket.""" + + # Overrides SocketServer.ThreadingMixIn.daemon_threads + daemon_threads = True + # Overrides BaseHTTPServer.HTTPServer.allow_reuse_address + allow_reuse_address = True + + def __init__(self, options): + """Override SocketServer.TCPServer.__init__ to set SSL enabled + socket object to self.socket before server_bind and server_activate, + if necessary. + """ + + # Fall back to None for embedders that don't know about the + # handler_encoding option. + handler_encoding = getattr(options, "handler_encoding", None) + + # Share a Dispatcher among request handlers to save time for + # instantiation. Dispatcher can be shared because it is thread-safe. + options.dispatcher = dispatch.Dispatcher( + options.websock_handlers, options.scan_dir, + options.allow_handlers_outside_root_dir, handler_encoding) + if options.websock_handlers_map_file: + _alias_handlers(options.dispatcher, + options.websock_handlers_map_file) + warnings = options.dispatcher.source_warnings() + if warnings: + for warning in warnings: + logging.warning('Warning in source loading: %s' % warning) + + self._logger = util.get_class_logger(self) + + self.request_queue_size = options.request_queue_size + self.__ws_is_shut_down = threading.Event() + self.__ws_serving = False + + socketserver.BaseServer.__init__(self, + (options.server_host, options.port), + WebSocketRequestHandler) + + # Expose the options object to allow handler objects access it. We name + # it with websocket_ prefix to avoid conflict. + self.websocket_server_options = options + + self._create_sockets() + self.server_bind() + self.server_activate() + + def _create_sockets(self): + self.server_name, self.server_port = self.server_address + self._sockets = [] + if not self.server_name: + # On platforms that doesn't support IPv6, the first bind fails. + # On platforms that supports IPv6 + # - If it binds both IPv4 and IPv6 on call with AF_INET6, the + # first bind succeeds and the second fails (we'll see 'Address + # already in use' error). + # - If it binds only IPv6 on call with AF_INET6, both call are + # expected to succeed to listen both protocol. + addrinfo_array = [(socket.AF_INET6, socket.SOCK_STREAM, '', '', + ''), + (socket.AF_INET, socket.SOCK_STREAM, '', '', '')] + else: + addrinfo_array = socket.getaddrinfo(self.server_name, + self.server_port, + socket.AF_UNSPEC, + socket.SOCK_STREAM, + socket.IPPROTO_TCP) + for addrinfo in addrinfo_array: + self._logger.info('Create socket on: %r', addrinfo) + family, socktype, proto, canonname, sockaddr = addrinfo + try: + socket_ = socket.socket(family, socktype) + except Exception as e: + self._logger.info('Skip by failure: %r', e) + continue + server_options = self.websocket_server_options + if server_options.use_tls: + if server_options.tls_client_auth: + if server_options.tls_client_cert_optional: + client_cert_ = ssl.CERT_OPTIONAL + else: + client_cert_ = ssl.CERT_REQUIRED + else: + client_cert_ = ssl.CERT_NONE + ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ssl_context.verify_mode = client_cert_ + ssl_context.load_cert_chain(keyfile=server_options.private_key, + certfile=server_options.certificate) + if client_cert_ != ssl.CERT_NONE: + ssl_context.load_verify_locations(cafile=server_options.tls_client_ca) + socket_ = ssl_context.wrap_socket(socket_, server_side=True) + self._sockets.append((socket_, addrinfo)) + + def server_bind(self): + """Override SocketServer.TCPServer.server_bind to enable multiple + sockets bind. + """ + + failed_sockets = [] + + for socketinfo in self._sockets: + socket_, addrinfo = socketinfo + self._logger.info('Bind on: %r', addrinfo) + if self.allow_reuse_address: + socket_.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + try: + socket_.bind(self.server_address) + except Exception as e: + self._logger.info('Skip by failure: %r', e) + socket_.close() + failed_sockets.append(socketinfo) + if self.server_address[1] == 0: + # The operating system assigns the actual port number for port + # number 0. This case, the second and later sockets should use + # the same port number. Also self.server_port is rewritten + # because it is exported, and will be used by external code. + self.server_address = (self.server_name, + socket_.getsockname()[1]) + self.server_port = self.server_address[1] + self._logger.info('Port %r is assigned', self.server_port) + + for socketinfo in failed_sockets: + self._sockets.remove(socketinfo) + + def server_activate(self): + """Override SocketServer.TCPServer.server_activate to enable multiple + sockets listen. + """ + + failed_sockets = [] + + for socketinfo in self._sockets: + socket_, addrinfo = socketinfo + self._logger.info('Listen on: %r', addrinfo) + try: + socket_.listen(self.request_queue_size) + except Exception as e: + self._logger.info('Skip by failure: %r', e) + socket_.close() + failed_sockets.append(socketinfo) + + for socketinfo in failed_sockets: + self._sockets.remove(socketinfo) + + if len(self._sockets) == 0: + self._logger.critical( + 'No sockets activated. Use info log level to see the reason.') + + def server_close(self): + """Override SocketServer.TCPServer.server_close to enable multiple + sockets close. + """ + + for socketinfo in self._sockets: + socket_, addrinfo = socketinfo + self._logger.info('Close on: %r', addrinfo) + socket_.close() + + def fileno(self): + """Override SocketServer.TCPServer.fileno.""" + + self._logger.critical('Not supported: fileno') + return self._sockets[0][0].fileno() + + def handle_error(self, request, client_address): + """Override SocketServer.handle_error.""" + + self._logger.error('Exception in processing request from: %r\n%s', + client_address, traceback.format_exc()) + # Note: client_address is a tuple. + + def get_request(self): + """Override TCPServer.get_request.""" + + accepted_socket, client_address = self.socket.accept() + + server_options = self.websocket_server_options + if server_options.use_tls: + # Print cipher in use. Handshake is done on accept. + self._logger.debug('Cipher: %s', accepted_socket.cipher()) + self._logger.debug('Client cert: %r', + accepted_socket.getpeercert()) + + return accepted_socket, client_address + + def serve_forever(self, poll_interval=0.5): + """Override SocketServer.BaseServer.serve_forever.""" + + self.__ws_serving = True + self.__ws_is_shut_down.clear() + handle_request = self.handle_request + if hasattr(self, '_handle_request_noblock'): + handle_request = self._handle_request_noblock + else: + self._logger.warning('Fallback to blocking request handler') + try: + while self.__ws_serving: + r, w, e = select.select( + [socket_[0] for socket_ in self._sockets], [], [], + poll_interval) + for socket_ in r: + self.socket = socket_ + handle_request() + self.socket = None + finally: + self.__ws_is_shut_down.set() + + def shutdown(self): + """Override SocketServer.BaseServer.shutdown.""" + + self.__ws_serving = False + self.__ws_is_shut_down.wait() + + +# vi:sts=4 sw=4 et diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/six/six.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/six/six.py new file mode 100644 index 0000000000..4e15675d8b --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/third_party/six/six.py @@ -0,0 +1,998 @@ +# Copyright (c) 2010-2020 Benjamin Peterson +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +"""Utilities for writing code that runs on Python 2 and 3""" + +from __future__ import absolute_import + +import functools +import itertools +import operator +import sys +import types + +__author__ = "Benjamin Peterson " +__version__ = "1.16.0" + + +# Useful for very coarse version differentiation. +PY2 = sys.version_info[0] == 2 +PY3 = sys.version_info[0] == 3 +PY34 = sys.version_info[0:2] >= (3, 4) + +if PY3: + string_types = str, + integer_types = int, + class_types = type, + text_type = str + binary_type = bytes + + MAXSIZE = sys.maxsize +else: + string_types = basestring, + integer_types = (int, long) + class_types = (type, types.ClassType) + text_type = unicode + binary_type = str + + if sys.platform.startswith("java"): + # Jython always uses 32 bits. + MAXSIZE = int((1 << 31) - 1) + else: + # It's possible to have sizeof(long) != sizeof(Py_ssize_t). + class X(object): + + def __len__(self): + return 1 << 31 + try: + len(X()) + except OverflowError: + # 32-bit + MAXSIZE = int((1 << 31) - 1) + else: + # 64-bit + MAXSIZE = int((1 << 63) - 1) + del X + +if PY34: + from importlib.util import spec_from_loader +else: + spec_from_loader = None + + +def _add_doc(func, doc): + """Add documentation to a function.""" + func.__doc__ = doc + + +def _import_module(name): + """Import module, returning the module after the last dot.""" + __import__(name) + return sys.modules[name] + + +class _LazyDescr(object): + + def __init__(self, name): + self.name = name + + def __get__(self, obj, tp): + result = self._resolve() + setattr(obj, self.name, result) # Invokes __set__. + try: + # This is a bit ugly, but it avoids running this again by + # removing this descriptor. + delattr(obj.__class__, self.name) + except AttributeError: + pass + return result + + +class MovedModule(_LazyDescr): + + def __init__(self, name, old, new=None): + super(MovedModule, self).__init__(name) + if PY3: + if new is None: + new = name + self.mod = new + else: + self.mod = old + + def _resolve(self): + return _import_module(self.mod) + + def __getattr__(self, attr): + _module = self._resolve() + value = getattr(_module, attr) + setattr(self, attr, value) + return value + + +class _LazyModule(types.ModuleType): + + def __init__(self, name): + super(_LazyModule, self).__init__(name) + self.__doc__ = self.__class__.__doc__ + + def __dir__(self): + attrs = ["__doc__", "__name__"] + attrs += [attr.name for attr in self._moved_attributes] + return attrs + + # Subclasses should override this + _moved_attributes = [] + + +class MovedAttribute(_LazyDescr): + + def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None): + super(MovedAttribute, self).__init__(name) + if PY3: + if new_mod is None: + new_mod = name + self.mod = new_mod + if new_attr is None: + if old_attr is None: + new_attr = name + else: + new_attr = old_attr + self.attr = new_attr + else: + self.mod = old_mod + if old_attr is None: + old_attr = name + self.attr = old_attr + + def _resolve(self): + module = _import_module(self.mod) + return getattr(module, self.attr) + + +class _SixMetaPathImporter(object): + + """ + A meta path importer to import six.moves and its submodules. + + This class implements a PEP302 finder and loader. It should be compatible + with Python 2.5 and all existing versions of Python3 + """ + + def __init__(self, six_module_name): + self.name = six_module_name + self.known_modules = {} + + def _add_module(self, mod, *fullnames): + for fullname in fullnames: + self.known_modules[self.name + "." + fullname] = mod + + def _get_module(self, fullname): + return self.known_modules[self.name + "." + fullname] + + def find_module(self, fullname, path=None): + if fullname in self.known_modules: + return self + return None + + def find_spec(self, fullname, path, target=None): + if fullname in self.known_modules: + return spec_from_loader(fullname, self) + return None + + def __get_module(self, fullname): + try: + return self.known_modules[fullname] + except KeyError: + raise ImportError("This loader does not know module " + fullname) + + def load_module(self, fullname): + try: + # in case of a reload + return sys.modules[fullname] + except KeyError: + pass + mod = self.__get_module(fullname) + if isinstance(mod, MovedModule): + mod = mod._resolve() + else: + mod.__loader__ = self + sys.modules[fullname] = mod + return mod + + def is_package(self, fullname): + """ + Return true, if the named module is a package. + + We need this method to get correct spec objects with + Python 3.4 (see PEP451) + """ + return hasattr(self.__get_module(fullname), "__path__") + + def get_code(self, fullname): + """Return None + + Required, if is_package is implemented""" + self.__get_module(fullname) # eventually raises ImportError + return None + get_source = get_code # same as get_code + + def create_module(self, spec): + return self.load_module(spec.name) + + def exec_module(self, module): + pass + +_importer = _SixMetaPathImporter(__name__) + + +class _MovedItems(_LazyModule): + + """Lazy loading of moved objects""" + __path__ = [] # mark as package + + +_moved_attributes = [ + MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"), + MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"), + MovedAttribute("filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse"), + MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"), + MovedAttribute("intern", "__builtin__", "sys"), + MovedAttribute("map", "itertools", "builtins", "imap", "map"), + MovedAttribute("getcwd", "os", "os", "getcwdu", "getcwd"), + MovedAttribute("getcwdb", "os", "os", "getcwd", "getcwdb"), + MovedAttribute("getoutput", "commands", "subprocess"), + MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"), + MovedAttribute("reload_module", "__builtin__", "importlib" if PY34 else "imp", "reload"), + MovedAttribute("reduce", "__builtin__", "functools"), + MovedAttribute("shlex_quote", "pipes", "shlex", "quote"), + MovedAttribute("StringIO", "StringIO", "io"), + MovedAttribute("UserDict", "UserDict", "collections"), + MovedAttribute("UserList", "UserList", "collections"), + MovedAttribute("UserString", "UserString", "collections"), + MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"), + MovedAttribute("zip", "itertools", "builtins", "izip", "zip"), + MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"), + MovedModule("builtins", "__builtin__"), + MovedModule("configparser", "ConfigParser"), + MovedModule("collections_abc", "collections", "collections.abc" if sys.version_info >= (3, 3) else "collections"), + MovedModule("copyreg", "copy_reg"), + MovedModule("dbm_gnu", "gdbm", "dbm.gnu"), + MovedModule("dbm_ndbm", "dbm", "dbm.ndbm"), + MovedModule("_dummy_thread", "dummy_thread", "_dummy_thread" if sys.version_info < (3, 9) else "_thread"), + MovedModule("http_cookiejar", "cookielib", "http.cookiejar"), + MovedModule("http_cookies", "Cookie", "http.cookies"), + MovedModule("html_entities", "htmlentitydefs", "html.entities"), + MovedModule("html_parser", "HTMLParser", "html.parser"), + MovedModule("http_client", "httplib", "http.client"), + MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"), + MovedModule("email_mime_image", "email.MIMEImage", "email.mime.image"), + MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"), + MovedModule("email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart"), + MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"), + MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"), + MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"), + MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"), + MovedModule("cPickle", "cPickle", "pickle"), + MovedModule("queue", "Queue"), + MovedModule("reprlib", "repr"), + MovedModule("socketserver", "SocketServer"), + MovedModule("_thread", "thread", "_thread"), + MovedModule("tkinter", "Tkinter"), + MovedModule("tkinter_dialog", "Dialog", "tkinter.dialog"), + MovedModule("tkinter_filedialog", "FileDialog", "tkinter.filedialog"), + MovedModule("tkinter_scrolledtext", "ScrolledText", "tkinter.scrolledtext"), + MovedModule("tkinter_simpledialog", "SimpleDialog", "tkinter.simpledialog"), + MovedModule("tkinter_tix", "Tix", "tkinter.tix"), + MovedModule("tkinter_ttk", "ttk", "tkinter.ttk"), + MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"), + MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"), + MovedModule("tkinter_colorchooser", "tkColorChooser", + "tkinter.colorchooser"), + MovedModule("tkinter_commondialog", "tkCommonDialog", + "tkinter.commondialog"), + MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"), + MovedModule("tkinter_font", "tkFont", "tkinter.font"), + MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"), + MovedModule("tkinter_tksimpledialog", "tkSimpleDialog", + "tkinter.simpledialog"), + MovedModule("urllib_parse", __name__ + ".moves.urllib_parse", "urllib.parse"), + MovedModule("urllib_error", __name__ + ".moves.urllib_error", "urllib.error"), + MovedModule("urllib", __name__ + ".moves.urllib", __name__ + ".moves.urllib"), + MovedModule("urllib_robotparser", "robotparser", "urllib.robotparser"), + MovedModule("xmlrpc_client", "xmlrpclib", "xmlrpc.client"), + MovedModule("xmlrpc_server", "SimpleXMLRPCServer", "xmlrpc.server"), +] +# Add windows specific modules. +if sys.platform == "win32": + _moved_attributes += [ + MovedModule("winreg", "_winreg"), + ] + +for attr in _moved_attributes: + setattr(_MovedItems, attr.name, attr) + if isinstance(attr, MovedModule): + _importer._add_module(attr, "moves." + attr.name) +del attr + +_MovedItems._moved_attributes = _moved_attributes + +moves = _MovedItems(__name__ + ".moves") +_importer._add_module(moves, "moves") + + +class Module_six_moves_urllib_parse(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_parse""" + + +_urllib_parse_moved_attributes = [ + MovedAttribute("ParseResult", "urlparse", "urllib.parse"), + MovedAttribute("SplitResult", "urlparse", "urllib.parse"), + MovedAttribute("parse_qs", "urlparse", "urllib.parse"), + MovedAttribute("parse_qsl", "urlparse", "urllib.parse"), + MovedAttribute("urldefrag", "urlparse", "urllib.parse"), + MovedAttribute("urljoin", "urlparse", "urllib.parse"), + MovedAttribute("urlparse", "urlparse", "urllib.parse"), + MovedAttribute("urlsplit", "urlparse", "urllib.parse"), + MovedAttribute("urlunparse", "urlparse", "urllib.parse"), + MovedAttribute("urlunsplit", "urlparse", "urllib.parse"), + MovedAttribute("quote", "urllib", "urllib.parse"), + MovedAttribute("quote_plus", "urllib", "urllib.parse"), + MovedAttribute("unquote", "urllib", "urllib.parse"), + MovedAttribute("unquote_plus", "urllib", "urllib.parse"), + MovedAttribute("unquote_to_bytes", "urllib", "urllib.parse", "unquote", "unquote_to_bytes"), + MovedAttribute("urlencode", "urllib", "urllib.parse"), + MovedAttribute("splitquery", "urllib", "urllib.parse"), + MovedAttribute("splittag", "urllib", "urllib.parse"), + MovedAttribute("splituser", "urllib", "urllib.parse"), + MovedAttribute("splitvalue", "urllib", "urllib.parse"), + MovedAttribute("uses_fragment", "urlparse", "urllib.parse"), + MovedAttribute("uses_netloc", "urlparse", "urllib.parse"), + MovedAttribute("uses_params", "urlparse", "urllib.parse"), + MovedAttribute("uses_query", "urlparse", "urllib.parse"), + MovedAttribute("uses_relative", "urlparse", "urllib.parse"), +] +for attr in _urllib_parse_moved_attributes: + setattr(Module_six_moves_urllib_parse, attr.name, attr) +del attr + +Module_six_moves_urllib_parse._moved_attributes = _urllib_parse_moved_attributes + +_importer._add_module(Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse"), + "moves.urllib_parse", "moves.urllib.parse") + + +class Module_six_moves_urllib_error(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_error""" + + +_urllib_error_moved_attributes = [ + MovedAttribute("URLError", "urllib2", "urllib.error"), + MovedAttribute("HTTPError", "urllib2", "urllib.error"), + MovedAttribute("ContentTooShortError", "urllib", "urllib.error"), +] +for attr in _urllib_error_moved_attributes: + setattr(Module_six_moves_urllib_error, attr.name, attr) +del attr + +Module_six_moves_urllib_error._moved_attributes = _urllib_error_moved_attributes + +_importer._add_module(Module_six_moves_urllib_error(__name__ + ".moves.urllib.error"), + "moves.urllib_error", "moves.urllib.error") + + +class Module_six_moves_urllib_request(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_request""" + + +_urllib_request_moved_attributes = [ + MovedAttribute("urlopen", "urllib2", "urllib.request"), + MovedAttribute("install_opener", "urllib2", "urllib.request"), + MovedAttribute("build_opener", "urllib2", "urllib.request"), + MovedAttribute("pathname2url", "urllib", "urllib.request"), + MovedAttribute("url2pathname", "urllib", "urllib.request"), + MovedAttribute("getproxies", "urllib", "urllib.request"), + MovedAttribute("Request", "urllib2", "urllib.request"), + MovedAttribute("OpenerDirector", "urllib2", "urllib.request"), + MovedAttribute("HTTPDefaultErrorHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPRedirectHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPCookieProcessor", "urllib2", "urllib.request"), + MovedAttribute("ProxyHandler", "urllib2", "urllib.request"), + MovedAttribute("BaseHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPPasswordMgr", "urllib2", "urllib.request"), + MovedAttribute("HTTPPasswordMgrWithDefaultRealm", "urllib2", "urllib.request"), + MovedAttribute("AbstractBasicAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPBasicAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("ProxyBasicAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("AbstractDigestAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPDigestAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("ProxyDigestAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPSHandler", "urllib2", "urllib.request"), + MovedAttribute("FileHandler", "urllib2", "urllib.request"), + MovedAttribute("FTPHandler", "urllib2", "urllib.request"), + MovedAttribute("CacheFTPHandler", "urllib2", "urllib.request"), + MovedAttribute("UnknownHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPErrorProcessor", "urllib2", "urllib.request"), + MovedAttribute("urlretrieve", "urllib", "urllib.request"), + MovedAttribute("urlcleanup", "urllib", "urllib.request"), + MovedAttribute("URLopener", "urllib", "urllib.request"), + MovedAttribute("FancyURLopener", "urllib", "urllib.request"), + MovedAttribute("proxy_bypass", "urllib", "urllib.request"), + MovedAttribute("parse_http_list", "urllib2", "urllib.request"), + MovedAttribute("parse_keqv_list", "urllib2", "urllib.request"), +] +for attr in _urllib_request_moved_attributes: + setattr(Module_six_moves_urllib_request, attr.name, attr) +del attr + +Module_six_moves_urllib_request._moved_attributes = _urllib_request_moved_attributes + +_importer._add_module(Module_six_moves_urllib_request(__name__ + ".moves.urllib.request"), + "moves.urllib_request", "moves.urllib.request") + + +class Module_six_moves_urllib_response(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_response""" + + +_urllib_response_moved_attributes = [ + MovedAttribute("addbase", "urllib", "urllib.response"), + MovedAttribute("addclosehook", "urllib", "urllib.response"), + MovedAttribute("addinfo", "urllib", "urllib.response"), + MovedAttribute("addinfourl", "urllib", "urllib.response"), +] +for attr in _urllib_response_moved_attributes: + setattr(Module_six_moves_urllib_response, attr.name, attr) +del attr + +Module_six_moves_urllib_response._moved_attributes = _urllib_response_moved_attributes + +_importer._add_module(Module_six_moves_urllib_response(__name__ + ".moves.urllib.response"), + "moves.urllib_response", "moves.urllib.response") + + +class Module_six_moves_urllib_robotparser(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_robotparser""" + + +_urllib_robotparser_moved_attributes = [ + MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser"), +] +for attr in _urllib_robotparser_moved_attributes: + setattr(Module_six_moves_urllib_robotparser, attr.name, attr) +del attr + +Module_six_moves_urllib_robotparser._moved_attributes = _urllib_robotparser_moved_attributes + +_importer._add_module(Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser"), + "moves.urllib_robotparser", "moves.urllib.robotparser") + + +class Module_six_moves_urllib(types.ModuleType): + + """Create a six.moves.urllib namespace that resembles the Python 3 namespace""" + __path__ = [] # mark as package + parse = _importer._get_module("moves.urllib_parse") + error = _importer._get_module("moves.urllib_error") + request = _importer._get_module("moves.urllib_request") + response = _importer._get_module("moves.urllib_response") + robotparser = _importer._get_module("moves.urllib_robotparser") + + def __dir__(self): + return ['parse', 'error', 'request', 'response', 'robotparser'] + +_importer._add_module(Module_six_moves_urllib(__name__ + ".moves.urllib"), + "moves.urllib") + + +def add_move(move): + """Add an item to six.moves.""" + setattr(_MovedItems, move.name, move) + + +def remove_move(name): + """Remove item from six.moves.""" + try: + delattr(_MovedItems, name) + except AttributeError: + try: + del moves.__dict__[name] + except KeyError: + raise AttributeError("no such move, %r" % (name,)) + + +if PY3: + _meth_func = "__func__" + _meth_self = "__self__" + + _func_closure = "__closure__" + _func_code = "__code__" + _func_defaults = "__defaults__" + _func_globals = "__globals__" +else: + _meth_func = "im_func" + _meth_self = "im_self" + + _func_closure = "func_closure" + _func_code = "func_code" + _func_defaults = "func_defaults" + _func_globals = "func_globals" + + +try: + advance_iterator = next +except NameError: + def advance_iterator(it): + return it.next() +next = advance_iterator + + +try: + callable = callable +except NameError: + def callable(obj): + return any("__call__" in klass.__dict__ for klass in type(obj).__mro__) + + +if PY3: + def get_unbound_function(unbound): + return unbound + + create_bound_method = types.MethodType + + def create_unbound_method(func, cls): + return func + + Iterator = object +else: + def get_unbound_function(unbound): + return unbound.im_func + + def create_bound_method(func, obj): + return types.MethodType(func, obj, obj.__class__) + + def create_unbound_method(func, cls): + return types.MethodType(func, None, cls) + + class Iterator(object): + + def next(self): + return type(self).__next__(self) + + callable = callable +_add_doc(get_unbound_function, + """Get the function out of a possibly unbound function""") + + +get_method_function = operator.attrgetter(_meth_func) +get_method_self = operator.attrgetter(_meth_self) +get_function_closure = operator.attrgetter(_func_closure) +get_function_code = operator.attrgetter(_func_code) +get_function_defaults = operator.attrgetter(_func_defaults) +get_function_globals = operator.attrgetter(_func_globals) + + +if PY3: + def iterkeys(d, **kw): + return iter(d.keys(**kw)) + + def itervalues(d, **kw): + return iter(d.values(**kw)) + + def iteritems(d, **kw): + return iter(d.items(**kw)) + + def iterlists(d, **kw): + return iter(d.lists(**kw)) + + viewkeys = operator.methodcaller("keys") + + viewvalues = operator.methodcaller("values") + + viewitems = operator.methodcaller("items") +else: + def iterkeys(d, **kw): + return d.iterkeys(**kw) + + def itervalues(d, **kw): + return d.itervalues(**kw) + + def iteritems(d, **kw): + return d.iteritems(**kw) + + def iterlists(d, **kw): + return d.iterlists(**kw) + + viewkeys = operator.methodcaller("viewkeys") + + viewvalues = operator.methodcaller("viewvalues") + + viewitems = operator.methodcaller("viewitems") + +_add_doc(iterkeys, "Return an iterator over the keys of a dictionary.") +_add_doc(itervalues, "Return an iterator over the values of a dictionary.") +_add_doc(iteritems, + "Return an iterator over the (key, value) pairs of a dictionary.") +_add_doc(iterlists, + "Return an iterator over the (key, [values]) pairs of a dictionary.") + + +if PY3: + def b(s): + return s.encode("latin-1") + + def u(s): + return s + unichr = chr + import struct + int2byte = struct.Struct(">B").pack + del struct + byte2int = operator.itemgetter(0) + indexbytes = operator.getitem + iterbytes = iter + import io + StringIO = io.StringIO + BytesIO = io.BytesIO + del io + _assertCountEqual = "assertCountEqual" + if sys.version_info[1] <= 1: + _assertRaisesRegex = "assertRaisesRegexp" + _assertRegex = "assertRegexpMatches" + _assertNotRegex = "assertNotRegexpMatches" + else: + _assertRaisesRegex = "assertRaisesRegex" + _assertRegex = "assertRegex" + _assertNotRegex = "assertNotRegex" +else: + def b(s): + return s + # Workaround for standalone backslash + + def u(s): + return unicode(s.replace(r'\\', r'\\\\'), "unicode_escape") + unichr = unichr + int2byte = chr + + def byte2int(bs): + return ord(bs[0]) + + def indexbytes(buf, i): + return ord(buf[i]) + iterbytes = functools.partial(itertools.imap, ord) + import StringIO + StringIO = BytesIO = StringIO.StringIO + _assertCountEqual = "assertItemsEqual" + _assertRaisesRegex = "assertRaisesRegexp" + _assertRegex = "assertRegexpMatches" + _assertNotRegex = "assertNotRegexpMatches" +_add_doc(b, """Byte literal""") +_add_doc(u, """Text literal""") + + +def assertCountEqual(self, *args, **kwargs): + return getattr(self, _assertCountEqual)(*args, **kwargs) + + +def assertRaisesRegex(self, *args, **kwargs): + return getattr(self, _assertRaisesRegex)(*args, **kwargs) + + +def assertRegex(self, *args, **kwargs): + return getattr(self, _assertRegex)(*args, **kwargs) + + +def assertNotRegex(self, *args, **kwargs): + return getattr(self, _assertNotRegex)(*args, **kwargs) + + +if PY3: + exec_ = getattr(moves.builtins, "exec") + + def reraise(tp, value, tb=None): + try: + if value is None: + value = tp() + if value.__traceback__ is not tb: + raise value.with_traceback(tb) + raise value + finally: + value = None + tb = None + +else: + def exec_(_code_, _globs_=None, _locs_=None): + """Execute code in a namespace.""" + if _globs_ is None: + frame = sys._getframe(1) + _globs_ = frame.f_globals + if _locs_ is None: + _locs_ = frame.f_locals + del frame + elif _locs_ is None: + _locs_ = _globs_ + exec("""exec _code_ in _globs_, _locs_""") + + exec_("""def reraise(tp, value, tb=None): + try: + raise tp, value, tb + finally: + tb = None +""") + + +if sys.version_info[:2] > (3,): + exec_("""def raise_from(value, from_value): + try: + raise value from from_value + finally: + value = None +""") +else: + def raise_from(value, from_value): + raise value + + +print_ = getattr(moves.builtins, "print", None) +if print_ is None: + def print_(*args, **kwargs): + """The new-style print function for Python 2.4 and 2.5.""" + fp = kwargs.pop("file", sys.stdout) + if fp is None: + return + + def write(data): + if not isinstance(data, basestring): + data = str(data) + # If the file has an encoding, encode unicode with it. + if (isinstance(fp, file) and + isinstance(data, unicode) and + fp.encoding is not None): + errors = getattr(fp, "errors", None) + if errors is None: + errors = "strict" + data = data.encode(fp.encoding, errors) + fp.write(data) + want_unicode = False + sep = kwargs.pop("sep", None) + if sep is not None: + if isinstance(sep, unicode): + want_unicode = True + elif not isinstance(sep, str): + raise TypeError("sep must be None or a string") + end = kwargs.pop("end", None) + if end is not None: + if isinstance(end, unicode): + want_unicode = True + elif not isinstance(end, str): + raise TypeError("end must be None or a string") + if kwargs: + raise TypeError("invalid keyword arguments to print()") + if not want_unicode: + for arg in args: + if isinstance(arg, unicode): + want_unicode = True + break + if want_unicode: + newline = unicode("\n") + space = unicode(" ") + else: + newline = "\n" + space = " " + if sep is None: + sep = space + if end is None: + end = newline + for i, arg in enumerate(args): + if i: + write(sep) + write(arg) + write(end) +if sys.version_info[:2] < (3, 3): + _print = print_ + + def print_(*args, **kwargs): + fp = kwargs.get("file", sys.stdout) + flush = kwargs.pop("flush", False) + _print(*args, **kwargs) + if flush and fp is not None: + fp.flush() + +_add_doc(reraise, """Reraise an exception.""") + +if sys.version_info[0:2] < (3, 4): + # This does exactly the same what the :func:`py3:functools.update_wrapper` + # function does on Python versions after 3.2. It sets the ``__wrapped__`` + # attribute on ``wrapper`` object and it doesn't raise an error if any of + # the attributes mentioned in ``assigned`` and ``updated`` are missing on + # ``wrapped`` object. + def _update_wrapper(wrapper, wrapped, + assigned=functools.WRAPPER_ASSIGNMENTS, + updated=functools.WRAPPER_UPDATES): + for attr in assigned: + try: + value = getattr(wrapped, attr) + except AttributeError: + continue + else: + setattr(wrapper, attr, value) + for attr in updated: + getattr(wrapper, attr).update(getattr(wrapped, attr, {})) + wrapper.__wrapped__ = wrapped + return wrapper + _update_wrapper.__doc__ = functools.update_wrapper.__doc__ + + def wraps(wrapped, assigned=functools.WRAPPER_ASSIGNMENTS, + updated=functools.WRAPPER_UPDATES): + return functools.partial(_update_wrapper, wrapped=wrapped, + assigned=assigned, updated=updated) + wraps.__doc__ = functools.wraps.__doc__ + +else: + wraps = functools.wraps + + +def with_metaclass(meta, *bases): + """Create a base class with a metaclass.""" + # This requires a bit of explanation: the basic idea is to make a dummy + # metaclass for one level of class instantiation that replaces itself with + # the actual metaclass. + class metaclass(type): + + def __new__(cls, name, this_bases, d): + if sys.version_info[:2] >= (3, 7): + # This version introduced PEP 560 that requires a bit + # of extra care (we mimic what is done by __build_class__). + resolved_bases = types.resolve_bases(bases) + if resolved_bases is not bases: + d['__orig_bases__'] = bases + else: + resolved_bases = bases + return meta(name, resolved_bases, d) + + @classmethod + def __prepare__(cls, name, this_bases): + return meta.__prepare__(name, bases) + return type.__new__(metaclass, 'temporary_class', (), {}) + + +def add_metaclass(metaclass): + """Class decorator for creating a class with a metaclass.""" + def wrapper(cls): + orig_vars = cls.__dict__.copy() + slots = orig_vars.get('__slots__') + if slots is not None: + if isinstance(slots, str): + slots = [slots] + for slots_var in slots: + orig_vars.pop(slots_var) + orig_vars.pop('__dict__', None) + orig_vars.pop('__weakref__', None) + if hasattr(cls, '__qualname__'): + orig_vars['__qualname__'] = cls.__qualname__ + return metaclass(cls.__name__, cls.__bases__, orig_vars) + return wrapper + + +def ensure_binary(s, encoding='utf-8', errors='strict'): + """Coerce **s** to six.binary_type. + + For Python 2: + - `unicode` -> encoded to `str` + - `str` -> `str` + + For Python 3: + - `str` -> encoded to `bytes` + - `bytes` -> `bytes` + """ + if isinstance(s, binary_type): + return s + if isinstance(s, text_type): + return s.encode(encoding, errors) + raise TypeError("not expecting type '%s'" % type(s)) + + +def ensure_str(s, encoding='utf-8', errors='strict'): + """Coerce *s* to `str`. + + For Python 2: + - `unicode` -> encoded to `str` + - `str` -> `str` + + For Python 3: + - `str` -> `str` + - `bytes` -> decoded to `str` + """ + # Optimization: Fast return for the common case. + if type(s) is str: + return s + if PY2 and isinstance(s, text_type): + return s.encode(encoding, errors) + elif PY3 and isinstance(s, binary_type): + return s.decode(encoding, errors) + elif not isinstance(s, (text_type, binary_type)): + raise TypeError("not expecting type '%s'" % type(s)) + return s + + +def ensure_text(s, encoding='utf-8', errors='strict'): + """Coerce *s* to six.text_type. + + For Python 2: + - `unicode` -> `unicode` + - `str` -> `unicode` + + For Python 3: + - `str` -> `str` + - `bytes` -> decoded to `str` + """ + if isinstance(s, binary_type): + return s.decode(encoding, errors) + elif isinstance(s, text_type): + return s + else: + raise TypeError("not expecting type '%s'" % type(s)) + + +def python_2_unicode_compatible(klass): + """ + A class decorator that defines __unicode__ and __str__ methods under Python 2. + Under Python 3 it does nothing. + + To support Python 2 and 3 with a single code base, define a __str__ method + returning text and apply this decorator to the class. + """ + if PY2: + if '__str__' not in klass.__dict__: + raise ValueError("@python_2_unicode_compatible cannot be applied " + "to %s because it doesn't define __str__()." % + klass.__name__) + klass.__unicode__ = klass.__str__ + klass.__str__ = lambda self: self.__unicode__().encode('utf-8') + return klass + + +# Complete the moves implementation. +# This code is at the end of this module to speed up module loading. +# Turn this module into a package. +__path__ = [] # required for PEP 302 and PEP 451 +__package__ = __name__ # see PEP 366 @ReservedAssignment +if globals().get("__spec__") is not None: + __spec__.submodule_search_locations = [] # PEP 451 @UndefinedVariable +# Remove other six meta path importers, since they cause problems. This can +# happen if six is removed from sys.modules and then reloaded. (Setuptools does +# this for some reason.) +if sys.meta_path: + for i, importer in enumerate(sys.meta_path): + # Here's some real nastiness: Another "instance" of the six module might + # be floating around. Therefore, we can't use isinstance() to check for + # the six meta path importer, since the other six instance will have + # inserted an importer with different class. + if (type(importer).__name__ == "_SixMetaPathImporter" and + importer.name == __name__): + del sys.meta_path[i] + break + del i, importer +# Finally, add the importer to the meta path import hook. +sys.meta_path.append(_importer) diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/__init__.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/__init__.py new file mode 100644 index 0000000000..a286bfe0b3 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/__init__.py @@ -0,0 +1,3 @@ +from .server import WebTestHttpd, WebTestServer, Router # noqa: F401 +from .request import Request # noqa: F401 +from .response import Response # noqa: F401 diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/cgi/LICENSE b/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/cgi/LICENSE new file mode 100644 index 0000000000..f26bcf4d2d --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/cgi/LICENSE @@ -0,0 +1,279 @@ +A. HISTORY OF THE SOFTWARE +========================== + +Python was created in the early 1990s by Guido van Rossum at Stichting +Mathematisch Centrum (CWI, see https://www.cwi.nl) in the Netherlands +as a successor of a language called ABC. Guido remains Python's +principal author, although it includes many contributions from others. + +In 1995, Guido continued his work on Python at the Corporation for +National Research Initiatives (CNRI, see https://www.cnri.reston.va.us) +in Reston, Virginia where he released several versions of the +software. + +In May 2000, Guido and the Python core development team moved to +BeOpen.com to form the BeOpen PythonLabs team. In October of the same +year, the PythonLabs team moved to Digital Creations, which became +Zope Corporation. In 2001, the Python Software Foundation (PSF, see +https://www.python.org/psf/) was formed, a non-profit organization +created specifically to own Python-related Intellectual Property. +Zope Corporation was a sponsoring member of the PSF. + +All Python releases are Open Source (see https://opensource.org for +the Open Source Definition). Historically, most, but not all, Python +releases have also been GPL-compatible; the table below summarizes +the various releases. + + Release Derived Year Owner GPL- + from compatible? (1) + + 0.9.0 thru 1.2 1991-1995 CWI yes + 1.3 thru 1.5.2 1.2 1995-1999 CNRI yes + 1.6 1.5.2 2000 CNRI no + 2.0 1.6 2000 BeOpen.com no + 1.6.1 1.6 2001 CNRI yes (2) + 2.1 2.0+1.6.1 2001 PSF no + 2.0.1 2.0+1.6.1 2001 PSF yes + 2.1.1 2.1+2.0.1 2001 PSF yes + 2.1.2 2.1.1 2002 PSF yes + 2.1.3 2.1.2 2002 PSF yes + 2.2 and above 2.1.1 2001-now PSF yes + +Footnotes: + +(1) GPL-compatible doesn't mean that we're distributing Python under + the GPL. All Python licenses, unlike the GPL, let you distribute + a modified version without making your changes open source. The + GPL-compatible licenses make it possible to combine Python with + other software that is released under the GPL; the others don't. + +(2) According to Richard Stallman, 1.6.1 is not GPL-compatible, + because its license has a choice of law clause. According to + CNRI, however, Stallman's lawyer has told CNRI's lawyer that 1.6.1 + is "not incompatible" with the GPL. + +Thanks to the many outside volunteers who have worked under Guido's +direction to make these releases possible. + + +B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON +=============================================================== + +Python software and documentation are licensed under the +Python Software Foundation License Version 2. + +Starting with Python 3.8.6, examples, recipes, and other code in +the documentation are dual licensed under the PSF License Version 2 +and the Zero-Clause BSD license. + +Some software incorporated into Python is under different licenses. +The licenses are listed with code falling under that license. + + +PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 +-------------------------------------------- + +1. This LICENSE AGREEMENT is between the Python Software Foundation +("PSF"), and the Individual or Organization ("Licensee") accessing and +otherwise using this software ("Python") in source or binary form and +its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, PSF hereby +grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, +analyze, test, perform and/or display publicly, prepare derivative works, +distribute, and otherwise use Python alone or in any derivative version, +provided, however, that PSF's License Agreement and PSF's notice of copyright, +i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, +2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023 Python Software Foundation; +All Rights Reserved" are retained in Python alone or in any derivative version +prepared by Licensee. + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python. + +4. PSF is making Python available to Licensee on an "AS IS" +basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. Nothing in this License Agreement shall be deemed to create any +relationship of agency, partnership, or joint venture between PSF and +Licensee. This License Agreement does not grant permission to use PSF +trademarks or trade name in a trademark sense to endorse or promote +products or services of Licensee, or any third party. + +8. By copying, installing or otherwise using Python, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. + + +BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0 +------------------------------------------- + +BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1 + +1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an +office at 160 Saratoga Avenue, Santa Clara, CA 95051, and the +Individual or Organization ("Licensee") accessing and otherwise using +this software in source or binary form and its associated +documentation ("the Software"). + +2. Subject to the terms and conditions of this BeOpen Python License +Agreement, BeOpen hereby grants Licensee a non-exclusive, +royalty-free, world-wide license to reproduce, analyze, test, perform +and/or display publicly, prepare derivative works, distribute, and +otherwise use the Software alone or in any derivative version, +provided, however, that the BeOpen Python License is retained in the +Software, alone or in any derivative version prepared by Licensee. + +3. BeOpen is making the Software available to Licensee on an "AS IS" +basis. BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE +SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS +AS A RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY +DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +5. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +6. This License Agreement shall be governed by and interpreted in all +respects by the law of the State of California, excluding conflict of +law provisions. Nothing in this License Agreement shall be deemed to +create any relationship of agency, partnership, or joint venture +between BeOpen and Licensee. This License Agreement does not grant +permission to use BeOpen trademarks or trade names in a trademark +sense to endorse or promote products or services of Licensee, or any +third party. As an exception, the "BeOpen Python" logos available at +http://www.pythonlabs.com/logos.html may be used according to the +permissions granted on that web page. + +7. By copying, installing or otherwise using the software, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. + + +CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1 +--------------------------------------- + +1. This LICENSE AGREEMENT is between the Corporation for National +Research Initiatives, having an office at 1895 Preston White Drive, +Reston, VA 20191 ("CNRI"), and the Individual or Organization +("Licensee") accessing and otherwise using Python 1.6.1 software in +source or binary form and its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, CNRI +hereby grants Licensee a nonexclusive, royalty-free, world-wide +license to reproduce, analyze, test, perform and/or display publicly, +prepare derivative works, distribute, and otherwise use Python 1.6.1 +alone or in any derivative version, provided, however, that CNRI's +License Agreement and CNRI's notice of copyright, i.e., "Copyright (c) +1995-2001 Corporation for National Research Initiatives; All Rights +Reserved" are retained in Python 1.6.1 alone or in any derivative +version prepared by Licensee. Alternately, in lieu of CNRI's License +Agreement, Licensee may substitute the following text (omitting the +quotes): "Python 1.6.1 is made available subject to the terms and +conditions in CNRI's License Agreement. This Agreement together with +Python 1.6.1 may be located on the internet using the following +unique, persistent identifier (known as a handle): 1895.22/1013. This +Agreement may also be obtained from a proxy server on the internet +using the following URL: http://hdl.handle.net/1895.22/1013". + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python 1.6.1 or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python 1.6.1. + +4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS" +basis. CNRI MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, CNRI MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 1.6.1 WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +1.6.1 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. This License Agreement shall be governed by the federal +intellectual property law of the United States, including without +limitation the federal copyright law, and, to the extent such +U.S. federal law does not apply, by the law of the Commonwealth of +Virginia, excluding Virginia's conflict of law provisions. +Notwithstanding the foregoing, with regard to derivative works based +on Python 1.6.1 that incorporate non-separable material that was +previously distributed under the GNU General Public License (GPL), the +law of the Commonwealth of Virginia shall govern this License +Agreement only as to issues arising under or with respect to +Paragraphs 4, 5, and 7 of this License Agreement. Nothing in this +License Agreement shall be deemed to create any relationship of +agency, partnership, or joint venture between CNRI and Licensee. This +License Agreement does not grant permission to use CNRI trademarks or +trade name in a trademark sense to endorse or promote products or +services of Licensee, or any third party. + +8. By clicking on the "ACCEPT" button where indicated, or by copying, +installing or otherwise using Python 1.6.1, Licensee agrees to be +bound by the terms and conditions of this License Agreement. + + ACCEPT + + +CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2 +-------------------------------------------------- + +Copyright (c) 1991 - 1995, Stichting Mathematisch Centrum Amsterdam, +The Netherlands. All rights reserved. + +Permission to use, copy, modify, and distribute this software and its +documentation for any purpose and without fee is hereby granted, +provided that the above copyright notice appear in all copies and that +both that copyright notice and this permission notice appear in +supporting documentation, and that the name of Stichting Mathematisch +Centrum or CWI not be used in advertising or publicity pertaining to +distribution of the software without specific, written prior +permission. + +STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO +THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE +FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT +OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +ZERO-CLAUSE BSD LICENSE FOR CODE IN THE PYTHON DOCUMENTATION +---------------------------------------------------------------------- + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/cgi/__init__.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/cgi/__init__.py new file mode 100644 index 0000000000..940f172771 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/cgi/__init__.py @@ -0,0 +1 @@ +from .cgi import * diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/cgi/cgi.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/cgi/cgi.py new file mode 100755 index 0000000000..297468efd5 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/cgi/cgi.py @@ -0,0 +1,1014 @@ +#! /usr/local/bin/python +# mypy: ignore-errors + +# NOTE: the above "/usr/local/bin/python" is NOT a mistake. It is +# intentionally NOT "/usr/bin/env python". On many systems +# (e.g. Solaris), /usr/local/bin is not in $PATH as passed to CGI +# scripts, and /usr/local/bin is the default directory where Python is +# installed, so /usr/bin/env would be unable to find python. Granted, +# binary installations by Linux vendors often install Python in +# /usr/bin. So let those vendors patch cgi.py to match their choice +# of installation. + +"""Support module for CGI (Common Gateway Interface) scripts. + +This module defines a number of utilities for use by CGI scripts +written in Python. + +The global variable maxlen can be set to an integer indicating the maximum size +of a POST request. POST requests larger than this size will result in a +ValueError being raised during parsing. The default value of this variable is 0, +meaning the request size is unlimited. +""" + +# History +# ------- +# +# Michael McLay started this module. Steve Majewski changed the +# interface to SvFormContentDict and FormContentDict. The multipart +# parsing was inspired by code submitted by Andreas Paepcke. Guido van +# Rossum rewrote, reformatted and documented the module and is currently +# responsible for its maintenance. +# + +__version__ = "2.6" + + +# Imports +# ======= + +from io import StringIO, BytesIO, TextIOWrapper +from collections.abc import Mapping +import sys +import os +import urllib.parse +from email.parser import FeedParser +from email.message import Message +import html +import locale +import tempfile +import warnings + +__all__ = ["MiniFieldStorage", "FieldStorage", "parse", "parse_multipart", + "parse_header", "test", "print_exception", "print_environ", + "print_form", "print_directory", "print_arguments", + "print_environ_usage"] + + +# Logging support +# =============== + +logfile = "" # Filename to log to, if not empty +logfp = None # File object to log to, if not None + +def initlog(*allargs): + """Write a log message, if there is a log file. + + Even though this function is called initlog(), you should always + use log(); log is a variable that is set either to initlog + (initially), to dolog (once the log file has been opened), or to + nolog (when logging is disabled). + + The first argument is a format string; the remaining arguments (if + any) are arguments to the % operator, so e.g. + log("%s: %s", "a", "b") + will write "a: b" to the log file, followed by a newline. + + If the global logfp is not None, it should be a file object to + which log data is written. + + If the global logfp is None, the global logfile may be a string + giving a filename to open, in append mode. This file should be + world writable!!! If the file can't be opened, logging is + silently disabled (since there is no safe place where we could + send an error message). + + """ + global log, logfile, logfp + warnings.warn("cgi.log() is deprecated as of 3.10. Use logging instead", + DeprecationWarning, stacklevel=2) + if logfile and not logfp: + try: + kwargs = {} + if sys.version_info[:2] >= (3, 10): + kwargs["encoding"] = "locale" + logfp = open(logfile, "a", **kwargs) + except OSError: + pass + if not logfp: + log = nolog + else: + log = dolog + log(*allargs) + +def dolog(fmt, *args): + """Write a log message to the log file. See initlog() for docs.""" + logfp.write(fmt%args + "\n") + +def nolog(*allargs): + """Dummy function, assigned to log when logging is disabled.""" + pass + +def closelog(): + """Close the log file.""" + global log, logfile, logfp + logfile = '' + if logfp: + logfp.close() + logfp = None + log = initlog + +log = initlog # The current logging function + + +# Parsing functions +# ================= + +# Maximum input we will accept when REQUEST_METHOD is POST +# 0 ==> unlimited input +maxlen = 0 + +def parse(fp=None, environ=os.environ, keep_blank_values=0, + strict_parsing=0, separator='&'): + """Parse a query in the environment or from a file (default stdin) + + Arguments, all optional: + + fp : file pointer; default: sys.stdin.buffer + + environ : environment dictionary; default: os.environ + + keep_blank_values: flag indicating whether blank values in + percent-encoded forms should be treated as blank strings. + A true value indicates that blanks should be retained as + blank strings. The default false value indicates that + blank values are to be ignored and treated as if they were + not included. + + strict_parsing: flag indicating what to do with parsing errors. + If false (the default), errors are silently ignored. + If true, errors raise a ValueError exception. + + separator: str. The symbol to use for separating the query arguments. + Defaults to &. + """ + if fp is None: + fp = sys.stdin + + # field keys and values (except for files) are returned as strings + # an encoding is required to decode the bytes read from self.fp + if hasattr(fp,'encoding'): + encoding = fp.encoding + else: + encoding = 'latin-1' + + # fp.read() must return bytes + if isinstance(fp, TextIOWrapper): + fp = fp.buffer + + if not 'REQUEST_METHOD' in environ: + environ['REQUEST_METHOD'] = 'GET' # For testing stand-alone + if environ['REQUEST_METHOD'] == 'POST': + ctype, pdict = parse_header(environ['CONTENT_TYPE']) + if ctype == 'multipart/form-data': + return parse_multipart(fp, pdict, separator=separator) + elif ctype == 'application/x-www-form-urlencoded': + clength = int(environ['CONTENT_LENGTH']) + if maxlen and clength > maxlen: + raise ValueError('Maximum content length exceeded') + qs = fp.read(clength).decode(encoding) + else: + qs = '' # Unknown content-type + if 'QUERY_STRING' in environ: + if qs: qs = qs + '&' + qs = qs + environ['QUERY_STRING'] + elif sys.argv[1:]: + if qs: qs = qs + '&' + qs = qs + sys.argv[1] + environ['QUERY_STRING'] = qs # XXX Shouldn't, really + elif 'QUERY_STRING' in environ: + qs = environ['QUERY_STRING'] + else: + if sys.argv[1:]: + qs = sys.argv[1] + else: + qs = "" + environ['QUERY_STRING'] = qs # XXX Shouldn't, really + return urllib.parse.parse_qs(qs, keep_blank_values, strict_parsing, + encoding=encoding, separator=separator) + + +def parse_multipart(fp, pdict, encoding="utf-8", errors="replace", separator='&'): + """Parse multipart input. + + Arguments: + fp : input file + pdict: dictionary containing other parameters of content-type header + encoding, errors: request encoding and error handler, passed to + FieldStorage + + Returns a dictionary just like parse_qs(): keys are the field names, each + value is a list of values for that field. For non-file fields, the value + is a list of strings. + """ + # RFC 2046, Section 5.1 : The "multipart" boundary delimiters are always + # represented as 7bit US-ASCII. + boundary = pdict['boundary'].decode('ascii') + ctype = "multipart/form-data; boundary={}".format(boundary) + headers = Message() + headers.set_type(ctype) + try: + headers['Content-Length'] = pdict['CONTENT-LENGTH'] + except KeyError: + pass + fs = FieldStorage(fp, headers=headers, encoding=encoding, errors=errors, + environ={'REQUEST_METHOD': 'POST'}, separator=separator) + return {k: fs.getlist(k) for k in fs} + +def _parseparam(s): + while s[:1] == ';': + s = s[1:] + end = s.find(';') + while end > 0 and (s.count('"', 0, end) - s.count('\\"', 0, end)) % 2: + end = s.find(';', end + 1) + if end < 0: + end = len(s) + f = s[:end] + yield f.strip() + s = s[end:] + +def parse_header(line): + """Parse a Content-type like header. + + Return the main content-type and a dictionary of options. + + """ + parts = _parseparam(';' + line) + key = parts.__next__() + pdict = {} + for p in parts: + i = p.find('=') + if i >= 0: + name = p[:i].strip().lower() + value = p[i+1:].strip() + if len(value) >= 2 and value[0] == value[-1] == '"': + value = value[1:-1] + value = value.replace('\\\\', '\\').replace('\\"', '"') + pdict[name] = value + return key, pdict + + +# Classes for field storage +# ========================= + +class MiniFieldStorage: + + """Like FieldStorage, for use when no file uploads are possible.""" + + # Dummy attributes + filename = None + list = None + type = None + file = None + type_options = {} + disposition = None + disposition_options = {} + headers = {} + + def __init__(self, name, value): + """Constructor from field name and value.""" + self.name = name + self.value = value + # self.file = StringIO(value) + + def __repr__(self): + """Return printable representation.""" + return "MiniFieldStorage(%r, %r)" % (self.name, self.value) + + +class FieldStorage: + + """Store a sequence of fields, reading multipart/form-data. + + This class provides naming, typing, files stored on disk, and + more. At the top level, it is accessible like a dictionary, whose + keys are the field names. (Note: None can occur as a field name.) + The items are either a Python list (if there's multiple values) or + another FieldStorage or MiniFieldStorage object. If it's a single + object, it has the following attributes: + + name: the field name, if specified; otherwise None + + filename: the filename, if specified; otherwise None; this is the + client side filename, *not* the file name on which it is + stored (that's a temporary file you don't deal with) + + value: the value as a *string*; for file uploads, this + transparently reads the file every time you request the value + and returns *bytes* + + file: the file(-like) object from which you can read the data *as + bytes* ; None if the data is stored a simple string + + type: the content-type, or None if not specified + + type_options: dictionary of options specified on the content-type + line + + disposition: content-disposition, or None if not specified + + disposition_options: dictionary of corresponding options + + headers: a dictionary(-like) object (sometimes email.message.Message or a + subclass thereof) containing *all* headers + + The class is subclassable, mostly for the purpose of overriding + the make_file() method, which is called internally to come up with + a file open for reading and writing. This makes it possible to + override the default choice of storing all files in a temporary + directory and unlinking them as soon as they have been opened. + + """ + def __init__(self, fp=None, headers=None, outerboundary=b'', + environ=os.environ, keep_blank_values=0, strict_parsing=0, + limit=None, encoding='utf-8', errors='replace', + max_num_fields=None, separator='&'): + """Constructor. Read multipart/* until last part. + + Arguments, all optional: + + fp : file pointer; default: sys.stdin.buffer + (not used when the request method is GET) + Can be : + 1. a TextIOWrapper object + 2. an object whose read() and readline() methods return bytes + + headers : header dictionary-like object; default: + taken from environ as per CGI spec + + outerboundary : terminating multipart boundary + (for internal use only) + + environ : environment dictionary; default: os.environ + + keep_blank_values: flag indicating whether blank values in + percent-encoded forms should be treated as blank strings. + A true value indicates that blanks should be retained as + blank strings. The default false value indicates that + blank values are to be ignored and treated as if they were + not included. + + strict_parsing: flag indicating what to do with parsing errors. + If false (the default), errors are silently ignored. + If true, errors raise a ValueError exception. + + limit : used internally to read parts of multipart/form-data forms, + to exit from the reading loop when reached. It is the difference + between the form content-length and the number of bytes already + read + + encoding, errors : the encoding and error handler used to decode the + binary stream to strings. Must be the same as the charset defined + for the page sending the form (content-type : meta http-equiv or + header) + + max_num_fields: int. If set, then __init__ throws a ValueError + if there are more than n fields read by parse_qsl(). + + """ + method = 'GET' + self.keep_blank_values = keep_blank_values + self.strict_parsing = strict_parsing + self.max_num_fields = max_num_fields + self.separator = separator + if 'REQUEST_METHOD' in environ: + method = environ['REQUEST_METHOD'].upper() + self.qs_on_post = None + if method == 'GET' or method == 'HEAD': + if 'QUERY_STRING' in environ: + qs = environ['QUERY_STRING'] + elif sys.argv[1:]: + qs = sys.argv[1] + else: + qs = "" + qs = qs.encode(locale.getpreferredencoding(), 'surrogateescape') + fp = BytesIO(qs) + if headers is None: + headers = {'content-type': + "application/x-www-form-urlencoded"} + if headers is None: + headers = {} + if method == 'POST': + # Set default content-type for POST to what's traditional + headers['content-type'] = "application/x-www-form-urlencoded" + if 'CONTENT_TYPE' in environ: + headers['content-type'] = environ['CONTENT_TYPE'] + if 'QUERY_STRING' in environ: + self.qs_on_post = environ['QUERY_STRING'] + if 'CONTENT_LENGTH' in environ: + headers['content-length'] = environ['CONTENT_LENGTH'] + else: + if not (isinstance(headers, (Mapping, Message))): + raise TypeError("headers must be mapping or an instance of " + "email.message.Message") + self.headers = headers + if fp is None: + self.fp = sys.stdin.buffer + # self.fp.read() must return bytes + elif isinstance(fp, TextIOWrapper): + self.fp = fp.buffer + else: + if not (hasattr(fp, 'read') and hasattr(fp, 'readline')): + raise TypeError("fp must be file pointer") + self.fp = fp + + self.encoding = encoding + self.errors = errors + + if not isinstance(outerboundary, bytes): + raise TypeError('outerboundary must be bytes, not %s' + % type(outerboundary).__name__) + self.outerboundary = outerboundary + + self.bytes_read = 0 + self.limit = limit + + # Process content-disposition header + cdisp, pdict = "", {} + if 'content-disposition' in self.headers: + cdisp, pdict = parse_header(self.headers['content-disposition']) + self.disposition = cdisp + self.disposition_options = pdict + self.name = None + if 'name' in pdict: + self.name = pdict['name'] + self.filename = None + if 'filename' in pdict: + self.filename = pdict['filename'] + self._binary_file = self.filename is not None + + # Process content-type header + # + # Honor any existing content-type header. But if there is no + # content-type header, use some sensible defaults. Assume + # outerboundary is "" at the outer level, but something non-false + # inside a multi-part. The default for an inner part is text/plain, + # but for an outer part it should be urlencoded. This should catch + # bogus clients which erroneously forget to include a content-type + # header. + # + # See below for what we do if there does exist a content-type header, + # but it happens to be something we don't understand. + if 'content-type' in self.headers: + ctype, pdict = parse_header(self.headers['content-type']) + elif self.outerboundary or method != 'POST': + ctype, pdict = "text/plain", {} + else: + ctype, pdict = 'application/x-www-form-urlencoded', {} + self.type = ctype + self.type_options = pdict + if 'boundary' in pdict: + self.innerboundary = pdict['boundary'].encode(self.encoding, + self.errors) + else: + self.innerboundary = b"" + + clen = -1 + if 'content-length' in self.headers: + try: + clen = int(self.headers['content-length']) + except ValueError: + pass + if maxlen and clen > maxlen: + raise ValueError('Maximum content length exceeded') + self.length = clen + if self.limit is None and clen >= 0: + self.limit = clen + + self.list = self.file = None + self.done = 0 + if ctype == 'application/x-www-form-urlencoded': + self.read_urlencoded() + elif ctype[:10] == 'multipart/': + self.read_multi(environ, keep_blank_values, strict_parsing) + else: + self.read_single() + + def __del__(self): + try: + self.file.close() + except AttributeError: + pass + + def __enter__(self): + return self + + def __exit__(self, *args): + self.file.close() + + def __repr__(self): + """Return a printable representation.""" + return "FieldStorage(%r, %r, %r)" % ( + self.name, self.filename, self.value) + + def __iter__(self): + return iter(self.keys()) + + def __getattr__(self, name): + if name != 'value': + raise AttributeError(name) + if self.file: + self.file.seek(0) + value = self.file.read() + self.file.seek(0) + elif self.list is not None: + value = self.list + else: + value = None + return value + + def __getitem__(self, key): + """Dictionary style indexing.""" + if self.list is None: + raise TypeError("not indexable") + found = [] + for item in self.list: + if item.name == key: found.append(item) + if not found: + raise KeyError(key) + if len(found) == 1: + return found[0] + else: + return found + + def getvalue(self, key, default=None): + """Dictionary style get() method, including 'value' lookup.""" + if key in self: + value = self[key] + if isinstance(value, list): + return [x.value for x in value] + else: + return value.value + else: + return default + + def getfirst(self, key, default=None): + """ Return the first value received.""" + if key in self: + value = self[key] + if isinstance(value, list): + return value[0].value + else: + return value.value + else: + return default + + def getlist(self, key): + """ Return list of received values.""" + if key in self: + value = self[key] + if isinstance(value, list): + return [x.value for x in value] + else: + return [value.value] + else: + return [] + + def keys(self): + """Dictionary style keys() method.""" + if self.list is None: + raise TypeError("not indexable") + return list(set(item.name for item in self.list)) + + def __contains__(self, key): + """Dictionary style __contains__ method.""" + if self.list is None: + raise TypeError("not indexable") + return any(item.name == key for item in self.list) + + def __len__(self): + """Dictionary style len(x) support.""" + return len(self.keys()) + + def __bool__(self): + if self.list is None: + raise TypeError("Cannot be converted to bool.") + return bool(self.list) + + def read_urlencoded(self): + """Internal: read data in query string format.""" + qs = self.fp.read(self.length) + if not isinstance(qs, bytes): + raise ValueError("%s should return bytes, got %s" \ + % (self.fp, type(qs).__name__)) + qs = qs.decode(self.encoding, self.errors) + if self.qs_on_post: + qs += '&' + self.qs_on_post + query = urllib.parse.parse_qsl( + qs, self.keep_blank_values, self.strict_parsing, + encoding=self.encoding, errors=self.errors, + max_num_fields=self.max_num_fields, separator=self.separator) + self.list = [MiniFieldStorage(key, value) for key, value in query] + self.skip_lines() + + FieldStorageClass = None + + def read_multi(self, environ, keep_blank_values, strict_parsing): + """Internal: read a part that is itself multipart.""" + ib = self.innerboundary + if not valid_boundary(ib): + raise ValueError('Invalid boundary in multipart form: %r' % (ib,)) + self.list = [] + if self.qs_on_post: + query = urllib.parse.parse_qsl( + self.qs_on_post, self.keep_blank_values, self.strict_parsing, + encoding=self.encoding, errors=self.errors, + max_num_fields=self.max_num_fields, separator=self.separator) + self.list.extend(MiniFieldStorage(key, value) for key, value in query) + + klass = self.FieldStorageClass or self.__class__ + first_line = self.fp.readline() # bytes + if not isinstance(first_line, bytes): + raise ValueError("%s should return bytes, got %s" \ + % (self.fp, type(first_line).__name__)) + self.bytes_read += len(first_line) + + # Ensure that we consume the file until we've hit our inner boundary + while (first_line.strip() != (b"--" + self.innerboundary) and + first_line): + first_line = self.fp.readline() + self.bytes_read += len(first_line) + + # Propagate max_num_fields into the sub class appropriately + max_num_fields = self.max_num_fields + if max_num_fields is not None: + max_num_fields -= len(self.list) + + while True: + parser = FeedParser() + hdr_text = b"" + while True: + data = self.fp.readline() + hdr_text += data + if not data.strip(): + break + if not hdr_text: + break + # parser takes strings, not bytes + self.bytes_read += len(hdr_text) + parser.feed(hdr_text.decode(self.encoding, self.errors)) + headers = parser.close() + + # Some clients add Content-Length for part headers, ignore them + if 'content-length' in headers: + del headers['content-length'] + + limit = None if self.limit is None \ + else self.limit - self.bytes_read + part = klass(self.fp, headers, ib, environ, keep_blank_values, + strict_parsing, limit, + self.encoding, self.errors, max_num_fields, self.separator) + + if max_num_fields is not None: + max_num_fields -= 1 + if part.list: + max_num_fields -= len(part.list) + if max_num_fields < 0: + raise ValueError('Max number of fields exceeded') + + self.bytes_read += part.bytes_read + self.list.append(part) + if part.done or self.bytes_read >= self.length > 0: + break + self.skip_lines() + + def read_single(self): + """Internal: read an atomic part.""" + if self.length >= 0: + self.read_binary() + self.skip_lines() + else: + self.read_lines() + self.file.seek(0) + + bufsize = 8*1024 # I/O buffering size for copy to file + + def read_binary(self): + """Internal: read binary data.""" + self.file = self.make_file() + todo = self.length + if todo >= 0: + while todo > 0: + data = self.fp.read(min(todo, self.bufsize)) # bytes + if not isinstance(data, bytes): + raise ValueError("%s should return bytes, got %s" + % (self.fp, type(data).__name__)) + self.bytes_read += len(data) + if not data: + self.done = -1 + break + self.file.write(data) + todo = todo - len(data) + + def read_lines(self): + """Internal: read lines until EOF or outerboundary.""" + if self._binary_file: + self.file = self.__file = BytesIO() # store data as bytes for files + else: + self.file = self.__file = StringIO() # as strings for other fields + if self.outerboundary: + self.read_lines_to_outerboundary() + else: + self.read_lines_to_eof() + + def __write(self, line): + """line is always bytes, not string""" + if self.__file is not None: + if self.__file.tell() + len(line) > 1000: + self.file = self.make_file() + data = self.__file.getvalue() + self.file.write(data) + self.__file = None + if self._binary_file: + # keep bytes + self.file.write(line) + else: + # decode to string + self.file.write(line.decode(self.encoding, self.errors)) + + def read_lines_to_eof(self): + """Internal: read lines until EOF.""" + while 1: + line = self.fp.readline(1<<16) # bytes + self.bytes_read += len(line) + if not line: + self.done = -1 + break + self.__write(line) + + def read_lines_to_outerboundary(self): + """Internal: read lines until outerboundary. + Data is read as bytes: boundaries and line ends must be converted + to bytes for comparisons. + """ + next_boundary = b"--" + self.outerboundary + last_boundary = next_boundary + b"--" + delim = b"" + last_line_lfend = True + _read = 0 + while 1: + + if self.limit is not None and 0 <= self.limit <= _read: + break + line = self.fp.readline(1<<16) # bytes + self.bytes_read += len(line) + _read += len(line) + if not line: + self.done = -1 + break + if delim == b"\r": + line = delim + line + delim = b"" + if line.startswith(b"--") and last_line_lfend: + strippedline = line.rstrip() + if strippedline == next_boundary: + break + if strippedline == last_boundary: + self.done = 1 + break + odelim = delim + if line.endswith(b"\r\n"): + delim = b"\r\n" + line = line[:-2] + last_line_lfend = True + elif line.endswith(b"\n"): + delim = b"\n" + line = line[:-1] + last_line_lfend = True + elif line.endswith(b"\r"): + # We may interrupt \r\n sequences if they span the 2**16 + # byte boundary + delim = b"\r" + line = line[:-1] + last_line_lfend = False + else: + delim = b"" + last_line_lfend = False + self.__write(odelim + line) + + def skip_lines(self): + """Internal: skip lines until outer boundary if defined.""" + if not self.outerboundary or self.done: + return + next_boundary = b"--" + self.outerboundary + last_boundary = next_boundary + b"--" + last_line_lfend = True + while True: + line = self.fp.readline(1<<16) + self.bytes_read += len(line) + if not line: + self.done = -1 + break + if line.endswith(b"--") and last_line_lfend: + strippedline = line.strip() + if strippedline == next_boundary: + break + if strippedline == last_boundary: + self.done = 1 + break + last_line_lfend = line.endswith(b'\n') + + def make_file(self): + """Overridable: return a readable & writable file. + + The file will be used as follows: + - data is written to it + - seek(0) + - data is read from it + + The file is opened in binary mode for files, in text mode + for other fields + + This version opens a temporary file for reading and writing, + and immediately deletes (unlinks) it. The trick (on Unix!) is + that the file can still be used, but it can't be opened by + another process, and it will automatically be deleted when it + is closed or when the current process terminates. + + If you want a more permanent file, you derive a class which + overrides this method. If you want a visible temporary file + that is nevertheless automatically deleted when the script + terminates, try defining a __del__ method in a derived class + which unlinks the temporary files you have created. + + """ + if self._binary_file: + return tempfile.TemporaryFile("wb+") + else: + return tempfile.TemporaryFile("w+", + encoding=self.encoding, newline = '\n') + + +# Test/debug code +# =============== + +def test(environ=os.environ): + """Robust test CGI script, usable as main program. + + Write minimal HTTP headers and dump all information provided to + the script in HTML form. + + """ + print("Content-type: text/html") + print() + sys.stderr = sys.stdout + try: + form = FieldStorage() # Replace with other classes to test those + print_directory() + print_arguments() + print_form(form) + print_environ(environ) + print_environ_usage() + def f(): + exec("testing print_exception() -- italics?") + def g(f=f): + f() + print("

What follows is a test, not an actual exception:

") + g() + except: + print_exception() + + print("

Second try with a small maxlen...

") + + global maxlen + maxlen = 50 + try: + form = FieldStorage() # Replace with other classes to test those + print_directory() + print_arguments() + print_form(form) + print_environ(environ) + except: + print_exception() + +def print_exception(type=None, value=None, tb=None, limit=None): + if type is None: + type, value, tb = sys.exc_info() + import traceback + print() + print("

Traceback (most recent call last):

") + list = traceback.format_tb(tb, limit) + \ + traceback.format_exception_only(type, value) + print("
%s%s
" % ( + html.escape("".join(list[:-1])), + html.escape(list[-1]), + )) + del tb + +def print_environ(environ=os.environ): + """Dump the shell environment as HTML.""" + keys = sorted(environ.keys()) + print() + print("

Shell Environment:

") + print("
") + for key in keys: + print("
", html.escape(key), "
", html.escape(environ[key])) + print("
") + print() + +def print_form(form): + """Dump the contents of a form as HTML.""" + keys = sorted(form.keys()) + print() + print("

Form Contents:

") + if not keys: + print("

No form fields.") + print("

") + for key in keys: + print("
" + html.escape(key) + ":", end=' ') + value = form[key] + print("" + html.escape(repr(type(value))) + "") + print("
" + html.escape(repr(value))) + print("
") + print() + +def print_directory(): + """Dump the current directory as HTML.""" + print() + print("

Current Working Directory:

") + try: + pwd = os.getcwd() + except OSError as msg: + print("OSError:", html.escape(str(msg))) + else: + print(html.escape(pwd)) + print() + +def print_arguments(): + print() + print("

Command Line Arguments:

") + print() + print(sys.argv) + print() + +def print_environ_usage(): + """Dump a list of environment variables used by CGI as HTML.""" + print(""" +

These environment variables could have been set:

+
    +
  • AUTH_TYPE +
  • CONTENT_LENGTH +
  • CONTENT_TYPE +
  • DATE_GMT +
  • DATE_LOCAL +
  • DOCUMENT_NAME +
  • DOCUMENT_ROOT +
  • DOCUMENT_URI +
  • GATEWAY_INTERFACE +
  • LAST_MODIFIED +
  • PATH +
  • PATH_INFO +
  • PATH_TRANSLATED +
  • QUERY_STRING +
  • REMOTE_ADDR +
  • REMOTE_HOST +
  • REMOTE_IDENT +
  • REMOTE_USER +
  • REQUEST_METHOD +
  • SCRIPT_NAME +
  • SERVER_NAME +
  • SERVER_PORT +
  • SERVER_PROTOCOL +
  • SERVER_ROOT +
  • SERVER_SOFTWARE +
+In addition, HTTP headers sent by the server may be passed in the +environment as well. Here are some common variable names: +
    +
  • HTTP_ACCEPT +
  • HTTP_CONNECTION +
  • HTTP_HOST +
  • HTTP_PRAGMA +
  • HTTP_REFERER +
  • HTTP_USER_AGENT +
+""") + + +# Utilities +# ========= + +def valid_boundary(s): + import re + if isinstance(s, bytes): + _vb_pattern = b"^[ -~]{0,200}[!-~]$" + else: + _vb_pattern = "^[ -~]{0,200}[!-~]$" + return re.match(_vb_pattern, s) + +# Invoke mainline +# =============== + +# Call test() when this file is run as a script (not imported as a module) +if __name__ == '__main__': + test() diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/cgi/test_cgi.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/cgi/test_cgi.py new file mode 100644 index 0000000000..63d171f9c8 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/cgi/test_cgi.py @@ -0,0 +1,706 @@ +import os +import sys +import tempfile +import types +import unittest +import warnings +from collections import namedtuple +from io import StringIO, BytesIO + +from . import cgi + +# Inlined from Lib/test/support/__init__.py in Python 3.12 +def check__all__(test_case, module, name_of_module=None, extra=(), + not_exported=()): + """Assert that the __all__ variable of 'module' contains all public names. + + The module's public names (its API) are detected automatically based on + whether they match the public name convention and were defined in + 'module'. + + The 'name_of_module' argument can specify (as a string or tuple thereof) + what module(s) an API could be defined in in order to be detected as a + public API. One case for this is when 'module' imports part of its public + API from other modules, possibly a C backend (like 'csv' and its '_csv'). + + The 'extra' argument can be a set of names that wouldn't otherwise be + automatically detected as "public", like objects without a proper + '__module__' attribute. If provided, it will be added to the + automatically detected ones. + + The 'not_exported' argument can be a set of names that must not be treated + as part of the public API even though their names indicate otherwise. + + Usage: + import bar + import foo + import unittest + from test import support + + class MiscTestCase(unittest.TestCase): + def test__all__(self): + support.check__all__(self, foo) + + class OtherTestCase(unittest.TestCase): + def test__all__(self): + extra = {'BAR_CONST', 'FOO_CONST'} + not_exported = {'baz'} # Undocumented name. + # bar imports part of its API from _bar. + support.check__all__(self, bar, ('bar', '_bar'), + extra=extra, not_exported=not_exported) + + """ + + if name_of_module is None: + name_of_module = (module.__name__, ) + elif isinstance(name_of_module, str): + name_of_module = (name_of_module, ) + + expected = set(extra) + + for name in dir(module): + if name.startswith('_') or name in not_exported: + continue + obj = getattr(module, name) + if (getattr(obj, '__module__', None) in name_of_module or + (not hasattr(obj, '__module__') and + not isinstance(obj, types.ModuleType))): + expected.add(name) + test_case.assertCountEqual(module.__all__, expected) + + +class HackedSysModule: + # The regression test will have real values in sys.argv, which + # will completely confuse the test of the cgi module + argv = [] + stdin = sys.stdin + version_info = sys.version_info + +cgi.sys = HackedSysModule() + +class ComparableException: + def __init__(self, err): + self.err = err + + def __str__(self): + return str(self.err) + + def __eq__(self, anExc): + if not isinstance(anExc, Exception): + return NotImplemented + return (self.err.__class__ == anExc.__class__ and + self.err.args == anExc.args) + + def __getattr__(self, attr): + return getattr(self.err, attr) + +def do_test(buf, method): + env = {} + if method == "GET": + fp = None + env['REQUEST_METHOD'] = 'GET' + env['QUERY_STRING'] = buf + elif method == "POST": + fp = BytesIO(buf.encode('latin-1')) # FieldStorage expects bytes + env['REQUEST_METHOD'] = 'POST' + env['CONTENT_TYPE'] = 'application/x-www-form-urlencoded' + env['CONTENT_LENGTH'] = str(len(buf)) + else: + raise ValueError("unknown method: %s" % method) + try: + return cgi.parse(fp, env, strict_parsing=1) + except Exception as err: + return ComparableException(err) + +parse_strict_test_cases = [ + ("", {}), + ("&", ValueError("bad query field: ''")), + ("&&", ValueError("bad query field: ''")), + # Should the next few really be valid? + ("=", {}), + ("=&=", {}), + # This rest seem to make sense + ("=a", {'': ['a']}), + ("&=a", ValueError("bad query field: ''")), + ("=a&", ValueError("bad query field: ''")), + ("=&a", ValueError("bad query field: 'a'")), + ("b=a", {'b': ['a']}), + ("b+=a", {'b ': ['a']}), + ("a=b=a", {'a': ['b=a']}), + ("a=+b=a", {'a': [' b=a']}), + ("&b=a", ValueError("bad query field: ''")), + ("b&=a", ValueError("bad query field: 'b'")), + ("a=a+b&b=b+c", {'a': ['a b'], 'b': ['b c']}), + ("a=a+b&a=b+a", {'a': ['a b', 'b a']}), + ("x=1&y=2.0&z=2-3.%2b0", {'x': ['1'], 'y': ['2.0'], 'z': ['2-3.+0']}), + ("Hbc5161168c542333633315dee1182227:key_store_seqid=400006&cuyer=r&view=bustomer&order_id=0bb2e248638833d48cb7fed300000f1b&expire=964546263&lobale=en-US&kid=130003.300038&ss=env", + {'Hbc5161168c542333633315dee1182227:key_store_seqid': ['400006'], + 'cuyer': ['r'], + 'expire': ['964546263'], + 'kid': ['130003.300038'], + 'lobale': ['en-US'], + 'order_id': ['0bb2e248638833d48cb7fed300000f1b'], + 'ss': ['env'], + 'view': ['bustomer'], + }), + + ("group_id=5470&set=custom&_assigned_to=31392&_status=1&_category=100&SUBMIT=Browse", + {'SUBMIT': ['Browse'], + '_assigned_to': ['31392'], + '_category': ['100'], + '_status': ['1'], + 'group_id': ['5470'], + 'set': ['custom'], + }) + ] + +# The behaviour of urllib.parse.parse_qs with empty input changed in Python 3.11 +if sys.version_info < (3, 11): + parse_strict_test_cases[0] = ("", ValueError("bad query field: ''")) + +def norm(seq): + return sorted(seq, key=repr) + +def first_elts(list): + return [p[0] for p in list] + +def first_second_elts(list): + return [(p[0], p[1][0]) for p in list] + +def gen_result(data, environ): + encoding = 'latin-1' + fake_stdin = BytesIO(data.encode(encoding)) + fake_stdin.seek(0) + form = cgi.FieldStorage(fp=fake_stdin, environ=environ, encoding=encoding) + + result = {} + for k, v in dict(form).items(): + result[k] = isinstance(v, list) and form.getlist(k) or v.value + + return result + +class CgiTests(unittest.TestCase): + + def test_parse_multipart(self): + fp = BytesIO(POSTDATA.encode('latin1')) + env = {'boundary': BOUNDARY.encode('latin1'), + 'CONTENT-LENGTH': '558'} + result = cgi.parse_multipart(fp, env) + expected = {'submit': [' Add '], 'id': ['1234'], + 'file': [b'Testing 123.\n'], 'title': ['']} + self.assertEqual(result, expected) + + def test_parse_multipart_without_content_length(self): + POSTDATA = '''--JfISa01 +Content-Disposition: form-data; name="submit-name" + +just a string + +--JfISa01-- +''' + fp = BytesIO(POSTDATA.encode('latin1')) + env = {'boundary': 'JfISa01'.encode('latin1')} + result = cgi.parse_multipart(fp, env) + expected = {'submit-name': ['just a string\n']} + self.assertEqual(result, expected) + + def test_parse_multipart_invalid_encoding(self): + BOUNDARY = "JfISa01" + POSTDATA = """--JfISa01 +Content-Disposition: form-data; name="submit-name" +Content-Length: 3 + +\u2603 +--JfISa01""" + fp = BytesIO(POSTDATA.encode('utf8')) + env = {'boundary': BOUNDARY.encode('latin1'), + 'CONTENT-LENGTH': str(len(POSTDATA.encode('utf8')))} + result = cgi.parse_multipart(fp, env, encoding="ascii", + errors="surrogateescape") + expected = {'submit-name': ["\udce2\udc98\udc83"]} + self.assertEqual(result, expected) + self.assertEqual("\u2603".encode('utf8'), + result["submit-name"][0].encode('utf8', 'surrogateescape')) + + def test_fieldstorage_properties(self): + fs = cgi.FieldStorage() + self.assertFalse(fs) + self.assertIn("FieldStorage", repr(fs)) + self.assertEqual(list(fs), list(fs.keys())) + fs.list.append(namedtuple('MockFieldStorage', 'name')('fieldvalue')) + self.assertTrue(fs) + + def test_fieldstorage_invalid(self): + self.assertRaises(TypeError, cgi.FieldStorage, "not-a-file-obj", + environ={"REQUEST_METHOD":"PUT"}) + self.assertRaises(TypeError, cgi.FieldStorage, "foo", "bar") + fs = cgi.FieldStorage(headers={'content-type':'text/plain'}) + self.assertRaises(TypeError, bool, fs) + + def test_strict(self): + for orig, expect in parse_strict_test_cases: + # Test basic parsing + d = do_test(orig, "GET") + self.assertEqual(d, expect, "Error parsing %s method GET" % repr(orig)) + d = do_test(orig, "POST") + self.assertEqual(d, expect, "Error parsing %s method POST" % repr(orig)) + + env = {'QUERY_STRING': orig} + fs = cgi.FieldStorage(environ=env) + if isinstance(expect, dict): + # test dict interface + self.assertEqual(len(expect), len(fs)) + self.assertCountEqual(expect.keys(), fs.keys()) + ##self.assertEqual(norm(expect.values()), norm(fs.values())) + ##self.assertEqual(norm(expect.items()), norm(fs.items())) + self.assertEqual(fs.getvalue("nonexistent field", "default"), "default") + # test individual fields + for key in expect.keys(): + expect_val = expect[key] + self.assertIn(key, fs) + if len(expect_val) > 1: + self.assertEqual(fs.getvalue(key), expect_val) + else: + self.assertEqual(fs.getvalue(key), expect_val[0]) + + def test_separator(self): + parse_semicolon = [ + ("x=1;y=2.0", {'x': ['1'], 'y': ['2.0']}), + ("x=1;y=2.0;z=2-3.%2b0", {'x': ['1'], 'y': ['2.0'], 'z': ['2-3.+0']}), + (";", ValueError("bad query field: ''")), + (";;", ValueError("bad query field: ''")), + ("=;a", ValueError("bad query field: 'a'")), + (";b=a", ValueError("bad query field: ''")), + ("b;=a", ValueError("bad query field: 'b'")), + ("a=a+b;b=b+c", {'a': ['a b'], 'b': ['b c']}), + ("a=a+b;a=b+a", {'a': ['a b', 'b a']}), + ] + for orig, expect in parse_semicolon: + env = {'QUERY_STRING': orig} + fs = cgi.FieldStorage(separator=';', environ=env) + if isinstance(expect, dict): + for key in expect.keys(): + expect_val = expect[key] + self.assertIn(key, fs) + if len(expect_val) > 1: + self.assertEqual(fs.getvalue(key), expect_val) + else: + self.assertEqual(fs.getvalue(key), expect_val[0]) + + def test_log(self): + with warnings.catch_warnings(): + warnings.simplefilter('ignore', category=DeprecationWarning) + cgi.log("Testing") + + cgi.logfp = StringIO() + cgi.initlog("%s", "Testing initlog 1") + cgi.log("%s", "Testing log 2") + self.assertEqual(cgi.logfp.getvalue(), "Testing initlog 1\nTesting log 2\n") + if os.path.exists(os.devnull): + cgi.logfp = None + cgi.logfile = os.devnull + cgi.initlog("%s", "Testing log 3") + self.addCleanup(cgi.closelog) + cgi.log("Testing log 4") + + def test_fieldstorage_readline(self): + # FieldStorage uses readline, which has the capacity to read all + # contents of the input file into memory; we use readline's size argument + # to prevent that for files that do not contain any newlines in + # non-GET/HEAD requests + class TestReadlineFile: + def __init__(self, file): + self.file = file + self.numcalls = 0 + + def readline(self, size=None): + self.numcalls += 1 + if size: + return self.file.readline(size) + else: + return self.file.readline() + + def __getattr__(self, name): + file = self.__dict__['file'] + a = getattr(file, name) + if not isinstance(a, int): + setattr(self, name, a) + return a + + f = TestReadlineFile(tempfile.TemporaryFile("wb+")) + self.addCleanup(f.close) + f.write(b'x' * 256 * 1024) + f.seek(0) + env = {'REQUEST_METHOD':'PUT'} + fs = cgi.FieldStorage(fp=f, environ=env) + self.addCleanup(fs.file.close) + # if we're not chunking properly, readline is only called twice + # (by read_binary); if we are chunking properly, it will be called 5 times + # as long as the chunksize is 1 << 16. + self.assertGreater(f.numcalls, 2) + f.close() + + def test_fieldstorage_multipart(self): + #Test basic FieldStorage multipart parsing + env = { + 'REQUEST_METHOD': 'POST', + 'CONTENT_TYPE': 'multipart/form-data; boundary={}'.format(BOUNDARY), + 'CONTENT_LENGTH': '558'} + fp = BytesIO(POSTDATA.encode('latin-1')) + fs = cgi.FieldStorage(fp, environ=env, encoding="latin-1") + self.assertEqual(len(fs.list), 4) + expect = [{'name':'id', 'filename':None, 'value':'1234'}, + {'name':'title', 'filename':None, 'value':''}, + {'name':'file', 'filename':'test.txt', 'value':b'Testing 123.\n'}, + {'name':'submit', 'filename':None, 'value':' Add '}] + for x in range(len(fs.list)): + for k, exp in expect[x].items(): + got = getattr(fs.list[x], k) + self.assertEqual(got, exp) + + def test_fieldstorage_multipart_leading_whitespace(self): + env = { + 'REQUEST_METHOD': 'POST', + 'CONTENT_TYPE': 'multipart/form-data; boundary={}'.format(BOUNDARY), + 'CONTENT_LENGTH': '560'} + # Add some leading whitespace to our post data that will cause the + # first line to not be the innerboundary. + fp = BytesIO(b"\r\n" + POSTDATA.encode('latin-1')) + fs = cgi.FieldStorage(fp, environ=env, encoding="latin-1") + self.assertEqual(len(fs.list), 4) + expect = [{'name':'id', 'filename':None, 'value':'1234'}, + {'name':'title', 'filename':None, 'value':''}, + {'name':'file', 'filename':'test.txt', 'value':b'Testing 123.\n'}, + {'name':'submit', 'filename':None, 'value':' Add '}] + for x in range(len(fs.list)): + for k, exp in expect[x].items(): + got = getattr(fs.list[x], k) + self.assertEqual(got, exp) + + def test_fieldstorage_multipart_non_ascii(self): + #Test basic FieldStorage multipart parsing + env = {'REQUEST_METHOD':'POST', + 'CONTENT_TYPE': 'multipart/form-data; boundary={}'.format(BOUNDARY), + 'CONTENT_LENGTH':'558'} + for encoding in ['iso-8859-1','utf-8']: + fp = BytesIO(POSTDATA_NON_ASCII.encode(encoding)) + fs = cgi.FieldStorage(fp, environ=env,encoding=encoding) + self.assertEqual(len(fs.list), 1) + expect = [{'name':'id', 'filename':None, 'value':'\xe7\xf1\x80'}] + for x in range(len(fs.list)): + for k, exp in expect[x].items(): + got = getattr(fs.list[x], k) + self.assertEqual(got, exp) + + def test_fieldstorage_multipart_maxline(self): + # Issue #18167 + maxline = 1 << 16 + self.maxDiff = None + def check(content): + data = """---123 +Content-Disposition: form-data; name="upload"; filename="fake.txt" +Content-Type: text/plain + +%s +---123-- +""".replace('\n', '\r\n') % content + environ = { + 'CONTENT_LENGTH': str(len(data)), + 'CONTENT_TYPE': 'multipart/form-data; boundary=-123', + 'REQUEST_METHOD': 'POST', + } + self.assertEqual(gen_result(data, environ), + {'upload': content.encode('latin1')}) + check('x' * (maxline - 1)) + check('x' * (maxline - 1) + '\r') + check('x' * (maxline - 1) + '\r' + 'y' * (maxline - 1)) + + def test_fieldstorage_multipart_w3c(self): + # Test basic FieldStorage multipart parsing (W3C sample) + env = { + 'REQUEST_METHOD': 'POST', + 'CONTENT_TYPE': 'multipart/form-data; boundary={}'.format(BOUNDARY_W3), + 'CONTENT_LENGTH': str(len(POSTDATA_W3))} + fp = BytesIO(POSTDATA_W3.encode('latin-1')) + fs = cgi.FieldStorage(fp, environ=env, encoding="latin-1") + self.assertEqual(len(fs.list), 2) + self.assertEqual(fs.list[0].name, 'submit-name') + self.assertEqual(fs.list[0].value, 'Larry') + self.assertEqual(fs.list[1].name, 'files') + files = fs.list[1].value + self.assertEqual(len(files), 2) + expect = [{'name': None, 'filename': 'file1.txt', 'value': b'... contents of file1.txt ...'}, + {'name': None, 'filename': 'file2.gif', 'value': b'...contents of file2.gif...'}] + for x in range(len(files)): + for k, exp in expect[x].items(): + got = getattr(files[x], k) + self.assertEqual(got, exp) + + def test_fieldstorage_part_content_length(self): + BOUNDARY = "JfISa01" + POSTDATA = """--JfISa01 +Content-Disposition: form-data; name="submit-name" +Content-Length: 5 + +Larry +--JfISa01""" + env = { + 'REQUEST_METHOD': 'POST', + 'CONTENT_TYPE': 'multipart/form-data; boundary={}'.format(BOUNDARY), + 'CONTENT_LENGTH': str(len(POSTDATA))} + fp = BytesIO(POSTDATA.encode('latin-1')) + fs = cgi.FieldStorage(fp, environ=env, encoding="latin-1") + self.assertEqual(len(fs.list), 1) + self.assertEqual(fs.list[0].name, 'submit-name') + self.assertEqual(fs.list[0].value, 'Larry') + + def test_field_storage_multipart_no_content_length(self): + fp = BytesIO(b"""--MyBoundary +Content-Disposition: form-data; name="my-arg"; filename="foo" + +Test + +--MyBoundary-- +""") + env = { + "REQUEST_METHOD": "POST", + "CONTENT_TYPE": "multipart/form-data; boundary=MyBoundary", + "wsgi.input": fp, + } + fields = cgi.FieldStorage(fp, environ=env) + + self.assertEqual(len(fields["my-arg"].file.read()), 5) + + def test_fieldstorage_as_context_manager(self): + fp = BytesIO(b'x' * 10) + env = {'REQUEST_METHOD': 'PUT'} + with cgi.FieldStorage(fp=fp, environ=env) as fs: + content = fs.file.read() + self.assertFalse(fs.file.closed) + self.assertTrue(fs.file.closed) + self.assertEqual(content, 'x' * 10) + with self.assertRaisesRegex(ValueError, 'I/O operation on closed file'): + fs.file.read() + + _qs_result = { + 'key1': 'value1', + 'key2': ['value2x', 'value2y'], + 'key3': 'value3', + 'key4': 'value4' + } + def testQSAndUrlEncode(self): + data = "key2=value2x&key3=value3&key4=value4" + environ = { + 'CONTENT_LENGTH': str(len(data)), + 'CONTENT_TYPE': 'application/x-www-form-urlencoded', + 'QUERY_STRING': 'key1=value1&key2=value2y', + 'REQUEST_METHOD': 'POST', + } + v = gen_result(data, environ) + self.assertEqual(self._qs_result, v) + + def test_max_num_fields(self): + # For application/x-www-form-urlencoded + data = '&'.join(['a=a']*11) + environ = { + 'CONTENT_LENGTH': str(len(data)), + 'CONTENT_TYPE': 'application/x-www-form-urlencoded', + 'REQUEST_METHOD': 'POST', + } + + with self.assertRaises(ValueError): + cgi.FieldStorage( + fp=BytesIO(data.encode()), + environ=environ, + max_num_fields=10, + ) + + # For multipart/form-data + data = """---123 +Content-Disposition: form-data; name="a" + +3 +---123 +Content-Type: application/x-www-form-urlencoded + +a=4 +---123 +Content-Type: application/x-www-form-urlencoded + +a=5 +---123-- +""" + environ = { + 'CONTENT_LENGTH': str(len(data)), + 'CONTENT_TYPE': 'multipart/form-data; boundary=-123', + 'QUERY_STRING': 'a=1&a=2', + 'REQUEST_METHOD': 'POST', + } + + # 2 GET entities + # 1 top level POST entities + # 1 entity within the second POST entity + # 1 entity within the third POST entity + with self.assertRaises(ValueError): + cgi.FieldStorage( + fp=BytesIO(data.encode()), + environ=environ, + max_num_fields=4, + ) + cgi.FieldStorage( + fp=BytesIO(data.encode()), + environ=environ, + max_num_fields=5, + ) + + def testQSAndFormData(self): + data = """---123 +Content-Disposition: form-data; name="key2" + +value2y +---123 +Content-Disposition: form-data; name="key3" + +value3 +---123 +Content-Disposition: form-data; name="key4" + +value4 +---123-- +""" + environ = { + 'CONTENT_LENGTH': str(len(data)), + 'CONTENT_TYPE': 'multipart/form-data; boundary=-123', + 'QUERY_STRING': 'key1=value1&key2=value2x', + 'REQUEST_METHOD': 'POST', + } + v = gen_result(data, environ) + self.assertEqual(self._qs_result, v) + + def testQSAndFormDataFile(self): + data = """---123 +Content-Disposition: form-data; name="key2" + +value2y +---123 +Content-Disposition: form-data; name="key3" + +value3 +---123 +Content-Disposition: form-data; name="key4" + +value4 +---123 +Content-Disposition: form-data; name="upload"; filename="fake.txt" +Content-Type: text/plain + +this is the content of the fake file + +---123-- +""" + environ = { + 'CONTENT_LENGTH': str(len(data)), + 'CONTENT_TYPE': 'multipart/form-data; boundary=-123', + 'QUERY_STRING': 'key1=value1&key2=value2x', + 'REQUEST_METHOD': 'POST', + } + result = self._qs_result.copy() + result.update({ + 'upload': b'this is the content of the fake file\n' + }) + v = gen_result(data, environ) + self.assertEqual(result, v) + + def test_parse_header(self): + self.assertEqual( + cgi.parse_header("text/plain"), + ("text/plain", {})) + self.assertEqual( + cgi.parse_header("text/vnd.just.made.this.up ; "), + ("text/vnd.just.made.this.up", {})) + self.assertEqual( + cgi.parse_header("text/plain;charset=us-ascii"), + ("text/plain", {"charset": "us-ascii"})) + self.assertEqual( + cgi.parse_header('text/plain ; charset="us-ascii"'), + ("text/plain", {"charset": "us-ascii"})) + self.assertEqual( + cgi.parse_header('text/plain ; charset="us-ascii"; another=opt'), + ("text/plain", {"charset": "us-ascii", "another": "opt"})) + self.assertEqual( + cgi.parse_header('attachment; filename="silly.txt"'), + ("attachment", {"filename": "silly.txt"})) + self.assertEqual( + cgi.parse_header('attachment; filename="strange;name"'), + ("attachment", {"filename": "strange;name"})) + self.assertEqual( + cgi.parse_header('attachment; filename="strange;name";size=123;'), + ("attachment", {"filename": "strange;name", "size": "123"})) + self.assertEqual( + cgi.parse_header('form-data; name="files"; filename="fo\\"o;bar"'), + ("form-data", {"name": "files", "filename": 'fo"o;bar'})) + + def test_all(self): + not_exported = { + "logfile", "logfp", "initlog", "dolog", "nolog", "closelog", "log", + "maxlen", "valid_boundary"} + check__all__(self, cgi, not_exported=not_exported) + + +BOUNDARY = "---------------------------721837373350705526688164684" + +POSTDATA = """-----------------------------721837373350705526688164684 +Content-Disposition: form-data; name="id" + +1234 +-----------------------------721837373350705526688164684 +Content-Disposition: form-data; name="title" + + +-----------------------------721837373350705526688164684 +Content-Disposition: form-data; name="file"; filename="test.txt" +Content-Type: text/plain + +Testing 123. + +-----------------------------721837373350705526688164684 +Content-Disposition: form-data; name="submit" + + Add\x20 +-----------------------------721837373350705526688164684-- +""" + +POSTDATA_NON_ASCII = """-----------------------------721837373350705526688164684 +Content-Disposition: form-data; name="id" + +\xe7\xf1\x80 +-----------------------------721837373350705526688164684 +""" + +# http://www.w3.org/TR/html401/interact/forms.html#h-17.13.4 +BOUNDARY_W3 = "AaB03x" +POSTDATA_W3 = """--AaB03x +Content-Disposition: form-data; name="submit-name" + +Larry +--AaB03x +Content-Disposition: form-data; name="files" +Content-Type: multipart/mixed; boundary=BbC04y + +--BbC04y +Content-Disposition: file; filename="file1.txt" +Content-Type: text/plain + +... contents of file1.txt ... +--BbC04y +Content-Disposition: file; filename="file2.gif" +Content-Type: image/gif +Content-Transfer-Encoding: binary + +...contents of file2.gif... +--BbC04y-- +--AaB03x-- +""" + +if __name__ == '__main__': + unittest.main() diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/config.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/config.py new file mode 100644 index 0000000000..50e20f05f0 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/config.py @@ -0,0 +1,339 @@ +# mypy: allow-untyped-defs + +import copy +import os +from collections import defaultdict +from typing import Any, Mapping + +from . import sslutils +from .utils import get_port + + +_renamed_props = { + "host": "browser_host", + "bind_hostname": "bind_address", + "external_host": "server_host", + "host_ip": "server_host", +} + + +def _merge_dict(base_dict, override_dict): + rv = base_dict.copy() + for key, value in base_dict.items(): + if key in override_dict: + if isinstance(value, dict): + rv[key] = _merge_dict(value, override_dict[key]) + else: + rv[key] = override_dict[key] + return rv + + +class Config(Mapping[str, Any]): + """wptserve configuration data + + Immutable configuration that's safe to be passed between processes. + + Inherits from Mapping for backwards compatibility with the old dict-based config + + :param data: - Extra configuration data + """ + def __init__(self, data): + for name in data.keys(): + if name.startswith("_"): + raise ValueError("Invalid configuration key %s" % name) + self.__dict__.update(data) + + def __str__(self): + return str(self.__dict__) + + def __setattr__(self, key, value): + raise ValueError("Config is immutable") + + def __setitem__(self, key, value): + raise ValueError("Config is immutable") + + def __getitem__(self, key): + try: + return getattr(self, key) + except AttributeError: + raise ValueError + + def __contains__(self, key): + return key in self.__dict__ + + def __iter__(self): + return (x for x in self.__dict__ if not x.startswith("_")) + + def __len__(self): + return len([item for item in self]) + + def as_dict(self): + return json_types(self.__dict__, skip={"_logger"}) + + +def json_types(obj, skip=None): + if skip is None: + skip = set() + if isinstance(obj, dict): + return {key: json_types(value) for key, value in obj.items() if key not in skip} + if (isinstance(obj, str) or + isinstance(obj, int) or + isinstance(obj, float) or + isinstance(obj, bool) or + obj is None): + return obj + if isinstance(obj, list) or hasattr(obj, "__iter__"): + return [json_types(value) for value in obj] + raise ValueError + + +class ConfigBuilder: + """Builder object for setting the wptserve config. + + Configuration can be passed in as a dictionary to the constructor, or + set via attributes after construction. Configuration options must match + the keys on the _default class property. + + The generated configuration is obtained by using the builder + object as a context manager; this returns a Config object + containing immutable configuration that may be shared between + threads and processes. In general the configuration is only valid + for the context used to obtain it. + + with ConfigBuilder() as config: + # Use the configuration + print config.browser_host + + The properties on the final configuration include those explicitly + supplied and computed properties. The computed properties are + defined by the computed_properties attribute on the class. This + is a list of property names, each corresponding to a _get_ + method on the class. These methods are called in the order defined + in computed_properties and are passed a single argument, a + dictionary containing the current set of properties. Thus computed + properties later in the list may depend on the value of earlier + ones. + + + :param logger: - A logger object. This is used for logging during + the creation of the configuration, but isn't + part of the configuration + :param subdomains: - A set of valid subdomains to include in the + configuration. + :param not_subdomains: - A set of invalid subdomains to include in + the configuration. + :param config_cls: - A class to use for the configuration. Defaults + to default_config_cls + """ + + _default = { + "browser_host": "localhost", + "alternate_hosts": {}, + "doc_root": os.path.dirname("__file__"), + "server_host": None, + "ports": {"http": [8000]}, + "check_subdomains": True, + "bind_address": True, + "ssl": { + "type": "none", + "encrypt_after_connect": False, + "none": {}, + "openssl": { + "openssl_binary": "openssl", + "base_path": "_certs", + "password": "web-platform-tests", + "force_regenerate": False, + "duration": 30, + "base_conf_path": None + }, + "pregenerated": { + "host_key_path": None, + "host_cert_path": None, + }, + }, + "aliases": [], + "logging": { + "level": "debug", + "suppress_handler_traceback": False, + } + } + default_config_cls = Config + + # Configuration properties that are computed. Each corresponds to a method + # _get_foo, which is called with the current data dictionary. The properties + # are computed in the order specified in the list. + computed_properties = ["logging", + "paths", + "server_host", + "ports", + "domains", + "not_domains", + "all_domains", + "domains_set", + "not_domains_set", + "all_domains_set", + "ssl_config"] + + def __init__(self, + logger, + subdomains=set(), + not_subdomains=set(), + config_cls=None, + **kwargs): + + self._logger = logger + self._data = self._default.copy() + self._ssl_env = None + + self._config_cls = config_cls or self.default_config_cls + + for k, v in self._default.items(): + self._data[k] = kwargs.pop(k, v) + + self._data["subdomains"] = subdomains + self._data["not_subdomains"] = not_subdomains + + for k, new_k in _renamed_props.items(): + if k in kwargs: + logger.warning( + "%s in config is deprecated; use %s instead" % ( + k, + new_k + ) + ) + self._data[new_k] = kwargs.pop(k) + + if kwargs: + raise TypeError("__init__() got unexpected keyword arguments %r" % (tuple(kwargs),)) + + def __setattr__(self, key, value): + if not key[0] == "_": + self._data[key] = value + else: + self.__dict__[key] = value + + def __getattr__(self, key): + try: + return self._data[key] + except KeyError as e: + raise AttributeError from e + + def update(self, override): + """Load an overrides dict to override config values""" + override = override.copy() + + for k in self._default: + if k in override: + self._set_override(k, override.pop(k)) + + for k, new_k in _renamed_props.items(): + if k in override: + self._logger.warning( + "%s in config is deprecated; use %s instead" % ( + k, + new_k + ) + ) + self._set_override(new_k, override.pop(k)) + + if override: + k = next(iter(override)) + raise KeyError("unknown config override '%s'" % k) + + def _set_override(self, k, v): + old_v = self._data[k] + if isinstance(old_v, dict): + self._data[k] = _merge_dict(old_v, v) + else: + self._data[k] = v + + def __enter__(self): + if self._ssl_env is not None: + raise ValueError("Tried to re-enter configuration") + data = self._data.copy() + prefix = "_get_" + for key in self.computed_properties: + data[key] = getattr(self, prefix + key)(data) + return self._config_cls(data) + + def __exit__(self, *args): + self._ssl_env.__exit__(*args) + self._ssl_env = None + + def _get_logging(self, data): + logging = data["logging"] + logging["level"] = logging["level"].upper() + return logging + + def _get_paths(self, data): + return {"doc_root": data["doc_root"]} + + def _get_server_host(self, data): + return data["server_host"] if data.get("server_host") is not None else data["browser_host"] + + def _get_ports(self, data): + new_ports = defaultdict(list) + for scheme, ports in data["ports"].items(): + if scheme in ["wss", "https"] and not sslutils.get_cls(data["ssl"]["type"]).ssl_enabled: + continue + for i, port in enumerate(ports): + real_port = get_port("") if port == "auto" else port + new_ports[scheme].append(real_port) + return new_ports + + def _get_domains(self, data): + hosts = data["alternate_hosts"].copy() + assert "" not in hosts + hosts[""] = data["browser_host"] + + rv = {} + for name, host in hosts.items(): + rv[name] = {subdomain: (subdomain.encode("idna").decode("ascii") + "." + host) + for subdomain in data["subdomains"]} + rv[name][""] = host + return rv + + def _get_not_domains(self, data): + hosts = data["alternate_hosts"].copy() + assert "" not in hosts + hosts[""] = data["browser_host"] + + rv = {} + for name, host in hosts.items(): + rv[name] = {subdomain: (subdomain.encode("idna").decode("ascii") + "." + host) + for subdomain in data["not_subdomains"]} + return rv + + def _get_all_domains(self, data): + rv = copy.deepcopy(data["domains"]) + nd = data["not_domains"] + for host in rv: + rv[host].update(nd[host]) + return rv + + def _get_domains_set(self, data): + return {domain + for per_host_domains in data["domains"].values() + for domain in per_host_domains.values()} + + def _get_not_domains_set(self, data): + return {domain + for per_host_domains in data["not_domains"].values() + for domain in per_host_domains.values()} + + def _get_all_domains_set(self, data): + return data["domains_set"] | data["not_domains_set"] + + def _get_ssl_config(self, data): + ssl_type = data["ssl"]["type"] + ssl_cls = sslutils.get_cls(ssl_type) + kwargs = data["ssl"].get(ssl_type, {}) + self._ssl_env = ssl_cls(self._logger, **kwargs) + self._ssl_env.__enter__() + if self._ssl_env.ssl_enabled: + key_path, cert_path = self._ssl_env.host_cert_path(data["domains_set"]) + ca_cert_path = self._ssl_env.ca_cert_path(data["domains_set"]) + return {"key_path": key_path, + "ca_cert_path": ca_cert_path, + "cert_path": cert_path, + "encrypt_after_connect": data["ssl"].get("encrypt_after_connect", False)} diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/constants.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/constants.py new file mode 100644 index 0000000000..935b5c881f --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/constants.py @@ -0,0 +1,98 @@ +from . import utils + +content_types = utils.invert_dict({ + "application/json": ["json"], + "application/wasm": ["wasm"], + "application/xhtml+xml": ["xht", "xhtm", "xhtml"], + "application/xml": ["xml"], + "application/x-xpinstall": ["xpi"], + "audio/mp4": ["m4a"], + "audio/mpeg": ["mp3"], + "audio/ogg": ["oga"], + "audio/webm": ["weba"], + "audio/x-wav": ["wav"], + "image/avif": ["avif"], + "image/bmp": ["bmp"], + "image/gif": ["gif"], + "image/jpeg": ["jpg", "jpeg"], + "image/jxl": ["jxl"], + "image/png": ["png"], + "image/svg+xml": ["svg"], + "text/cache-manifest": ["manifest"], + "text/css": ["css"], + "text/event-stream": ["event_stream"], + "text/html": ["htm", "html"], + "text/javascript": ["js", "mjs"], + "text/plain": ["txt", "md"], + "text/vtt": ["vtt"], + "video/mp4": ["mp4", "m4v"], + "video/webm": ["webm"], +}) + +response_codes = { + 100: ('Continue', 'Request received, please continue'), + 101: ('Switching Protocols', + 'Switching to new protocol; obey Upgrade header'), + + 200: ('OK', 'Request fulfilled, document follows'), + 201: ('Created', 'Document created, URL follows'), + 202: ('Accepted', + 'Request accepted, processing continues off-line'), + 203: ('Non-Authoritative Information', 'Request fulfilled from cache'), + 204: ('No Content', 'Request fulfilled, nothing follows'), + 205: ('Reset Content', 'Clear input form for further input.'), + 206: ('Partial Content', 'Partial content follows.'), + + 300: ('Multiple Choices', + 'Object has several resources -- see URI list'), + 301: ('Moved Permanently', 'Object moved permanently -- see URI list'), + 302: ('Found', 'Object moved temporarily -- see URI list'), + 303: ('See Other', 'Object moved -- see Method and URL list'), + 304: ('Not Modified', + 'Document has not changed since given time'), + 305: ('Use Proxy', + 'You must use proxy specified in Location to access this ' + 'resource.'), + 307: ('Temporary Redirect', + 'Object moved temporarily -- see URI list'), + + 400: ('Bad Request', + 'Bad request syntax or unsupported method'), + 401: ('Unauthorized', + 'No permission -- see authorization schemes'), + 402: ('Payment Required', + 'No payment -- see charging schemes'), + 403: ('Forbidden', + 'Request forbidden -- authorization will not help'), + 404: ('Not Found', 'Nothing matches the given URI'), + 405: ('Method Not Allowed', + 'Specified method is invalid for this resource.'), + 406: ('Not Acceptable', 'URI not available in preferred format.'), + 407: ('Proxy Authentication Required', 'You must authenticate with ' + 'this proxy before proceeding.'), + 408: ('Request Timeout', 'Request timed out; try again later.'), + 409: ('Conflict', 'Request conflict.'), + 410: ('Gone', + 'URI no longer exists and has been permanently removed.'), + 411: ('Length Required', 'Client must specify Content-Length.'), + 412: ('Precondition Failed', 'Precondition in headers is false.'), + 413: ('Request Entity Too Large', 'Entity is too large.'), + 414: ('Request-URI Too Long', 'URI is too long.'), + 415: ('Unsupported Media Type', 'Entity body in unsupported format.'), + 416: ('Requested Range Not Satisfiable', + 'Cannot satisfy request range.'), + 417: ('Expectation Failed', + 'Expect condition could not be satisfied.'), + + 500: ('Internal Server Error', 'Server got itself in trouble'), + 501: ('Not Implemented', + 'Server does not support this operation'), + 502: ('Bad Gateway', 'Invalid responses from another server/proxy.'), + 503: ('Service Unavailable', + 'The server cannot process the request due to a high load'), + 504: ('Gateway Timeout', + 'The gateway server did not receive a timely response'), + 505: ('HTTP Version Not Supported', 'Cannot fulfill request.'), +} + +h2_headers = ['method', 'scheme', 'host', 'path', 'authority', 'status'] diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/handlers.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/handlers.py new file mode 100644 index 0000000000..cde04b13cb --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/handlers.py @@ -0,0 +1,546 @@ +# mypy: allow-untyped-defs + +import json +import os +import pathlib +from collections import defaultdict + +from urllib.parse import quote, unquote, urljoin + +from .constants import content_types +from .pipes import Pipeline, template +from .ranges import RangeParser +from .request import Authentication +from .response import MultipartContent +from .utils import HTTPException + +from html import escape + +__all__ = ["file_handler", "python_script_handler", + "FunctionHandler", "handler", "json_handler", + "as_is_handler", "ErrorHandler", "BasicAuthHandler"] + + +def guess_content_type(path): + ext = os.path.splitext(path)[1].lstrip(".") + if ext in content_types: + return content_types[ext] + + return "application/octet-stream" + + +def filesystem_path(base_path, request, url_base="/"): + if base_path is None: + base_path = request.doc_root + + path = unquote(request.url_parts.path) + + if path.startswith(url_base): + path = path[len(url_base):] + + if ".." in path: + raise HTTPException(404) + + new_path = os.path.join(base_path, path) + + # Otherwise setting path to / allows access outside the root directory + if not new_path.startswith(base_path): + raise HTTPException(404) + + return new_path + + +class DirectoryHandler: + def __init__(self, base_path=None, url_base="/"): + self.base_path = base_path + self.url_base = url_base + + def __repr__(self): + return "<%s base_path:%s url_base:%s>" % (self.__class__.__name__, self.base_path, self.url_base) + + def __call__(self, request, response): + url_path = request.url_parts.path + + if not url_path.endswith("/"): + response.status = 301 + response.headers = [("Location", "%s/" % request.url)] + return + + path = filesystem_path(self.base_path, request, self.url_base) + + assert os.path.isdir(path) + + response.headers = [("Content-Type", "text/html")] + response.content = """ + +Directory listing for %(path)s +

Directory listing for %(path)s

+
    +%(items)s +
+""" % {"path": escape(url_path), + "items": "\n".join(self.list_items(url_path, path))} # noqa: E122 + + def list_items(self, base_path, path): + assert base_path.endswith("/") + + # TODO: this won't actually list all routes, only the + # ones that correspond to a real filesystem path. It's + # not possible to list every route that will match + # something, but it should be possible to at least list the + # statically defined ones + + if base_path != "/": + link = urljoin(base_path, "..") + yield ("""
  • %(name)s
    • """ % + {"link": link, "name": ".."}) + items = [] + prev_item = None + # This ensures that .headers always sorts after the file it provides the headers for. E.g., + # if we have x, x-y, and x.headers, the order will be x, x.headers, and then x-y. + for item in sorted(os.listdir(path), key=lambda x: (x[:-len(".headers")], x) if x.endswith(".headers") else (x, x)): + if prev_item and prev_item + ".headers" == item: + items[-1][1] = item + prev_item = None + continue + items.append([item, None]) + prev_item = item + for item, dot_headers in items: + link = escape(quote(item)) + dot_headers_markup = "" + if dot_headers is not None: + dot_headers_markup = (""" (.headers)""" % + {"link": escape(quote(dot_headers))}) + if os.path.isdir(os.path.join(path, item)): + link += "/" + class_ = "dir" + else: + class_ = "file" + yield ("""
  • %(name)s%(headers)s
    • """ % + {"link": link, "name": escape(item), "class": class_, + "headers": dot_headers_markup}) + + +def parse_qs(qs): + """Parse a query string given as a string argument (data of type + application/x-www-form-urlencoded). Data are returned as a dictionary. The + dictionary keys are the unique query variable names and the values are + lists of values for each name. + + This implementation is used instead of Python's built-in `parse_qs` method + in order to support the semicolon character (which the built-in method + interprets as a parameter delimiter).""" + pairs = [item.split("=", 1) for item in qs.split('&') if item] + rv = defaultdict(list) + for pair in pairs: + if len(pair) == 1 or len(pair[1]) == 0: + continue + name = unquote(pair[0].replace('+', ' ')) + value = unquote(pair[1].replace('+', ' ')) + rv[name].append(value) + return dict(rv) + + +def wrap_pipeline(path, request, response): + """Applies pipelines to a response. + + Pipelines are specified in the filename (.sub.) or the query param (?pipe). + """ + query = parse_qs(request.url_parts.query) + pipe_string = "" + + if ".sub." in path: + ml_extensions = {".html", ".htm", ".xht", ".xhtml", ".xml", ".svg"} + escape_type = "html" if os.path.splitext(path)[1] in ml_extensions else "none" + pipe_string = "sub(%s)" % escape_type + + if "pipe" in query: + if pipe_string: + pipe_string += "|" + + pipe_string += query["pipe"][-1] + + if pipe_string: + response = Pipeline(pipe_string)(request, response) + + return response + + +def load_headers(request, path): + """Loads headers from files for a given path. + + Attempts to load both the neighbouring __dir__{.sub}.headers and + PATH{.sub}.headers (applying template substitution if needed); results are + concatenated in that order. + """ + def _load(request, path): + headers_path = path + ".sub.headers" + if os.path.exists(headers_path): + use_sub = True + else: + headers_path = path + ".headers" + use_sub = False + + try: + with open(headers_path, "rb") as headers_file: + data = headers_file.read() + except OSError: + return [] + else: + if use_sub: + data = template(request, data, escape_type="none") + return [tuple(item.strip() for item in line.split(b":", 1)) + for line in data.splitlines() if line] + + return (_load(request, os.path.join(os.path.dirname(path), "__dir__")) + + _load(request, path)) + + +class FileHandler: + def __init__(self, base_path=None, url_base="/"): + self.base_path = base_path + self.url_base = url_base + self.directory_handler = DirectoryHandler(self.base_path, self.url_base) + + def __repr__(self): + return "<%s base_path:%s url_base:%s>" % (self.__class__.__name__, self.base_path, self.url_base) + + def __call__(self, request, response): + path = filesystem_path(self.base_path, request, self.url_base) + + if os.path.isdir(path): + return self.directory_handler(request, response) + try: + #This is probably racy with some other process trying to change the file + file_size = os.stat(path).st_size + response.headers.update(self.get_headers(request, path)) + if "Range" in request.headers: + try: + byte_ranges = RangeParser()(request.headers['Range'], file_size) + except HTTPException as e: + if e.code == 416: + response.headers.set("Content-Range", "bytes */%i" % file_size) + raise + else: + byte_ranges = None + data = self.get_data(response, path, byte_ranges) + response.content = data + response = wrap_pipeline(path, request, response) + return response + + except OSError: + raise HTTPException(404) + + def get_headers(self, request, path): + rv = load_headers(request, path) + + if not any(key.lower() == b"content-type" for (key, _) in rv): + rv.insert(0, (b"Content-Type", guess_content_type(path).encode("ascii"))) + + return rv + + def get_data(self, response, path, byte_ranges): + """Return either the handle to a file, or a string containing + the content of a chunk of the file, if we have a range request.""" + if byte_ranges is None: + return open(path, 'rb') + else: + with open(path, 'rb') as f: + response.status = 206 + if len(byte_ranges) > 1: + parts_content_type, content = self.set_response_multipart(response, + byte_ranges, + f) + for byte_range in byte_ranges: + content.append_part(self.get_range_data(f, byte_range), + parts_content_type, + [("Content-Range", byte_range.header_value())]) + return content + else: + response.headers.set("Content-Range", byte_ranges[0].header_value()) + return self.get_range_data(f, byte_ranges[0]) + + def set_response_multipart(self, response, ranges, f): + parts_content_type = response.headers.get("Content-Type") + if parts_content_type: + parts_content_type = parts_content_type[-1] + else: + parts_content_type = None + content = MultipartContent() + response.headers.set("Content-Type", "multipart/byteranges; boundary=%s" % content.boundary) + return parts_content_type, content + + def get_range_data(self, f, byte_range): + f.seek(byte_range.lower) + return f.read(byte_range.upper - byte_range.lower) + + +file_handler = FileHandler() # type: ignore + + +class PythonScriptHandler: + def __init__(self, base_path=None, url_base="/"): + self.base_path = base_path + self.url_base = url_base + + def __repr__(self): + return "<%s base_path:%s url_base:%s>" % (self.__class__.__name__, self.base_path, self.url_base) + + def _load_file(self, request, response, func): + """ + This loads the requested python file as an environ variable. + + If the requested file is a directory, this instead loads the first + lexicographically sorted file found in that directory that matches + "default*.py". + + Once the environ is loaded, the passed `func` is run with this loaded environ. + + :param request: The request object + :param response: The response object + :param func: The function to be run with the loaded environ with the modified filepath. Signature: (request, response, environ, path) + :return: The return of func + """ + path = filesystem_path(self.base_path, request, self.url_base) + + # Find a default Python file if the specified path is a directory + if os.path.isdir(path): + default_py_files = sorted(list(filter( + pathlib.Path.is_file, + pathlib.Path(path).glob("default*.py")))) + if default_py_files: + path = str(default_py_files[0]) + + try: + environ = {"__file__": path} + with open(path, 'rb') as f: + exec(compile(f.read(), path, 'exec'), environ, environ) + + if func is not None: + return func(request, response, environ, path) + + except OSError: + raise HTTPException(404) + + def __call__(self, request, response): + def func(request, response, environ, path): + if "main" in environ: + handler = FunctionHandler(environ["main"]) + handler(request, response) + wrap_pipeline(path, request, response) + else: + raise HTTPException(500, "No main function in script %s" % path) + + self._load_file(request, response, func) + + def frame_handler(self, request): + """ + This creates a FunctionHandler with one or more of the handling functions. + + Used by the H2 server. + + :param request: The request object used to generate the handler. + :return: A FunctionHandler object with one or more of these functions: `handle_headers`, `handle_data` or `main` + """ + def func(request, response, environ, path): + def _main(req, resp): + pass + + handler = FunctionHandler(_main) + if "main" in environ: + handler.func = environ["main"] + if "handle_headers" in environ: + handler.handle_headers = environ["handle_headers"] + if "handle_data" in environ: + handler.handle_data = environ["handle_data"] + + if handler.func is _main and not hasattr(handler, "handle_headers") and not hasattr(handler, "handle_data"): + raise HTTPException(500, "No main function or handlers in script %s" % path) + + return handler + return self._load_file(request, None, func) + + +python_script_handler = PythonScriptHandler() # type: ignore + + +class FunctionHandler: + def __init__(self, func): + self.func = func + + def __call__(self, request, response): + try: + rv = self.func(request, response) + except HTTPException: + raise + except Exception as e: + raise HTTPException(500) from e + if rv is not None: + if isinstance(rv, tuple): + if len(rv) == 3: + status, headers, content = rv + response.status = status + elif len(rv) == 2: + headers, content = rv + else: + raise HTTPException(500) + response.headers.update(headers) + else: + content = rv + response.content = content + wrap_pipeline('', request, response) + + +# The generic name here is so that this can be used as a decorator +def handler(func): + return FunctionHandler(func) + + +class JsonHandler: + def __init__(self, func): + self.func = func + + def __call__(self, request, response): + return FunctionHandler(self.handle_request)(request, response) + + def handle_request(self, request, response): + rv = self.func(request, response) + response.headers.set("Content-Type", "application/json") + enc = json.dumps + if isinstance(rv, tuple): + rv = list(rv) + value = tuple(rv[:-1] + [enc(rv[-1])]) + length = len(value[-1]) + else: + value = enc(rv) + length = len(value) + response.headers.set("Content-Length", length) + return value + + +def json_handler(func): + return JsonHandler(func) + + +class AsIsHandler: + def __init__(self, base_path=None, url_base="/"): + self.base_path = base_path + self.url_base = url_base + + def __call__(self, request, response): + path = filesystem_path(self.base_path, request, self.url_base) + if os.path.isdir(path): + raise HTTPException( + 500, "AsIsHandler cannot process directory, %s" % path) + + try: + with open(path, 'rb') as f: + response.writer.write_raw_content(f.read()) + wrap_pipeline(path, request, response) + response.close_connection = True + except OSError: + raise HTTPException(404) + + +as_is_handler = AsIsHandler() # type: ignore + + +class BasicAuthHandler: + def __init__(self, handler, user, password): + """ + A Basic Auth handler + + :Args: + - handler: a secondary handler for the request after authentication is successful (example file_handler) + - user: string of the valid user name or None if any / all credentials are allowed + - password: string of the password required + """ + self.user = user + self.password = password + self.handler = handler + + def __call__(self, request, response): + if "authorization" not in request.headers: + response.status = 401 + response.headers.set("WWW-Authenticate", "Basic") + return response + else: + auth = Authentication(request.headers) + if self.user is not None and (self.user != auth.username or self.password != auth.password): + response.set_error(403, "Invalid username or password") + return response + return self.handler(request, response) + + +basic_auth_handler = BasicAuthHandler(file_handler, None, None) # type: ignore + + +class ErrorHandler: + def __init__(self, status): + self.status = status + + def __call__(self, request, response): + response.set_error(self.status) + + +class StringHandler: + def __init__(self, data, content_type, **headers): + """Handler that returns a fixed data string and headers + + :param data: String to use + :param content_type: Content type header to server the response with + :param headers: List of headers to send with responses""" + + self.data = data + + self.resp_headers = [("Content-Type", content_type)] + for k, v in headers.items(): + self.resp_headers.append((k.replace("_", "-"), v)) + + self.handler = handler(self.handle_request) + + def handle_request(self, request, response): + return self.resp_headers, self.data + + def __call__(self, request, response): + rv = self.handler(request, response) + return rv + + +class StaticHandler: + def __init__(self, path, format_args, content_type, **headers): + """Handler that reads a file from a path and substitutes some fixed data + + Note that *.headers files have no effect in this handler. + + :param path: Path(s) to template files to use. If a sequence of paths is provided instead + of a single path, the contents of each file will be concatenated together before the + `format_args` are interpolated. + :param format_args: Dictionary of values to substitute into the template file + :param content_type: Content type header to server the response with + :param headers: List of headers to send with responses""" + self._paths = [path] if isinstance(path, str) else path + self._format_args = format_args + self._content_type = content_type + self._headers = headers + self._handler = None + + def __getnewargs_ex__(self): + # Do not pickle `self._handler`, which can be arbitrarily large. + args = self._paths, self._format_args, self._content_type + return args, self._headers + + def __call__(self, request, response): + # Load the static file contents lazily so that this handler can be + # pickled and sent to child processes efficiently. Transporting file + # contents across processes can slow `wptserve` startup by several + # seconds (crbug.com/1479850). + if not self._handler: + data = "" + for path in self._paths: + with open(path) as f: + data += f.read() + if self._format_args: + data = data % self._format_args + self._handler = StringHandler(data, self._content_type, **self._headers) + return self._handler(request, response) diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/logger.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/logger.py new file mode 100644 index 0000000000..8eff146a01 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/logger.py @@ -0,0 +1,5 @@ +import logging + +def get_logger() -> logging.Logger: + # Use the root logger + return logging.getLogger() diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/pipes.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/pipes.py new file mode 100644 index 0000000000..84b17c1228 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/pipes.py @@ -0,0 +1,561 @@ +# mypy: allow-untyped-defs + +from collections import deque +import base64 +import gzip as gzip_module +import hashlib +import os +import re +import time +import uuid + +from html import escape +from io import BytesIO +from typing import Any, Callable, ClassVar, Dict, Optional, TypeVar + +T = TypeVar('T') + + +def resolve_content(response): + return b"".join(item for item in response.iter_content(read_file=True)) + + +class Pipeline: + pipes: ClassVar[Dict[str, Callable[..., Any]]] = {} + + def __init__(self, pipe_string): + self.pipe_functions = self.parse(pipe_string) + + def parse(self, pipe_string): + functions = [] + for item in PipeTokenizer().tokenize(pipe_string): + if not item: + break + if item[0] == "function": + functions.append((self.pipes[item[1]], [])) + elif item[0] == "argument": + functions[-1][1].append(item[1]) + return functions + + def __call__(self, request, response): + for func, args in self.pipe_functions: + response = func(request, response, *args) + return response + + +class PipeTokenizer: + def __init__(self): + #This whole class can likely be replaced by some regexps + self.state = None + + def tokenize(self, string): + self.string = string + self.state = self.func_name_state + self._index = 0 + while self.state: + yield self.state() + yield None + + def get_char(self): + if self._index >= len(self.string): + return None + rv = self.string[self._index] + self._index += 1 + return rv + + def func_name_state(self): + rv = "" + while True: + char = self.get_char() + if char is None: + self.state = None + if rv: + return ("function", rv) + else: + return None + elif char == "(": + self.state = self.argument_state + return ("function", rv) + elif char == "|": + if rv: + return ("function", rv) + else: + rv += char + + def argument_state(self): + rv = "" + while True: + char = self.get_char() + if char is None: + self.state = None + return ("argument", rv) + elif char == "\\": + rv += self.get_escape() + if rv is None: + #This should perhaps be an error instead + return ("argument", rv) + elif char == ",": + return ("argument", rv) + elif char == ")": + self.state = self.func_name_state + return ("argument", rv) + else: + rv += char + + def get_escape(self): + char = self.get_char() + escapes = {"n": "\n", + "r": "\r", + "t": "\t"} + return escapes.get(char, char) + + +class pipe: + def __init__(self, *arg_converters: Callable[[str], Any]): + self.arg_converters = arg_converters + self.max_args = len(self.arg_converters) + self.min_args = 0 + opt_seen = False + for item in self.arg_converters: + if not opt_seen: + if isinstance(item, opt): + opt_seen = True + else: + self.min_args += 1 + else: + if not isinstance(item, opt): + raise ValueError("Non-optional argument cannot follow optional argument") + + def __call__(self, f): + def inner(request, response, *args): + if not (self.min_args <= len(args) <= self.max_args): + raise ValueError("Expected between %d and %d args, got %d" % + (self.min_args, self.max_args, len(args))) + arg_values = tuple(f(x) for f, x in zip(self.arg_converters, args)) + return f(request, response, *arg_values) + Pipeline.pipes[f.__name__] = inner + #We actually want the undecorated function in the main namespace + return f + + +class opt: + def __init__(self, f: Callable[[str], Any]): + self.f = f + + def __call__(self, arg: str) -> Any: + return self.f(arg) + + +def nullable(func: Callable[[str], T]) -> Callable[[str], Optional[T]]: + def inner(arg: str) -> Optional[T]: + if arg.lower() == "null": + return None + else: + return func(arg) + return inner + + +def boolean(arg: str) -> bool: + if arg.lower() in ("true", "1"): + return True + elif arg.lower() in ("false", "0"): + return False + raise ValueError + + +@pipe(int) +def status(request, response, code): + """Alter the status code. + + :param code: Status code to use for the response.""" + response.status = code + return response + + +@pipe(str, str, opt(boolean)) +def header(request, response, name, value, append=False): + """Set a HTTP header. + + Replaces any existing HTTP header of the same name unless + append is set, in which case the header is appended without + replacement. + + :param name: Name of the header to set. + :param value: Value to use for the header. + :param append: True if existing headers should not be replaced + """ + if not append: + response.headers.set(name, value) + else: + response.headers.append(name, value) + return response + + +@pipe(str) +def trickle(request, response, delays): + """Send the response in parts, with time delays. + + :param delays: A string of delays and amounts, in bytes, of the + response to send. Each component is separated by + a colon. Amounts in bytes are plain integers, whilst + delays are floats prefixed with a single d e.g. + d1:100:d2 + Would cause a 1 second delay, would then send 100 bytes + of the file, and then cause a 2 second delay, before sending + the remainder of the file. + + If the last token is of the form rN, instead of sending the + remainder of the file, the previous N instructions will be + repeated until the whole file has been sent e.g. + d1:100:d2:r2 + Causes a delay of 1s, then 100 bytes to be sent, then a 2s delay + and then a further 100 bytes followed by a two second delay + until the response has been fully sent. + """ + def parse_delays(): + parts = delays.split(":") + rv = [] + for item in parts: + if item.startswith("d"): + item_type = "delay" + item = item[1:] + value = float(item) + elif item.startswith("r"): + item_type = "repeat" + value = int(item[1:]) + if not value % 2 == 0: + raise ValueError + else: + item_type = "bytes" + value = int(item) + if len(rv) and rv[-1][0] == item_type: + rv[-1][1] += value + else: + rv.append((item_type, value)) + return rv + + delays = parse_delays() + if not delays: + return response + content = resolve_content(response) + offset = [0] + + if not ("Cache-Control" in response.headers or + "Pragma" in response.headers or + "Expires" in response.headers): + response.headers.set("Cache-Control", "no-cache, no-store, must-revalidate") + response.headers.set("Pragma", "no-cache") + response.headers.set("Expires", "0") + + def add_content(delays, repeat=False): + for i, (item_type, value) in enumerate(delays): + if item_type == "bytes": + yield content[offset[0]:offset[0] + value] + offset[0] += value + elif item_type == "delay": + time.sleep(value) + elif item_type == "repeat": + if i != len(delays) - 1: + continue + while offset[0] < len(content): + yield from add_content(delays[-(value + 1):-1], True) + + if not repeat and offset[0] < len(content): + yield content[offset[0]:] + + response.content = add_content(delays) + return response + + +@pipe(nullable(int), opt(nullable(int))) +def slice(request, response, start, end=None): + """Send a byte range of the response body + + :param start: The starting offset. Follows python semantics including + negative numbers. + + :param end: The ending offset, again with python semantics and None + (spelled "null" in a query string) to indicate the end of + the file. + """ + content = resolve_content(response)[start:end] + response.content = content + response.headers.set("Content-Length", len(content)) + return response + + +class ReplacementTokenizer: + def arguments(self, token): + unwrapped = token[1:-1].decode('utf8') + return ("arguments", re.split(r",\s*", unwrapped) if unwrapped else []) + + def ident(self, token): + return ("ident", token.decode('utf8')) + + def index(self, token): + token = token[1:-1].decode('utf8') + try: + index = int(token) + except ValueError: + index = token + return ("index", index) + + def var(self, token): + token = token[:-1].decode('utf8') + return ("var", token) + + def tokenize(self, string): + assert isinstance(string, bytes) + return self.scanner.scan(string)[0] + + # re.Scanner is missing from typeshed: + # https://github.com/python/typeshed/pull/3071 + scanner = re.Scanner([(br"\$\w+:", var), # type: ignore + (br"\$?\w+", ident), + (br"\[[^\]]*\]", index), + (br"\([^)]*\)", arguments)]) + + +class FirstWrapper: + def __init__(self, params): + self.params = params + + def __getitem__(self, key): + try: + if isinstance(key, str): + key = key.encode('iso-8859-1') + return self.params.first(key) + except KeyError: + return "" + + +@pipe(opt(nullable(str))) +def sub(request, response, escape_type="html"): + """Substitute environment information about the server and request into the script. + + :param escape_type: String detailing the type of escaping to use. Known values are + "html" and "none", with "html" the default for historic reasons. + + The format is a very limited template language. Substitutions are + enclosed by {{ and }}. There are several available substitutions: + + host + A simple string value and represents the primary host from which the + tests are being run. + domains + A dictionary of available domains indexed by subdomain name. + ports + A dictionary of lists of ports indexed by protocol. + location + A dictionary of parts of the request URL. Valid keys are + 'server, 'scheme', 'host', 'hostname', 'port', 'path' and 'query'. + 'server' is scheme://host:port, 'host' is hostname:port, and query + includes the leading '?', but other delimiters are omitted. + headers + A dictionary of HTTP headers in the request. + header_or_default(header, default) + The value of an HTTP header, or a default value if it is absent. + For example:: + + {{header_or_default(X-Test, test-header-absent)}} + + GET + A dictionary of query parameters supplied with the request. + uuid() + A pesudo-random UUID suitable for usage with stash + file_hash(algorithm, filepath) + The cryptographic hash of a file. Supported algorithms: md5, sha1, + sha224, sha256, sha384, and sha512. For example:: + + {{file_hash(md5, dom/interfaces.html)}} + + fs_path(filepath) + The absolute path to a file inside the wpt document root + + So for example in a setup running on localhost with a www + subdomain and a http server on ports 80 and 81:: + + {{host}} => localhost + {{domains[www]}} => www.localhost + {{ports[http][1]}} => 81 + + It is also possible to assign a value to a variable name, which must start + with the $ character, using the ":" syntax e.g.:: + + {{$id:uuid()}} + + Later substitutions in the same file may then refer to the variable + by name e.g.:: + + {{$id}} + """ + content = resolve_content(response) + + new_content = template(request, content, escape_type=escape_type) + + response.content = new_content + return response + +class SubFunctions: + @staticmethod + def uuid(request): + return str(uuid.uuid4()) + + # Maintain a list of supported algorithms, restricted to those that are + # available on all platforms [1]. This ensures that test authors do not + # unknowingly introduce platform-specific tests. + # + # [1] https://docs.python.org/2/library/hashlib.html + supported_algorithms = ("md5", "sha1", "sha224", "sha256", "sha384", "sha512") + + @staticmethod + def file_hash(request, algorithm, path): + assert isinstance(algorithm, str) + if algorithm not in SubFunctions.supported_algorithms: + raise ValueError("Unsupported encryption algorithm: '%s'" % algorithm) + + hash_obj = getattr(hashlib, algorithm)() + absolute_path = os.path.join(request.doc_root, path) + + try: + with open(absolute_path, "rb") as f: + hash_obj.update(f.read()) + except OSError: + # In this context, an unhandled IOError will be interpreted by the + # server as an indication that the template file is non-existent. + # Although the generic "Exception" is less precise, it avoids + # triggering a potentially-confusing HTTP 404 error in cases where + # the path to the file to be hashed is invalid. + raise Exception('Cannot open file for hash computation: "%s"' % absolute_path) + + return base64.b64encode(hash_obj.digest()).strip() + + @staticmethod + def fs_path(request, path): + if not path.startswith("/"): + subdir = request.request_path[len(request.url_base):] + if "/" in subdir: + subdir = subdir.rsplit("/", 1)[0] + root_rel_path = subdir + "/" + path + else: + root_rel_path = path[1:] + root_rel_path = root_rel_path.replace("/", os.path.sep) + absolute_path = os.path.abspath(os.path.join(request.doc_root, root_rel_path)) + if ".." in os.path.relpath(absolute_path, request.doc_root): + raise ValueError("Path outside wpt root") + return absolute_path + + @staticmethod + def header_or_default(request, name, default): + return request.headers.get(name, default) + +def template(request, content, escape_type="html"): + #TODO: There basically isn't any error handling here + tokenizer = ReplacementTokenizer() + + variables = {} + + def config_replacement(match): + content, = match.groups() + + tokens = tokenizer.tokenize(content) + tokens = deque(tokens) + + token_type, field = tokens.popleft() + assert isinstance(field, str) + + if token_type == "var": + variable = field + token_type, field = tokens.popleft() + assert isinstance(field, str) + else: + variable = None + + if token_type != "ident": + raise Exception("unexpected token type %s (token '%r'), expected ident" % (token_type, field)) + + if field in variables: + value = variables[field] + elif hasattr(SubFunctions, field): + value = getattr(SubFunctions, field) + elif field == "headers": + value = request.headers + elif field == "GET": + value = FirstWrapper(request.GET) + elif field == "hosts": + value = request.server.config.all_domains + elif field == "domains": + value = request.server.config.all_domains[""] + elif field == "host": + value = request.server.config["browser_host"] + elif field in request.server.config: + value = request.server.config[field] + elif field == "location": + value = {"server": "%s://%s:%s" % (request.url_parts.scheme, + request.url_parts.hostname, + request.url_parts.port), + "scheme": request.url_parts.scheme, + "host": "%s:%s" % (request.url_parts.hostname, + request.url_parts.port), + "hostname": request.url_parts.hostname, + "port": request.url_parts.port, + "path": request.url_parts.path, + "pathname": request.url_parts.path, + "query": "?%s" % request.url_parts.query} + elif field == "url_base": + value = request.url_base + else: + raise Exception("Undefined template variable %s" % field) + + while tokens: + ttype, field = tokens.popleft() + if ttype == "index": + value = value[field] + elif ttype == "arguments": + value = value(request, *field) + else: + raise Exception( + "unexpected token type %s (token '%r'), expected ident or arguments" % (ttype, field) + ) + + assert isinstance(value, (int, (bytes, str))), tokens + + if variable is not None: + variables[variable] = value + + escape_func = {"html": lambda x:escape(x, quote=True), + "none": lambda x:x}[escape_type] + + # Should possibly support escaping for other contexts e.g. script + # TODO: read the encoding of the response + # cgi.escape() only takes text strings in Python 3. + if isinstance(value, bytes): + value = value.decode("utf-8") + elif isinstance(value, int): + value = str(value) + return escape_func(value).encode("utf-8") + + template_regexp = re.compile(br"{{([^}]*)}}") + new_content = template_regexp.sub(config_replacement, content) + + return new_content + +@pipe() +def gzip(request, response): + """This pipe gzip-encodes response data. + + It sets (or overwrites) these HTTP headers: + Content-Encoding is set to gzip + Content-Length is set to the length of the compressed content + """ + content = resolve_content(response) + response.headers.set("Content-Encoding", "gzip") + + out = BytesIO() + with gzip_module.GzipFile(fileobj=out, mode="w") as f: + f.write(content) + response.content = out.getvalue() + + response.headers.set("Content-Length", len(response.content)) + + return response diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/ranges.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/ranges.py new file mode 100644 index 0000000000..622b807002 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/ranges.py @@ -0,0 +1,96 @@ +# mypy: allow-untyped-defs + +from .utils import HTTPException + + +class RangeParser: + def __call__(self, header, file_size): + try: + header = header.decode("ascii") + except UnicodeDecodeError: + raise HTTPException(400, "Non-ASCII range header value") + prefix = "bytes=" + if not header.startswith(prefix): + raise HTTPException(416, message=f"Unrecognised range type {header}") + + parts = header[len(prefix):].split(",") + ranges = [] + for item in parts: + components = item.split("-") + if len(components) != 2: + raise HTTPException(416, "Bad range specifier %s" % (item)) + data = [] + for component in components: + if component == "": + data.append(None) + else: + try: + data.append(int(component)) + except ValueError: + raise HTTPException(416, "Bad range specifier %s" % (item)) + try: + ranges.append(Range(data[0], data[1], file_size)) + except ValueError: + raise HTTPException(416, "Bad range specifier %s" % (item)) + + return self.coalesce_ranges(ranges, file_size) + + def coalesce_ranges(self, ranges, file_size): + rv = [] + target = None + for current in reversed(sorted(ranges)): + if target is None: + target = current + else: + new = target.coalesce(current) + target = new[0] + if len(new) > 1: + rv.append(new[1]) + rv.append(target) + + return rv[::-1] + + +class Range: + def __init__(self, lower, upper, file_size): + self.file_size = file_size + self.lower, self.upper = self._abs(lower, upper) + if self.lower >= self.upper or self.lower >= self.file_size: + raise ValueError + + def __repr__(self): + return f"" + + def __lt__(self, other): + return self.lower < other.lower + + def __gt__(self, other): + return self.lower > other.lower + + def __eq__(self, other): + return self.lower == other.lower and self.upper == other.upper + + def _abs(self, lower, upper): + if lower is None and upper is None: + lower, upper = 0, self.file_size + elif lower is None: + lower, upper = max(0, self.file_size - upper), self.file_size + elif upper is None: + lower, upper = lower, self.file_size + else: + lower, upper = lower, min(self.file_size, upper + 1) + + return lower, upper + + def coalesce(self, other): + assert self.file_size == other.file_size + + if (self.upper < other.lower or self.lower > other.upper): + return sorted([self, other]) + else: + return [Range(min(self.lower, other.lower), + max(self.upper, other.upper) - 1, + self.file_size)] + + def header_value(self): + return "bytes %i-%i/%i" % (self.lower, self.upper - 1, self.file_size) diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/request.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/request.py new file mode 100644 index 0000000000..9245b97fa5 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/request.py @@ -0,0 +1,710 @@ +# mypy: allow-untyped-defs + +import base64 +import tempfile + +from http.cookies import BaseCookie +from io import BytesIO +from typing import Dict, List, TypeVar +from urllib.parse import parse_qsl, urlsplit + +from . import stash +from .cgi import FieldStorage +from .utils import HTTPException, isomorphic_encode, isomorphic_decode + +KT = TypeVar('KT') +VT = TypeVar('VT') + +missing = object() + + +class Server: + """Data about the server environment + + .. attribute:: config + + Environment configuration information with information about the + various servers running, their hostnames and ports. + + .. attribute:: stash + + Stash object holding state stored on the server between requests. + + """ + config = None + + def __init__(self, request): + self._stash = None + self._request = request + + @property + def stash(self): + if self._stash is None: + address, authkey = stash.load_env_config() + self._stash = stash.Stash(self._request.url_parts.path, address, authkey) + return self._stash + + +class InputFile: + max_buffer_size = 1024*1024 + + def __init__(self, rfile, length): + """File-like object used to provide a seekable view of request body data""" + self._file = rfile + self.length = length + + self._file_position = 0 + + if length > self.max_buffer_size: + self._buf = tempfile.TemporaryFile() + else: + self._buf = BytesIO() + + def close(self): + self._buf.close() + + def __enter__(self): + return self + + def __exit__(self, *exc): + self.close() + return False + + @property + def _buf_position(self): + rv = self._buf.tell() + assert rv <= self._file_position + return rv + + def read(self, bytes=-1): + assert self._buf_position <= self._file_position + + if bytes < 0: + bytes = self.length - self._buf_position + bytes_remaining = min(bytes, self.length - self._buf_position) + + if bytes_remaining == 0: + return b"" + + if self._buf_position != self._file_position: + buf_bytes = min(bytes_remaining, self._file_position - self._buf_position) + old_data = self._buf.read(buf_bytes) + bytes_remaining -= buf_bytes + else: + old_data = b"" + + assert bytes_remaining == 0 or self._buf_position == self._file_position, ( + "Before reading buffer position (%i) didn't match file position (%i)" % + (self._buf_position, self._file_position)) + new_data = self._file.read(bytes_remaining) + self._buf.write(new_data) + self._file_position += bytes_remaining + assert bytes_remaining == 0 or self._buf_position == self._file_position, ( + "After reading buffer position (%i) didn't match file position (%i)" % + (self._buf_position, self._file_position)) + + return old_data + new_data + + def tell(self): + return self._buf_position + + def seek(self, offset): + if offset > self.length or offset < 0: + raise ValueError + if offset <= self._file_position: + self._buf.seek(offset) + else: + self.read(offset - self._file_position) + + def readline(self, max_bytes=None): + if max_bytes is None: + max_bytes = self.length - self._buf_position + + if self._buf_position < self._file_position: + data = self._buf.readline(max_bytes) + if data.endswith(b"\n") or len(data) == max_bytes: + return data + else: + data = b"" + + assert self._buf_position == self._file_position + + initial_position = self._file_position + found = False + buf = [] + max_bytes -= len(data) + while not found: + readahead = self.read(min(2, max_bytes)) + max_bytes -= len(readahead) + for i, c in enumerate(readahead): + if c == b"\n"[0]: + buf.append(readahead[:i+1]) + found = True + break + if not found: + buf.append(readahead) + if not readahead or not max_bytes: + break + new_data = b"".join(buf) + data += new_data + self.seek(initial_position + len(new_data)) + return data + + def readlines(self): + rv = [] + while True: + data = self.readline() + if data: + rv.append(data) + else: + break + return rv + + def __next__(self): + data = self.readline() + if data: + return data + else: + raise StopIteration + + next = __next__ + + def __iter__(self): + return self + + +class Request: + """Object representing a HTTP request. + + .. attribute:: doc_root + + The local directory to use as a base when resolving paths + + .. attribute:: route_match + + Regexp match object from matching the request path to the route + selected for the request. + + .. attribute:: client_address + + Contains a tuple of the form (host, port) representing the client's address. + + .. attribute:: protocol_version + + HTTP version specified in the request. + + .. attribute:: method + + HTTP method in the request. + + .. attribute:: request_path + + Request path as it appears in the HTTP request. + + .. attribute:: url_base + + The prefix part of the path; typically / unless the handler has a url_base set + + .. attribute:: url + + Absolute URL for the request. + + .. attribute:: url_parts + + Parts of the requested URL as obtained by urlparse.urlsplit(path) + + .. attribute:: request_line + + Raw request line + + .. attribute:: headers + + RequestHeaders object providing a dictionary-like representation of + the request headers. + + .. attribute:: raw_headers. + + Dictionary of non-normalized request headers. + + .. attribute:: body + + Request body as a string + + .. attribute:: raw_input + + File-like object representing the body of the request. + + .. attribute:: GET + + MultiDict representing the parameters supplied with the request. + Note that these may be present on non-GET requests; the name is + chosen to be familiar to users of other systems such as PHP. + Both keys and values are binary strings. + + .. attribute:: POST + + MultiDict representing the request body parameters. Most parameters + are present as string values, but file uploads have file-like + values. All string values (including keys) have binary type. + + .. attribute:: cookies + + A Cookies object representing cookies sent with the request with a + dictionary-like interface. + + .. attribute:: auth + + An instance of Authentication with username and password properties + representing any credentials supplied using HTTP authentication. + + .. attribute:: server + + Server object containing information about the server environment. + """ + + def __init__(self, request_handler): + self.doc_root = request_handler.server.router.doc_root + self.route_match = None # Set by the router + self.client_address = request_handler.client_address + + self.protocol_version = request_handler.protocol_version + self.method = request_handler.command + + # Keys and values in raw headers are native strings. + self._headers = None + self.raw_headers = request_handler.headers + + scheme = request_handler.server.scheme + host = self.raw_headers.get("Host") + port = request_handler.server.server_address[1] + + if host is None: + host = request_handler.server.server_address[0] + else: + if ":" in host: + host, port = host.split(":", 1) + + self.request_path = request_handler.path + self.url_base = "/" + + if self.request_path.startswith(scheme + "://"): + self.url = self.request_path + else: + # TODO(#23362): Stop using native strings for URLs. + self.url = "%s://%s:%s%s" % ( + scheme, host, port, self.request_path) + self.url_parts = urlsplit(self.url) + + self.request_line = request_handler.raw_requestline + + self.raw_input = InputFile(request_handler.rfile, + int(self.raw_headers.get("Content-Length", 0))) + + self._body = None + + self._GET = None + self._POST = None + self._cookies = None + self._auth = None + + self.server = Server(self) + + def close(self): + return self.raw_input.close() + + def __enter__(self): + return self + + def __exit__(self, *exc): + self.close() + return False + + def __repr__(self): + return "" % (self.method, self.url) + + @property + def GET(self): + if self._GET is None: + kwargs = { + "keep_blank_values": True, + "encoding": "iso-8859-1", + } + params = parse_qsl(self.url_parts.query, **kwargs) + self._GET = MultiDict() + for key, value in params: + self._GET.add(isomorphic_encode(key), isomorphic_encode(value)) + return self._GET + + @property + def POST(self): + if self._POST is None: + # Work out the post parameters + pos = self.raw_input.tell() + self.raw_input.seek(0) + kwargs = { + "fp": self.raw_input, + "environ": {"REQUEST_METHOD": self.method}, + "headers": self.raw_headers, + "keep_blank_values": True, + "encoding": "iso-8859-1", + } + fs = FieldStorage(**kwargs) + self._POST = MultiDict.from_field_storage(fs) + self.raw_input.seek(pos) + return self._POST + + @property + def cookies(self): + if self._cookies is None: + parser = BinaryCookieParser() + cookie_headers = self.headers.get("cookie", b"") + parser.load(cookie_headers) + cookies = Cookies() + for key, value in parser.items(): + cookies[isomorphic_encode(key)] = CookieValue(value) + self._cookies = cookies + return self._cookies + + @property + def headers(self): + if self._headers is None: + self._headers = RequestHeaders(self.raw_headers) + return self._headers + + @property + def body(self): + if self._body is None: + pos = self.raw_input.tell() + self.raw_input.seek(0) + self._body = self.raw_input.read() + self.raw_input.seek(pos) + return self._body + + @property + def auth(self): + if self._auth is None: + self._auth = Authentication(self.headers) + return self._auth + + +class H2Request(Request): + def __init__(self, request_handler): + self.h2_stream_id = request_handler.h2_stream_id + self.frames = [] + super().__init__(request_handler) + + +class RequestHeaders(Dict[bytes, List[bytes]]): + """Read-only dictionary-like API for accessing request headers. + + Unlike BaseHTTPRequestHandler.headers, this class always returns all + headers with the same name (separated by commas). And it ensures all keys + (i.e. names of headers) and values have binary type. + """ + def __init__(self, items): + for header in items.keys(): + key = isomorphic_encode(header).lower() + # get all headers with the same name + values = items.getallmatchingheaders(header) + if len(values) > 1: + # collect the multiple variations of the current header + multiples = [] + # loop through the values from getallmatchingheaders + for value in values: + # getallmatchingheaders returns raw header lines, so + # split to get name, value + multiples.append(isomorphic_encode(value).split(b':', 1)[1].strip()) + headers = multiples + else: + headers = [isomorphic_encode(items[header])] + dict.__setitem__(self, key, headers) + + def __getitem__(self, key): + """Get all headers of a certain (case-insensitive) name. If there is + more than one, the values are returned comma separated""" + key = isomorphic_encode(key) + values = dict.__getitem__(self, key.lower()) + if len(values) == 1: + return values[0] + else: + return b", ".join(values) + + def __setitem__(self, name, value): + raise Exception + + def get(self, key, default=None): + """Get a string representing all headers with a particular value, + with multiple headers separated by a comma. If no header is found + return a default value + + :param key: The header name to look up (case-insensitive) + :param default: The value to return in the case of no match + """ + try: + return self[key] + except KeyError: + return default + + def get_list(self, key, default=missing): + """Get all the header values for a particular field name as + a list""" + key = isomorphic_encode(key) + try: + return dict.__getitem__(self, key.lower()) + except KeyError: + if default is not missing: + return default + else: + raise + + def __contains__(self, key): + key = isomorphic_encode(key) + return dict.__contains__(self, key.lower()) + + def iteritems(self): + for item in self: + yield item, self[item] + + def itervalues(self): + for item in self: + yield self[item] + + +class CookieValue: + """Representation of cookies. + + Note that cookies are considered read-only and the string value + of the cookie will not change if you update the field values. + However this is not enforced. + + .. attribute:: key + + The name of the cookie. + + .. attribute:: value + + The value of the cookie + + .. attribute:: expires + + The expiry date of the cookie + + .. attribute:: path + + The path of the cookie + + .. attribute:: comment + + The comment of the cookie. + + .. attribute:: domain + + The domain with which the cookie is associated + + .. attribute:: max_age + + The max-age value of the cookie. + + .. attribute:: secure + + Whether the cookie is marked as secure + + .. attribute:: httponly + + Whether the cookie is marked as httponly + + """ + def __init__(self, morsel): + self.key = morsel.key + self.value = morsel.value + + for attr in ["expires", "path", + "comment", "domain", "max-age", + "secure", "version", "httponly"]: + setattr(self, attr.replace("-", "_"), morsel[attr]) + + self._str = morsel.OutputString() + + def __str__(self): + return self._str + + def __repr__(self): + return self._str + + def __eq__(self, other): + """Equality comparison for cookies. Compares to other cookies + based on value alone and on non-cookies based on the equality + of self.value with the other object so that a cookie with value + "ham" compares equal to the string "ham" + """ + if hasattr(other, "value"): + return self.value == other.value + return self.value == other + + +class MultiDict(Dict[KT, VT]): + """Dictionary type that holds multiple values for each key""" + # TODO: this should perhaps also order the keys + def __init__(self): + pass + + def __setitem__(self, name, value): + dict.__setitem__(self, name, [value]) + + def add(self, name, value): + if name in self: + dict.__getitem__(self, name).append(value) + else: + dict.__setitem__(self, name, [value]) + + def __getitem__(self, key): + """Get the first value with a given key""" + return self.first(key) + + def first(self, key, default=missing): + """Get the first value with a given key + + :param key: The key to lookup + :param default: The default to return if key is + not found (throws if nothing is + specified) + """ + if key in self and dict.__getitem__(self, key): + return dict.__getitem__(self, key)[0] + elif default is not missing: + return default + raise KeyError(key) + + def last(self, key, default=missing): + """Get the last value with a given key + + :param key: The key to lookup + :param default: The default to return if key is + not found (throws if nothing is + specified) + """ + if key in self and dict.__getitem__(self, key): + return dict.__getitem__(self, key)[-1] + elif default is not missing: + return default + raise KeyError(key) + + # We need to explicitly override dict.get; otherwise, it won't call + # __getitem__ and would return a list instead. + def get(self, key, default=None): + """Get the first value with a given key + + :param key: The key to lookup + :param default: The default to return if key is + not found (None by default) + """ + return self.first(key, default) + + def get_list(self, key): + """Get all values with a given key as a list + + :param key: The key to lookup + """ + if key in self: + return dict.__getitem__(self, key) + else: + return [] + + @classmethod + def from_field_storage(cls, fs): + """Construct a MultiDict from a FieldStorage + + Note that all keys and values are binary strings. + """ + self = cls() + if fs.list is None: + return self + for key in fs: + values = fs[key] + if not isinstance(values, list): + values = [values] + + for value in values: + if not value.filename: + value = isomorphic_encode(value.value) + else: + assert isinstance(value, FieldStorage) + self.add(isomorphic_encode(key), value) + return self + + +class BinaryCookieParser(BaseCookie): # type: ignore + """A subclass of BaseCookie that returns values in binary strings + + This is not intended to store the cookies; use Cookies instead. + """ + def value_decode(self, val): + """Decode value from network to (real_value, coded_value). + + Override BaseCookie.value_decode. + """ + return isomorphic_encode(val), val + + def value_encode(self, val): + raise NotImplementedError('BinaryCookieParser is not for setting cookies') + + def load(self, rawdata): + """Load cookies from a binary string. + + This overrides and calls BaseCookie.load. Unlike BaseCookie.load, it + does not accept dictionaries. + """ + assert isinstance(rawdata, bytes) + # BaseCookie.load expects a native string + super().load(isomorphic_decode(rawdata)) + + +class Cookies(MultiDict[bytes, CookieValue]): + """MultiDict specialised for Cookie values + + Keys are binary strings and values are CookieValue objects. + """ + def __init__(self): + pass + + def __getitem__(self, key): + return self.last(key) + + +class Authentication: + """Object for dealing with HTTP Authentication + + .. attribute:: username + + The username supplied in the HTTP Authorization + header, or None + + .. attribute:: password + + The password supplied in the HTTP Authorization + header, or None + + Both attributes are binary strings (`str` in Py2, `bytes` in Py3), since + RFC7617 Section 2.1 does not specify the encoding for username & password + (as long it's compatible with ASCII). UTF-8 should be a relatively safe + choice if callers need to decode them as most browsers use it. + """ + def __init__(self, headers): + self.username = None + self.password = None + + auth_schemes = {b"Basic": self.decode_basic} + + if "authorization" in headers: + header = headers.get("authorization") + assert isinstance(header, bytes) + auth_type, data = header.split(b" ", 1) + if auth_type in auth_schemes: + self.username, self.password = auth_schemes[auth_type](data) + else: + raise HTTPException(400, "Unsupported authentication scheme %s" % auth_type) + + def decode_basic(self, data): + assert isinstance(data, bytes) + decoded_data = base64.b64decode(data) + return decoded_data.split(b":", 1) diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/response.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/response.py new file mode 100644 index 0000000000..8d0e01bbdd --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/response.py @@ -0,0 +1,842 @@ +# mypy: allow-untyped-defs + +import json +import uuid +import traceback +from collections import OrderedDict +from datetime import datetime, timedelta, timezone +from io import BytesIO + +from hpack.struct import HeaderTuple +from http.cookies import BaseCookie, Morsel +from hyperframe.frame import HeadersFrame, DataFrame, ContinuationFrame + +from .constants import response_codes, h2_headers +from .logger import get_logger +from .utils import isomorphic_decode, isomorphic_encode + +missing = object() + + +class Response: + """Object representing the response to a HTTP request + + :param handler: RequestHandler being used for this response + :param request: Request that this is the response for + + .. attribute:: request + + Request associated with this Response. + + .. attribute:: encoding + + The encoding to use when converting unicode to strings for output. + + .. attribute:: add_required_headers + + Boolean indicating whether mandatory headers should be added to the + response. + + .. attribute:: send_body_for_head_request + + Boolean, default False, indicating whether the body content should be + sent when the request method is HEAD. + + .. attribute:: writer + + The ResponseWriter for this response + + .. attribute:: status + + Status tuple (code, message). Can be set to an integer in which case the + message part is filled in automatically, or a tuple (code, message) in + which case code is an int and message is a text or binary string. + + .. attribute:: headers + + List of HTTP headers to send with the response. Each item in the list is a + tuple of (name, value). + + .. attribute:: content + + The body of the response. This can either be a string or a iterable of response + parts. If it is an iterable, any item may be a string or a function of zero + parameters which, when called, returns a string.""" + + def __init__(self, handler, request, response_writer_cls=None): + self.request = request + self.encoding = "utf8" + + self.add_required_headers = True + self.send_body_for_head_request = False + self.close_connection = False + + self.logger = get_logger() + self.writer = response_writer_cls(handler, self) if response_writer_cls else ResponseWriter(handler, self) + + self._status = (200, None) + self.headers = ResponseHeaders() + self.content = [] + + @property + def status(self): + return self._status + + @status.setter + def status(self, value): + if hasattr(value, "__len__"): + if len(value) != 2: + raise ValueError + else: + code = int(value[0]) + message = value[1] + # Only call str() if message is not a string type, so that we + # don't get `str(b"foo") == "b'foo'"` in Python 3. + if not isinstance(message, (bytes, str)): + message = str(message) + self._status = (code, message) + else: + self._status = (int(value), None) + + def set_cookie(self, name, value, path="/", domain=None, max_age=None, + expires=None, samesite=None, secure=False, httponly=False, + comment=None): + """Set a cookie to be sent with a Set-Cookie header in the + response + + :param name: name of the cookie (a binary string) + :param value: value of the cookie (a binary string, or None) + :param max_age: datetime.timedelta int representing the time (in seconds) + until the cookie expires + :param path: String path to which the cookie applies + :param domain: String domain to which the cookie applies + :param samesit: String indicating whether the cookie should be + restricted to same site context + :param secure: Boolean indicating whether the cookie is marked as secure + :param httponly: Boolean indicating whether the cookie is marked as + HTTP Only + :param comment: String comment + :param expires: datetime.datetime or datetime.timedelta indicating a + time or interval from now when the cookie expires + + """ + # TODO(Python 3): Convert other parameters (e.g. path) to bytes, too. + if value is None: + value = b'' + max_age = 0 + expires = timedelta(days=-1) + + name = isomorphic_decode(name) + value = isomorphic_decode(value) + + days = {i+1: name for i, name in enumerate(["jan", "feb", "mar", + "apr", "may", "jun", + "jul", "aug", "sep", + "oct", "nov", "dec"])} + + if isinstance(expires, timedelta): + expires = datetime.now(timezone.utc) + expires + + if expires is not None: + expires_str = expires.strftime("%d %%s %Y %H:%M:%S GMT") + expires_str = expires_str % days[expires.month] + expires = expires_str + + if max_age is not None: + if hasattr(max_age, "total_seconds"): + max_age = int(max_age.total_seconds()) + max_age = "%.0d" % max_age + + m = Morsel() + + def maybe_set(key, value): + if value is not None and value is not False: + m[key] = value + + m.set(name, value, value) + maybe_set("path", path) + maybe_set("domain", domain) + maybe_set("comment", comment) + maybe_set("expires", expires) + maybe_set("max-age", max_age) + maybe_set("secure", secure) + maybe_set("httponly", httponly) + maybe_set("samesite", samesite) + + self.headers.append("Set-Cookie", m.OutputString()) + + def unset_cookie(self, name): + """Remove a cookie from those that are being sent with the response""" + name = isomorphic_decode(name) + cookies = self.headers.get("Set-Cookie") + parser = BaseCookie() + for cookie in cookies: + parser.load(isomorphic_decode(cookie)) + + if name in parser.keys(): + del self.headers["Set-Cookie"] + for m in parser.values(): + if m.key != name: + self.headers.append(("Set-Cookie", m.OutputString())) + + def delete_cookie(self, name, path="/", domain=None): + """Delete a cookie on the client by setting it to the empty string + and to expire in the past""" + self.set_cookie(name, None, path=path, domain=domain, max_age=0, + expires=timedelta(days=-1)) + + def iter_content(self, read_file=False): + """Iterator returning chunks of response body content. + + If any part of the content is a function, this will be called + and the resulting value (if any) returned. + + :param read_file: boolean controlling the behaviour when content is a + file handle. When set to False the handle will be + returned directly allowing the file to be passed to + the output in small chunks. When set to True, the + entire content of the file will be returned as a + string facilitating non-streaming operations like + template substitution. + """ + if isinstance(self.content, bytes): + yield self.content + elif isinstance(self.content, str): + yield self.content.encode(self.encoding) + elif hasattr(self.content, "read"): + # Read the file in chunks rather than reading the whole file into + # memory at once. (See also ResponseWriter.file_chunk_size) + while True: + read = self.content.read(32 * 1024) + if len(read) == 0: + break + yield read + self.content.close() + else: + for item in self.content: + if hasattr(item, "__call__"): + value = item() + else: + value = item + if value: + yield value + + def write_status_headers(self): + """Write out the status line and headers for the response""" + self.writer.write_status(*self.status) + for item in self.headers: + self.writer.write_header(*item) + self.writer.end_headers() + + def write_content(self): + """Write out the response content""" + if self.request.method != "HEAD" or self.send_body_for_head_request: + for item in self.iter_content(): + self.writer.write_content(item) + + def write(self): + """Write the whole response""" + self.write_status_headers() + self.write_content() + + def set_error(self, code, err=None): + """Set the response status headers and return a JSON error object: + + {"error": {"code": code, "message": message}} + code is an int (HTTP status code), and message is a text string. + """ + if 500 <= code < 600: + message = self._format_server_error(err) + self.logger.warning(message) + else: + if err is None: + message = "" + else: + message = str(err) + + data = json.dumps({"error": { + "code": code, + "message": message} + }) + self.status = code + self.headers = [("Content-Type", "application/json"), + ("Content-Length", len(data))] + self.content = data + + def _format_server_error(self, err): + if err is None: + suffix = "" + elif isinstance(err, str): + suffix = err + elif self.request.server.config.logging["suppress_handler_traceback"]: + frame = traceback.extract_tb(err.__traceback__)[-1] + suffix = (f"""File "{frame.filename}", line {frame.lineno} """ + f"""in {frame.name} (traceback suppressed)""") + else: + tb = "\n".join(f" {line}" + for line in traceback.format_tb(err.__traceback__)) + suffix = f"""Traceback (most recent call last): +{tb} {type(err).__name__}: {err} +""" + return f"Internal server error loading {self.request.url}:\n {suffix}" + + +class MultipartContent: + def __init__(self, boundary=None, default_content_type=None): + self.items = [] + if boundary is None: + boundary = str(uuid.uuid4()) + self.boundary = boundary + self.default_content_type = default_content_type + + def __call__(self): + boundary = b"--" + self.boundary.encode("ascii") + rv = [b"", boundary] + for item in self.items: + rv.append(item.to_bytes()) + rv.append(boundary) + rv[-1] += b"--" + return b"\r\n".join(rv) + + def append_part(self, data, content_type=None, headers=None): + if content_type is None: + content_type = self.default_content_type + self.items.append(MultipartPart(data, content_type, headers)) + + def __iter__(self): + #This is hackish; when writing the response we need an iterable + #or a string. For a multipart/byterange response we want an + #iterable that contains a single callable; the MultipartContent + #object itself + yield self + + +class MultipartPart: + def __init__(self, data, content_type=None, headers=None): + assert isinstance(data, bytes), data + self.headers = ResponseHeaders() + + if content_type is not None: + self.headers.set("Content-Type", content_type) + + if headers is not None: + for name, value in headers: + if name.lower() == b"content-type": + func = self.headers.set + else: + func = self.headers.append + func(name, value) + + self.data = data + + def to_bytes(self): + rv = [] + for key, value in self.headers: + assert isinstance(key, bytes) + assert isinstance(value, bytes) + rv.append(b"%s: %s" % (key, value)) + rv.append(b"") + rv.append(self.data) + return b"\r\n".join(rv) + + +def _maybe_encode(s): + """Encode a string or an int into binary data using isomorphic_encode().""" + if isinstance(s, int): + return b"%i" % (s,) + return isomorphic_encode(s) + + +class ResponseHeaders: + """Dictionary-like object holding the headers for the response""" + def __init__(self): + self.data = OrderedDict() + + def set(self, key, value): + """Set a header to a specific value, overwriting any previous header + with the same name + + :param key: Name of the header to set + :param value: Value to set the header to + """ + key = _maybe_encode(key) + value = _maybe_encode(value) + self.data[key.lower()] = (key, [value]) + + def append(self, key, value): + """Add a new header with a given name, not overwriting any existing + headers with the same name + + :param key: Name of the header to add + :param value: Value to set for the header + """ + key = _maybe_encode(key) + value = _maybe_encode(value) + if key.lower() in self.data: + self.data[key.lower()][1].append(value) + else: + self.set(key, value) + + def get(self, key, default=missing): + """Get the set values for a particular header.""" + key = _maybe_encode(key) + try: + return self[key] + except KeyError: + if default is missing: + return [] + return default + + def __getitem__(self, key): + """Get a list of values for a particular header + + """ + key = _maybe_encode(key) + return self.data[key.lower()][1] + + def __delitem__(self, key): + key = _maybe_encode(key) + del self.data[key.lower()] + + def __contains__(self, key): + key = _maybe_encode(key) + return key.lower() in self.data + + def __setitem__(self, key, value): + self.set(key, value) + + def __iter__(self): + for key, values in self.data.values(): + for value in values: + yield key, value + + def items(self): + return list(self) + + def update(self, items_iter): + for name, value in items_iter: + self.append(name, value) + + def __repr__(self): + return repr(self.data) + + +class H2Response(Response): + + def __init__(self, handler, request): + super().__init__(handler, request, response_writer_cls=H2ResponseWriter) + + def write_status_headers(self): + self.writer.write_headers(self.headers, *self.status) + + # Hacky way of detecting last item in generator + def write_content(self): + """Write out the response content""" + if self.request.method != "HEAD" or self.send_body_for_head_request: + item = None + item_iter = self.iter_content() + try: + item = next(item_iter) + while True: + check_last = next(item_iter) + self.writer.write_data(item, last=False) + item = check_last + except StopIteration: + if item: + self.writer.write_data(item, last=True) + + +class H2ResponseWriter: + + def __init__(self, handler, response): + self.socket = handler.request + self.h2conn = handler.conn + self._response = response + self._handler = handler + self.stream_ended = False + self.content_written = False + self.request = response.request + self.logger = response.logger + + def write_headers(self, headers, status_code, status_message=None, stream_id=None, last=False): + """ + Send a HEADER frame that is tracked by the local state machine. + + Write a HEADER frame using the H2 Connection object, will only work if the stream is in a state to send + HEADER frames. + + :param headers: List of (header, value) tuples + :param status_code: The HTTP status code of the response + :param stream_id: Id of stream to send frame on. Will use the request stream ID if None + :param last: Flag to signal if this is the last frame in stream. + """ + formatted_headers = [] + secondary_headers = [] # Non ':' prefixed headers are to be added afterwards + + for header, value in headers: + # h2_headers are native strings + # header field names are strings of ASCII + if isinstance(header, bytes): + header = header.decode('ascii') + # value in headers can be either string or integer + if isinstance(value, bytes): + value = self.decode(value) + if header in h2_headers: + header = ':' + header + formatted_headers.append((header, str(value))) + else: + secondary_headers.append((header, str(value))) + + formatted_headers.append((':status', str(status_code))) + formatted_headers.extend(secondary_headers) + + with self.h2conn as connection: + connection.send_headers( + stream_id=self.request.h2_stream_id if stream_id is None else stream_id, + headers=formatted_headers, + end_stream=last or self.request.method == "HEAD" + ) + + self.write(connection) + + def write_data(self, item, last=False, stream_id=None): + """ + Send a DATA frame that is tracked by the local state machine. + + Write a DATA frame using the H2 Connection object, will only work if the stream is in a state to send + DATA frames. Uses flow control to split data into multiple data frames if it exceeds the size that can + be in a single frame. + + :param item: The content of the DATA frame + :param last: Flag to signal if this is the last frame in stream. + :param stream_id: Id of stream to send frame on. Will use the request stream ID if None + """ + if isinstance(item, (str, bytes)): + data = BytesIO(self.encode(item)) + else: + data = item + + # Find the length of the data + data.seek(0, 2) + data_len = data.tell() + data.seek(0) + + # If the data is longer than max payload size, need to write it in chunks + payload_size = self.get_max_payload_size() + while data_len > payload_size: + self.write_data_frame(data.read(payload_size), False, stream_id) + data_len -= payload_size + payload_size = self.get_max_payload_size() + + self.write_data_frame(data.read(), last, stream_id) + + def write_data_frame(self, data, last, stream_id=None): + with self.h2conn as connection: + connection.send_data( + stream_id=self.request.h2_stream_id if stream_id is None else stream_id, + data=data, + end_stream=last, + ) + self.write(connection) + self.stream_ended = last + + def write_push(self, promise_headers, push_stream_id=None, status=None, response_headers=None, response_data=None): + """Write a push promise, and optionally write the push content. + + This will write a push promise to the request stream. If you do not provide headers and data for the response, + then no response will be pushed, and you should push them yourself using the ID returned from this function + + :param promise_headers: A list of header tuples that matches what the client would use to + request the pushed response + :param push_stream_id: The ID of the stream the response should be pushed to. If none given, will + use the next available id. + :param status: The status code of the response, REQUIRED if response_headers given + :param response_headers: The headers of the response + :param response_data: The response data. + :return: The ID of the push stream + """ + with self.h2conn as connection: + push_stream_id = push_stream_id if push_stream_id is not None else connection.get_next_available_stream_id() + connection.push_stream(self.request.h2_stream_id, push_stream_id, promise_headers) + self.write(connection) + + has_data = response_data is not None + if response_headers is not None: + assert status is not None + self.write_headers(response_headers, status, stream_id=push_stream_id, last=not has_data) + + if has_data: + self.write_data(response_data, last=True, stream_id=push_stream_id) + + return push_stream_id + + def end_stream(self, stream_id=None): + """Ends the stream with the given ID, or the one that request was made on if no ID given.""" + with self.h2conn as connection: + connection.end_stream(stream_id if stream_id is not None else self.request.h2_stream_id) + self.write(connection) + self.stream_ended = True + + def write_raw_header_frame(self, headers, stream_id=None, end_stream=False, end_headers=False, frame_cls=HeadersFrame): + """ + Ignores the statemachine of the stream and sends a HEADER frame regardless. + + Unlike `write_headers`, this does not check to see if a stream is in the correct state to have HEADER frames + sent through to it. It will build a HEADER frame and send it without using the H2 Connection object other than + to HPACK encode the headers. + + :param headers: List of (header, value) tuples + :param stream_id: Id of stream to send frame on. Will use the request stream ID if None + :param end_stream: Set to True to add END_STREAM flag to frame + :param end_headers: Set to True to add END_HEADERS flag to frame + """ + if not stream_id: + stream_id = self.request.h2_stream_id + + header_t = [] + for header, value in headers: + header_t.append(HeaderTuple(header, value)) + + with self.h2conn as connection: + frame = frame_cls(stream_id, data=connection.encoder.encode(header_t)) + + if end_stream: + self.stream_ended = True + frame.flags.add('END_STREAM') + if end_headers: + frame.flags.add('END_HEADERS') + + data = frame.serialize() + self.write_raw(data) + + def write_raw_data_frame(self, data, stream_id=None, end_stream=False): + """ + Ignores the statemachine of the stream and sends a DATA frame regardless. + + Unlike `write_data`, this does not check to see if a stream is in the correct state to have DATA frames + sent through to it. It will build a DATA frame and send it without using the H2 Connection object. It will + not perform any flow control checks. + + :param data: The data to be sent in the frame + :param stream_id: Id of stream to send frame on. Will use the request stream ID if None + :param end_stream: Set to True to add END_STREAM flag to frame + """ + if not stream_id: + stream_id = self.request.h2_stream_id + + frame = DataFrame(stream_id, data=data) + + if end_stream: + self.stream_ended = True + frame.flags.add('END_STREAM') + + data = frame.serialize() + self.write_raw(data) + + def write_raw_continuation_frame(self, headers, stream_id=None, end_headers=False): + """ + Ignores the statemachine of the stream and sends a CONTINUATION frame regardless. + + This provides the ability to create and write a CONTINUATION frame to the stream, which is not exposed by + `write_headers` as the h2 library handles the split between HEADER and CONTINUATION internally. Will perform + HPACK encoding on the headers. + + :param headers: List of (header, value) tuples + :param stream_id: Id of stream to send frame on. Will use the request stream ID if None + :param end_headers: Set to True to add END_HEADERS flag to frame + """ + self.write_raw_header_frame(headers, stream_id=stream_id, end_headers=end_headers, frame_cls=ContinuationFrame) + + + def get_max_payload_size(self, stream_id=None): + """Returns the maximum size of a payload for the given stream.""" + stream_id = stream_id if stream_id is not None else self.request.h2_stream_id + with self.h2conn as connection: + return min(connection.remote_settings.max_frame_size, connection.local_flow_control_window(stream_id)) - 9 + + def write(self, connection): + self.content_written = True + data = connection.data_to_send() + self.socket.sendall(data) + + def write_raw(self, raw_data): + """Used for sending raw bytes/data through the socket""" + + self.content_written = True + self.socket.sendall(raw_data) + + def decode(self, data): + """Convert bytes to unicode according to response.encoding.""" + if isinstance(data, bytes): + return data.decode(self._response.encoding) + elif isinstance(data, str): + return data + else: + raise ValueError(type(data)) + + def encode(self, data): + """Convert unicode to bytes according to response.encoding.""" + if isinstance(data, bytes): + return data + elif isinstance(data, str): + return data.encode(self._response.encoding) + else: + raise ValueError + + +class ResponseWriter: + """Object providing an API to write out a HTTP response. + + :param handler: The RequestHandler being used. + :param response: The Response associated with this writer.""" + def __init__(self, handler, response): + self._wfile = handler.wfile + self._response = response + self._handler = handler + self._status_written = False + self._headers_seen = set() + self._headers_complete = False + self.content_written = False + self.request = response.request + self.file_chunk_size = 32 * 1024 + self.default_status = 200 + + def _seen_header(self, name): + return self.encode(name.lower()) in self._headers_seen + + def write_status(self, code, message=None): + """Write out the status line of a response. + + :param code: The integer status code of the response. + :param message: The message of the response. Defaults to the message commonly used + with the status code.""" + if message is None: + if code in response_codes: + message = response_codes[code][0] + else: + message = '' + self.write(b"%s %d %s\r\n" % + (isomorphic_encode(self._response.request.protocol_version), code, isomorphic_encode(message))) + self._status_written = True + + def write_header(self, name, value): + """Write out a single header for the response. + + If a status has not been written, a default status will be written (currently 200) + + :param name: Name of the header field + :param value: Value of the header field + :return: A boolean indicating whether the write succeeds + """ + if not self._status_written: + self.write_status(self.default_status) + self._headers_seen.add(self.encode(name.lower())) + if not self.write(name): + return False + if not self.write(b": "): + return False + if isinstance(value, int): + if not self.write(str(value)): + return False + elif not self.write(value): + return False + return self.write(b"\r\n") + + def write_default_headers(self): + for name, f in [("Server", self._handler.version_string), + ("Date", self._handler.date_time_string)]: + if not self._seen_header(name): + if not self.write_header(name, f()): + return False + + if (isinstance(self._response.content, (bytes, str)) and + not self._seen_header("content-length")): + #Would be nice to avoid double-encoding here + if not self.write_header("Content-Length", len(self.encode(self._response.content))): + return False + + return True + + def end_headers(self): + """Finish writing headers and write the separator. + + Unless add_required_headers on the response is False, + this will also add HTTP-mandated headers that have not yet been supplied + to the response headers. + :return: A boolean indicating whether the write succeeds + """ + + if self._response.add_required_headers: + if not self.write_default_headers(): + return False + + if not self.write("\r\n"): + return False + if not self._seen_header("content-length"): + self._response.close_connection = True + self._headers_complete = True + + return True + + def write_content(self, data): + """Write the body of the response. + + HTTP-mandated headers will be automatically added with status default to 200 if they have + not been explicitly set. + :return: A boolean indicating whether the write succeeds + """ + if not self._status_written: + self.write_status(self.default_status) + if not self._headers_complete: + self._response.content = data + self.end_headers() + return self.write_raw_content(data) + + def write_raw_content(self, data): + """Writes the data 'as is'""" + if data is None: + raise ValueError('data cannot be None') + if isinstance(data, (str, bytes)): + # Deliberately allows both text and binary types. See `self.encode`. + return self.write(data) + else: + return self.write_content_file(data) + + def write(self, data): + """Write directly to the response, converting unicode to bytes + according to response.encoding. + :return: A boolean indicating whether the write succeeds + """ + self.content_written = True + try: + self._wfile.write(self.encode(data)) + return True + except OSError: + # This can happen if the socket got closed by the remote end + return False + + def write_content_file(self, data): + """Write a file-like object directly to the response in chunks.""" + self.content_written = True + success = True + while True: + buf = data.read(self.file_chunk_size) + if not buf: + success = False + break + try: + self._wfile.write(buf) + except OSError: + success = False + break + data.close() + return success + + def encode(self, data): + """Convert unicode to bytes according to response.encoding.""" + if isinstance(data, bytes): + return data + elif isinstance(data, str): + return data.encode(self._response.encoding) + else: + raise ValueError("data %r should be text or binary, but is %s" % (data, type(data))) diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/router.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/router.py new file mode 100644 index 0000000000..92c1b04a46 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/router.py @@ -0,0 +1,180 @@ +# mypy: allow-untyped-defs + +import itertools +import re +import sys + +from .logger import get_logger + +any_method = object() + +class RouteTokenizer: + def literal(self, scanner, token): + return ("literal", token) + + def slash(self, scanner, token): + return ("slash", None) + + def group(self, scanner, token): + return ("group", token[1:-1]) + + def star(self, scanner, token): + return ("star", token[1:-3]) + + def scan(self, input_str): + scanner = re.Scanner([(r"/", self.slash), + (r"{\w*}", self.group), + (r"\*", self.star), + (r"(?:\\.|[^{\*/])*", self.literal),]) + return scanner.scan(input_str) + +class RouteCompiler: + def __init__(self): + self.reset() + + def reset(self): + self.star_seen = False + + def compile(self, tokens): + self.reset() + + func_map = {"slash":self.process_slash, + "literal":self.process_literal, + "group":self.process_group, + "star":self.process_star} + + re_parts = ["^"] + + if not tokens or tokens[0][0] != "slash": + tokens = itertools.chain([("slash", None)], tokens) + + for token in tokens: + re_parts.append(func_map[token[0]](token)) + + if self.star_seen: + re_parts.append(")") + re_parts.append("$") + + return re.compile("".join(re_parts)) + + def process_literal(self, token): + return re.escape(token[1]) + + def process_slash(self, token): + return "/" + + def process_group(self, token): + if self.star_seen: + raise ValueError("Group seen after star in regexp") + return "(?P<%s>[^/]+)" % token[1] + + def process_star(self, token): + if self.star_seen: + raise ValueError("Star seen after star in regexp") + self.star_seen = True + return "(.*" + +def compile_path_match(route_pattern): + """tokens: / or literal or match or *""" + + tokenizer = RouteTokenizer() + tokens, unmatched = tokenizer.scan(route_pattern) + + assert unmatched == "", unmatched + + compiler = RouteCompiler() + + return compiler.compile(tokens) + +class Router: + """Object for matching handler functions to requests. + + :param doc_root: Absolute path of the filesystem location from + which to serve tests + :param routes: Initial routes to add; a list of three item tuples + (method, path_pattern, handler_function), defined + as for register() + """ + + def __init__(self, doc_root, routes): + self.doc_root = doc_root + self.routes = [] + self.logger = get_logger() + + # Add the doc_root to the Python path, so that any Python handler can + # correctly locate helper scripts (see RFC_TO_BE_LINKED). + # + # TODO: In a perfect world, Router would not need to know about this + # and the handler itself would take care of it. Currently, however, we + # treat handlers like functions and so there's no easy way to do that. + if self.doc_root not in sys.path: + sys.path.insert(0, self.doc_root) + + for route in reversed(routes): + self.register(*route) + + def register(self, methods, path, handler): + r"""Register a handler for a set of paths. + + :param methods: Set of methods this should match. "*" is a + special value indicating that all methods should + be matched. + + :param path_pattern: Match pattern that will be used to determine if + a request path matches this route. Match patterns + consist of either literal text, match groups, + denoted {name}, which match any character except /, + and, at most one \*, which matches and character and + creates a match group to the end of the string. + If there is no leading "/" on the pattern, this is + automatically implied. For example:: + + api/{resource}/*.json + + Would match `/api/test/data.json` or + `/api/test/test2/data.json`, but not `/api/test/data.py`. + + The match groups are made available in the request object + as a dictionary through the route_match property. For + example, given the route pattern above and the path + `/api/test/data.json`, the route_match property would + contain:: + + {"resource": "test", "*": "data.json"} + + :param handler: Function that will be called to process matching + requests. This must take two parameters, the request + object and the response object. + + """ + if isinstance(methods, (bytes, str)) or methods is any_method: + methods = [methods] + for method in methods: + self.routes.append((method, compile_path_match(path), handler)) + self.logger.debug("Route pattern: %s" % self.routes[-1][1].pattern) + + def get_handler(self, request): + """Get a handler for a request or None if there is no handler. + + :param request: Request to get a handler for. + :rtype: Callable or None + """ + for method, regexp, handler in reversed(self.routes): + if (request.method == method or + method in (any_method, "*") or + (request.method == "HEAD" and method == "GET")): + m = regexp.match(request.url_parts.path) + if m: + if not hasattr(handler, "__class__"): + name = handler.__name__ + else: + name = handler.__class__.__name__ + self.logger.debug("Found handler %s" % name) + + match_parts = m.groupdict().copy() + if len(match_parts) < len(m.groups()): + match_parts["*"] = m.groups()[-1] + request.route_match = match_parts + + return handler + return None diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/routes.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/routes.py new file mode 100644 index 0000000000..b6e3800018 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/routes.py @@ -0,0 +1,6 @@ +from . import handlers +from .router import any_method +routes = [(any_method, "*.py", handlers.python_script_handler), + ("GET", "*.asis", handlers.as_is_handler), + ("GET", "*", handlers.file_handler), + ] diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/server.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/server.py new file mode 100644 index 0000000000..142cb64702 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/server.py @@ -0,0 +1,1048 @@ +# mypy: allow-untyped-defs + +import errno +import http +import http.server +import ipaddress +import os +import platform +import selectors +import socket +import socketserver +import ssl +import sys +import threading +import time +import traceback +import uuid +from collections import OrderedDict +from queue import Empty, Queue +from typing import Dict + +from h2.config import H2Configuration +from h2.connection import H2Connection +from h2.events import RequestReceived, ConnectionTerminated, DataReceived, StreamReset, StreamEnded +from h2.exceptions import StreamClosedError, ProtocolError +from h2.settings import SettingCodes +from h2.utilities import extract_method_header + +from urllib.parse import urlsplit, urlunsplit + +from pywebsocket3 import dispatch +from pywebsocket3.handshake import HandshakeException, AbortedByUserException + +from . import routes as default_routes +from .config import ConfigBuilder +from .logger import get_logger +from .request import Server, Request, H2Request +from .response import Response, H2Response +from .router import Router +from .utils import HTTPException, get_error_cause, isomorphic_decode, isomorphic_encode +from .constants import h2_headers +from .ws_h2_handshake import WsH2Handshaker + +# We need to stress test that browsers can send/receive many headers (there is +# no specified limit), but the Python stdlib has an arbitrary limit of 100 +# headers. Hitting the limit leads to HTTP 431, so we monkey patch it higher. +# https://bugs.python.org/issue26586 +# https://github.com/web-platform-tests/wpt/pull/24451 +import http.client +assert isinstance(getattr(http.client, '_MAXHEADERS'), int) +setattr(http.client, '_MAXHEADERS', 512) + +""" +HTTP server designed for testing purposes. + +The server is designed to provide flexibility in the way that +requests are handled, and to provide control both of exactly +what bytes are put on the wire for the response, and in the +timing of sending those bytes. + +The server is based on the stdlib HTTPServer, but with some +notable differences in the way that requests are processed. +Overall processing is handled by a WebTestRequestHandler, +which is a subclass of BaseHTTPRequestHandler. This is responsible +for parsing the incoming request. A RequestRewriter is then +applied and may change the request data if it matches a +supplied rule. + +Once the request data had been finalised, Request and Response +objects are constructed. These are used by the other parts of the +system to read information about the request and manipulate the +response. + +Each request is handled by a particular handler function. The +mapping between Request and the appropriate handler is determined +by a Router. By default handlers are installed to interpret files +under the document root with .py extensions as executable python +files (see handlers.py for the api for such files), .asis files as +bytestreams to be sent literally and all other files to be served +statically. + +The handler functions are responsible for either populating the +fields of the response object, which will then be written when the +handler returns, or for directly writing to the output stream. +""" + + +class RequestRewriter: + def __init__(self, rules): + """Object for rewriting the request path. + + :param rules: Initial rules to add; a list of three item tuples + (method, input_path, output_path), defined as for + register() + """ + self.rules = {} + for rule in reversed(rules): + self.register(*rule) + self.logger = get_logger() + + def register(self, methods, input_path, output_path): + """Register a rewrite rule. + + :param methods: Set of methods this should match. "*" is a + special value indicating that all methods should + be matched. + + :param input_path: Path to match for the initial request. + + :param output_path: Path to replace the input path with in + the request. + """ + if isinstance(methods, (bytes, str)): + methods = [methods] + self.rules[input_path] = (methods, output_path) + + def rewrite(self, request_handler): + """Rewrite the path in a BaseHTTPRequestHandler instance, if + it matches a rule. + + :param request_handler: BaseHTTPRequestHandler for which to + rewrite the request. + """ + split_url = urlsplit(request_handler.path) + if split_url.path in self.rules: + methods, destination = self.rules[split_url.path] + if "*" in methods or request_handler.command in methods: + self.logger.debug("Rewriting request path %s to %s" % + (request_handler.path, destination)) + new_url = list(split_url) + new_url[2] = destination + new_url = urlunsplit(new_url) + request_handler.path = new_url + + +class WebTestServer(http.server.ThreadingHTTPServer): + allow_reuse_address = True + # Older versions of Python might throw `OSError: [Errno 0] Error` + # instead of `SSLEOFError`. + acceptable_errors = (errno.EPIPE, errno.ECONNABORTED, 0) + request_queue_size = 2000 + + # Ensure that we don't hang on shutdown waiting for requests + daemon_threads = True + + def __init__(self, server_address, request_handler_cls, + router, rewriter, bind_address, ws_doc_root=None, + config=None, use_ssl=False, key_file=None, certificate=None, + encrypt_after_connect=False, latency=None, http2=False, **kwargs): + """Server for HTTP(s) Requests + + :param server_address: tuple of (server_name, port) + + :param request_handler_cls: BaseHTTPRequestHandler-like class to use for + handling requests. + + :param router: Router instance to use for matching requests to handler + functions + + :param rewriter: RequestRewriter-like instance to use for preprocessing + requests before they are routed + + :param config: Dictionary holding environment configuration settings for + handlers to read, or None to use the default values. + + :param use_ssl: Boolean indicating whether the server should use SSL + + :param key_file: Path to key file to use if SSL is enabled. + + :param certificate: Path to certificate to use if SSL is enabled. + + :param ws_doc_root: Document root for websockets + + :param encrypt_after_connect: For each connection, don't start encryption + until a CONNECT message has been received. + This enables the server to act as a + self-proxy. + + :param bind_address True to bind the server to both the IP address and + port specified in the server_address parameter. + False to bind the server only to the port in the + server_address parameter, but not to the address. + :param latency: Delay in ms to wait before serving each response, or + callable that returns a delay in ms + """ + self._shutdown_event = threading.Event() + self._shutdown_write_sock = None + + self.router = router + self.rewriter = rewriter + + self.scheme = "http2" if http2 else "https" if use_ssl else "http" + self.logger = get_logger() + + self.latency = latency + + if bind_address: + hostname_port = server_address + else: + hostname_port = ("",server_address[1]) + + super().__init__(hostname_port, request_handler_cls) + + if config is not None: + Server.config = config + else: + self.logger.debug("Using default configuration") + with ConfigBuilder(self.logger, + browser_host=server_address[0], + ports={"http": [self.server_address[1]]}) as config: + assert config["ssl_config"] is None + Server.config = config + + + + self.ws_doc_root = ws_doc_root + self.key_file = key_file + self.certificate = certificate + self.encrypt_after_connect = use_ssl and encrypt_after_connect + + if use_ssl and not encrypt_after_connect: + if http2: + ssl_context = ssl.create_default_context(purpose=ssl.Purpose.CLIENT_AUTH) + ssl_context.load_cert_chain(keyfile=self.key_file, certfile=self.certificate) + ssl_context.set_alpn_protocols(['h2']) + self.socket = ssl_context.wrap_socket(self.socket, + do_handshake_on_connect=False, + server_side=True) + + else: + ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ssl_context.load_cert_chain(keyfile=self.key_file, certfile=self.certificate) + self.socket = ssl_context.wrap_socket(self.socket, + do_handshake_on_connect=False, + server_side=True) + + def server_bind(self): + if platform.system() != "Darwin": + super().server_bind() + else: + # We override this on macOS to workaround gethostbyaddr triggering the local + # network alert even when passed "localhost" (rdar://153097791); this should + # be the same as the superclass implementation except for the addition of + # our check. + socketserver.TCPServer.server_bind(self) + host, port = self.server_address[:2] + if ( + ipaddress.ip_address(host).is_loopback and + ipaddress.ip_address(socket.gethostbyname("localhost")).is_loopback + ): + self.server_name = "localhost" + else: + self.server_name = socket.getfqdn(host) + self.server_port = port + + def serve_forever(self, poll_interval=0.5): + """Handle one request at a time until shutdown. + + This overrides the superclass implementation to use a socket pair to process + shutdown requests, avoiding waiting the poll_interval before shutting down. + It does, however, still call service_actions() every poll_interval. + + """ + shutdown_read_sock, self._shutdown_write_sock = socket.socketpair() + self._shutdown_event.clear() + + try: + with selectors.DefaultSelector() as selector: + selector.register(self, selectors.EVENT_READ) + selector.register(shutdown_read_sock, selectors.EVENT_READ) + + while True: + events = selector.select(timeout=poll_interval) + + # Handle shutdown requests before any request + if any( + key.fileobj == shutdown_read_sock and mask == selectors.EVENT_READ + for key, mask in events + ): + shutdown_read_sock.recv(1) + break + + for key, mask in events: + if key.fileobj == self and mask == selectors.EVENT_READ: + super()._handle_request_noblock() + else: + assert False, "unreachable" + else: + self.service_actions() + + finally: + shutdown_read_sock.close() + self._shutdown_write_sock.close() + self._shutdown_event.set() + + def shutdown(self): + """Stops the serve_forever loop and waits for it to finish.""" + self._shutdown_write_sock.send(b'x') + self._shutdown_event.wait() + + def finish_request(self, request, client_address): + if isinstance(self.socket, ssl.SSLSocket): + request.do_handshake() + super().finish_request(request, client_address) + + def handle_error(self, request, client_address): + error = sys.exc_info()[1] + + if ((isinstance(error, OSError) and + isinstance(error.args, tuple) and + error.args[0] in self.acceptable_errors) or + (isinstance(error, IOError) and + error.errno in self.acceptable_errors) or + # `SSLEOFError` and `SSLError` may occur when a client + # (e.g., wptrunner's `TestEnvironment`) tests for connectivity + # but doesn't perform the handshake. + isinstance(error, ssl.SSLEOFError) or isinstance(error, ssl.SSLError)): + pass # remote hang up before the result is sent + else: + msg = traceback.format_exc() + self.logger.error(f"{type(error)} {error}") + self.logger.info(msg) + + +class BaseWebTestRequestHandler(http.server.BaseHTTPRequestHandler): + """RequestHandler for WebTestHttpd""" + + def __init__(self, *args, **kwargs): + self.logger = get_logger() + super().__init__(*args, **kwargs) + + def finish_handling_h1(self, request_line_is_valid): + + self.server.rewriter.rewrite(self) + + with Request(self) as request: + response = Response(self, request) + + if request.method == "CONNECT": + self.handle_connect(response) + return + + if not request_line_is_valid: + response.set_error(414) + response.write() + return + + self.logger.debug(f"{request.method} {request.request_path}") + handler = self.server.router.get_handler(request) + self.finish_handling(request, response, handler) + + def finish_handling(self, request, response, handler): + # If the handler we used for the request had a non-default base path + # set update the doc_root of the request to reflect this + if hasattr(handler, "base_path") and handler.base_path: + request.doc_root = handler.base_path + if hasattr(handler, "url_base") and handler.url_base != "/": + request.url_base = handler.url_base + + if self.server.latency is not None: + if callable(self.server.latency): + latency = self.server.latency() + else: + latency = self.server.latency + self.logger.warning("Latency enabled. Sleeping %i ms" % latency) + time.sleep(latency / 1000.) + + if handler is None: + self.logger.debug("No Handler found!") + response.set_error(404) + else: + try: + handler(request, response) + except HTTPException as e: + exc = get_error_cause(e) if 500 <= e.code < 600 else e + response.set_error(e.code, exc) + except Exception as e: + response.set_error(500, e) + self.logger.debug("%i %s %s (%s) %i" % (response.status[0], + request.method, + request.request_path, + request.headers.get('Referer'), + request.raw_input.length)) + + if not response.writer.content_written: + response.write() + + # If a python handler has been used, the old ones won't send a END_STR data frame, so this + # allows for backwards compatibility by accounting for these handlers that don't close streams + if isinstance(response, H2Response) and not response.writer.stream_ended: + response.writer.end_stream() + + # If we want to remove this in the future, a solution is needed for + # scripts that produce a non-string iterable of content, since these + # can't set a Content-Length header. A notable example of this kind of + # problem is with the trickle pipe i.e. foo.js?pipe=trickle(d1) + if response.close_connection: + self.close_connection = True + + if not self.close_connection: + # Ensure that the whole request has been read from the socket + request.raw_input.read() + + def handle_connect(self, response): + self.logger.debug("Got CONNECT") + response.status = 200 + response.write() + if self.server.encrypt_after_connect: + self.logger.debug("Enabling SSL for connection") + ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ssl_context.load_cert_chain(keyfile=self.server.key_file, certfile=self.server.certificate) + self.request = ssl_context.wrap_socket(self.connection, + server_side=True) + self.setup() + return + + def log_request(self, code="-", size="-"): + if isinstance(code, http.HTTPStatus): + code = code.value + + self.logger.debug( + "{} - - [{}] {!r} {!s} {!s}".format( + self.address_string(), + self.log_date_time_string(), + self.requestline, + code, + size, + ) + ) + + def log_error(self, format, *args): + self.logger.error( + "{} - - [{}] {}".format( + self.address_string(), self.log_date_time_string(), format % args + ) + ) + + def log_message(self, format, *args): + self.logger.info( + "{} - - [{}] {}".format( + self.address_string(), self.log_date_time_string(), format % args + ) + ) + + +class Http2WebTestRequestHandler(BaseWebTestRequestHandler): + protocol_version = "HTTP/2.0" + + def handle_one_request(self): + """ + This is the main HTTP/2 Handler. + + When a browser opens a connection to the server + on the HTTP/2 port, the server enters this which will initiate the h2 connection + and keep running throughout the duration of the interaction, and will read/write directly + from the socket. + + Because there can be multiple H2 connections active at the same + time, a UUID is created for each so that it is easier to tell them apart in the logs. + """ + + config = H2Configuration(client_side=False) + self.conn = H2ConnectionGuard(H2Connection(config=config)) + self.close_connection = False + + # Generate a UUID to make it easier to distinguish different H2 connection debug messages + self.uid = str(uuid.uuid4())[:8] + + self.logger.debug('(%s) Initiating h2 Connection' % self.uid) + + with self.conn as connection: + # Bootstrapping WebSockets with HTTP/2 specification requires + # ENABLE_CONNECT_PROTOCOL to be set in order to enable WebSocket + # over HTTP/2 + new_settings = dict(connection.local_settings) + new_settings[SettingCodes.ENABLE_CONNECT_PROTOCOL] = 1 + connection.local_settings.update(new_settings) + connection.local_settings.acknowledge() + + connection.initiate_connection() + data = connection.data_to_send() + window_size = connection.remote_settings.initial_window_size + + try: + self.request.sendall(data) + except ConnectionResetError: + self.logger.warning("Connection reset during h2 setup") + return + + # Dict of { stream_id: (thread, queue) } + stream_queues = {} + + try: + while not self.close_connection: + data = self.request.recv(window_size) + if data == '': + self.logger.debug('(%s) Socket Closed' % self.uid) + self.close_connection = True + continue + + with self.conn as connection: + frames = connection.receive_data(data) + window_size = connection.remote_settings.initial_window_size + + self.logger.debug('(%s) Frames Received: ' % self.uid + str(frames)) + + for frame in frames: + if isinstance(frame, ConnectionTerminated): + self.logger.debug('(%s) Connection terminated by remote peer ' % self.uid) + self.close_connection = True + + # Flood all the streams with connection terminated, this will cause them to stop + for stream_id, (thread, queue) in stream_queues.items(): + queue.put(frame) + + elif hasattr(frame, 'stream_id'): + if frame.stream_id not in stream_queues: + queue = Queue() + stream_queues[frame.stream_id] = (self.start_stream_thread(frame, queue), queue) + stream_queues[frame.stream_id][1].put(frame) + + if isinstance(frame, StreamEnded) or getattr(frame, "stream_ended", False): + del stream_queues[frame.stream_id] + + except OSError as e: + self.logger.error(f'({self.uid}) Closing Connection - \n{str(e)}') + if not self.close_connection: + self.close_connection = True + except Exception as e: + self.logger.error(f'({self.uid}) Unexpected Error - \n{str(e)}') + finally: + for stream_id, (thread, queue) in stream_queues.items(): + queue.put(None) + thread.join() + + def _is_extended_connect_frame(self, frame): + if not isinstance(frame, RequestReceived): + return False + + method = extract_method_header(frame.headers) + if method != b"CONNECT": + return False + + protocol = "" + for key, value in frame.headers: + if key in (b':protocol', ':protocol'): + protocol = isomorphic_encode(value) + break + if protocol != b"websocket": + raise ProtocolError(f"Invalid protocol {protocol} with CONNECT METHOD") + + return True + + def start_stream_thread(self, frame, queue): + """ + This starts a new thread to handle frames for a specific stream. + :param frame: The first frame on the stream + :param queue: A queue object that the thread will use to check for new frames + :return: The thread object that has already been started + """ + if self._is_extended_connect_frame(frame): + target = Http2WebTestRequestHandler._stream_ws_thread + else: + target = Http2WebTestRequestHandler._stream_thread + t = threading.Thread( + target=target, + args=(self, frame.stream_id, queue) + ) + t.start() + return t + + def _stream_ws_thread(self, stream_id, queue): + frame = queue.get(True, None) + + if frame is None: + return + + # Needs to be unbuffered for websockets. + rfile, wfile = os.pipe() + with os.fdopen(rfile, 'rb') as rfile, os.fdopen(wfile, 'wb', 0) as wfile: + stream_handler = H2HandlerCopy(self, frame, rfile) + + h2request = H2Request(stream_handler) + h2response = H2Response(stream_handler, h2request) + + dispatcher = dispatch.Dispatcher(self.server.ws_doc_root, None, False) + if not dispatcher.get_handler_suite(stream_handler.path): + h2response.set_error(404) + h2response.write() + return + + request_wrapper = _WebSocketRequest(stream_handler, h2response) + + handshaker = WsH2Handshaker(request_wrapper, dispatcher) + try: + handshaker.do_handshake() + except HandshakeException as e: + self.logger.info("Handshake failed") + h2response.set_error(e.status, e) + h2response.write() + return + except AbortedByUserException: + h2response.write() + return + + # h2 Handshaker prepares the headers but does not send them down the + # wire. Flush the headers here. + try: + h2response.write_status_headers() + except StreamClosedError: + # work around https://github.com/web-platform-tests/wpt/issues/27786 + # The stream was already closed. + return + + request_wrapper._dispatcher = dispatcher + + # we need two threads: + # - one to handle the frame queue + # - one to handle the request (dispatcher.transfer_data is blocking) + # the alternative is to have only one (blocking) thread. That thread + # will call transfer_data. That would require a special case in + # handle_one_request, to bypass the queue and write data to wfile + # directly. + t = threading.Thread( + target=Http2WebTestRequestHandler._stream_ws_sub_thread, + args=(self, request_wrapper, stream_handler, queue) + ) + t.start() + + while not self.close_connection: + try: + frame = queue.get(True, 1) + except Empty: + continue + + if isinstance(frame, DataReceived): + wfile.write(frame.data) + if frame.stream_ended: + raise NotImplementedError("frame.stream_ended") + elif frame is None or isinstance(frame, (StreamReset, StreamEnded, ConnectionTerminated)): + self.logger.error(f'({self.uid} - {stream_id}) Stream Reset, Thread Closing') + break + + t.join() + + def _stream_ws_sub_thread(self, request, stream_handler, queue): + dispatcher = request._dispatcher + try: + dispatcher.transfer_data(request) + except (StreamClosedError, ProtocolError): + # work around https://github.com/web-platform-tests/wpt/issues/27786 + # The stream was already closed. + queue.put(None) + return + + stream_id = stream_handler.h2_stream_id + with stream_handler.conn as connection: + try: + connection.end_stream(stream_id) + data = connection.data_to_send() + stream_handler.request.sendall(data) + except (StreamClosedError, ProtocolError): # maybe the stream has already been closed + pass + queue.put(None) + + def _stream_thread(self, stream_id, queue): + """ + This thread processes frames for a specific stream. It waits for frames to be placed + in the queue, and processes them. When it receives a request frame, it will start processing + immediately, even if there are data frames to follow. One of the reasons for this is that it + can detect invalid requests before needing to read the rest of the frames. + """ + + # The file-like pipe object that will be used to share data to request object if data is received + wfile = None + rfile = None + request = None + response = None + req_handler = None + + def cleanup(): + # Try to close the files + # Ignore any exception (e.g. if the file handle was already closed for some reason). + if rfile: + try: + rfile.close() + except OSError: + pass + if wfile: + try: + wfile.close() + except OSError: + pass + + while not self.close_connection: + try: + frame = queue.get(True, 1) + except Empty: + # Restart to check for close_connection + continue + + self.logger.debug(f'({self.uid} - {stream_id}) {str(frame)}') + if isinstance(frame, RequestReceived): + cleanup() + + pipe_rfile, pipe_wfile = os.pipe() + (rfile, wfile) = os.fdopen(pipe_rfile, 'rb'), os.fdopen(pipe_wfile, 'wb') + + stream_handler = H2HandlerCopy(self, frame, rfile) + + stream_handler.server.rewriter.rewrite(stream_handler) + request = H2Request(stream_handler) + response = H2Response(stream_handler, request) + + req_handler = stream_handler.server.router.get_handler(request) + + if hasattr(req_handler, "frame_handler"): + # Convert this to a handler that will utilise H2 specific functionality, such as handling individual frames + req_handler = self.frame_handler(request, response, req_handler) + + if hasattr(req_handler, 'handle_headers'): + req_handler.handle_headers(frame, request, response) + + elif isinstance(frame, DataReceived): + wfile.write(frame.data) + + if hasattr(req_handler, 'handle_data'): + req_handler.handle_data(frame, request, response) + + elif frame is None or isinstance(frame, (StreamReset, StreamEnded, ConnectionTerminated)): + self.logger.debug(f'({self.uid} - {stream_id}) Stream Reset, Thread Closing') + break + + if request is not None: + request.frames.append(frame) + + if getattr(frame, "stream_ended", False): + try: + self.finish_handling(request, response, req_handler) + except StreamClosedError: + self.logger.debug('(%s - %s) Unable to write response; stream closed' % + (self.uid, stream_id)) + break + + cleanup() + + def frame_handler(self, request, response, handler): + try: + return handler.frame_handler(request) + except HTTPException as e: + exc = get_error_cause(e) if 500 <= e.code < 600 else e + response.set_error(exc.code, exc) + response.write() + except Exception as e: + response.set_error(500, e) + response.write() + + +class H2ConnectionGuard: + """H2Connection objects are not threadsafe, so this keeps thread safety""" + def __init__(self, obj): + assert isinstance(obj, H2Connection) + self.obj = obj + self.lock = threading.Lock() + + def __enter__(self): + self.lock.acquire() + return self.obj + + def __exit__(self, exception_type, exception_value, traceback): + self.lock.release() + + +class H2Headers(Dict[bytes, bytes]): + def __init__(self, headers): + self.raw_headers = OrderedDict() + for key, val in headers: + key = isomorphic_decode(key) + val = isomorphic_decode(val) + self.raw_headers[key] = val + dict.__setitem__(self, self._convert_h2_header_to_h1(key), val) + + def _convert_h2_header_to_h1(self, header_key): + if header_key[1:] in h2_headers and header_key[0] == ':': + return header_key[1:] + else: + return header_key + + # TODO This does not seem relevant for H2 headers, so using a dummy function for now + def getallmatchingheaders(self, header): + return ['dummy function'] + + +class H2HandlerCopy: + def __init__(self, handler, req_frame, rfile): + self.headers = H2Headers(req_frame.headers) + self.command = self.headers['method'] + self.path = self.headers['path'] + self.h2_stream_id = req_frame.stream_id + self.server = handler.server + self.protocol_version = handler.protocol_version + self.client_address = handler.client_address + self.raw_requestline = '' + self.rfile = rfile + self.request = handler.request + self.conn = handler.conn + +class Http1WebTestRequestHandler(BaseWebTestRequestHandler): + protocol_version = "HTTP/1.1" + + def handle_one_request(self): + response = None + + try: + self.close_connection = False + + request_line_is_valid = self.get_request_line() + + if self.close_connection: + return + + request_is_valid = self.parse_request() + if not request_is_valid: + #parse_request() actually sends its own error responses + return + + self.finish_handling_h1(request_line_is_valid) + + except socket.timeout as e: + self.log_error("Request timed out: %r", e) + self.close_connection = True + return + + except Exception as e: + if response: + response.set_error(500, e) + response.write() + + def get_request_line(self): + try: + self.raw_requestline = self.rfile.readline(65537) + except OSError: + self.close_connection = True + return False + if len(self.raw_requestline) > 65536: + self.requestline = '' + self.request_version = '' + self.command = '' + return False + if not self.raw_requestline: + self.close_connection = True + return True + +class WebTestHttpd: + """ + :param host: Host from which to serve (default: 127.0.0.1) + :param port: Port from which to serve (default: 8000) + :param server_cls: Class to use for the server (default depends on ssl vs non-ssl) + :param handler_cls: Class to use for the RequestHandler + :param use_ssl: Use a SSL server if no explicit server_cls is supplied + :param key_file: Path to key file to use if ssl is enabled + :param certificate: Path to certificate file to use if ssl is enabled + :param encrypt_after_connect: For each connection, don't start encryption + until a CONNECT message has been received. + This enables the server to act as a + self-proxy. + :param router_cls: Router class to use when matching URLs to handlers + :param doc_root: Document root for serving files + :param ws_doc_root: Document root for websockets + :param routes: List of routes with which to initialize the router + :param rewriter_cls: Class to use for request rewriter + :param rewrites: List of rewrites with which to initialize the rewriter_cls + :param config: Dictionary holding environment configuration settings for + handlers to read, or None to use the default values. + :param bind_address: Boolean indicating whether to bind server to IP address. + :param latency: Delay in ms to wait before serving each response, or + callable that returns a delay in ms + + HTTP server designed for testing scenarios. + + Takes a router class which provides one method get_handler which takes a Request + and returns a handler function. + + .. attribute:: host + + The host name or ip address of the server + + .. attribute:: port + + The port on which the server is running + + .. attribute:: router + + The Router object used to associate requests with resources for this server + + .. attribute:: rewriter + + The Rewriter object used for URL rewriting + + .. attribute:: use_ssl + + Boolean indicating whether the server is using ssl + + .. attribute:: started + + Boolean indicating whether the server is running + + """ + def __init__(self, host="127.0.0.1", port=8000, + server_cls=None, handler_cls=Http1WebTestRequestHandler, + use_ssl=False, key_file=None, certificate=None, encrypt_after_connect=False, + router_cls=Router, doc_root=os.curdir, ws_doc_root=None, routes=None, + rewriter_cls=RequestRewriter, bind_address=True, rewrites=None, + latency=None, config=None, http2=False): + + if routes is None: + routes = default_routes.routes + + self.host = host + + self.router = router_cls(doc_root, routes) + self.rewriter = rewriter_cls(rewrites if rewrites is not None else []) + + self.use_ssl = use_ssl + self.http2 = http2 + self.logger = get_logger() + + if server_cls is None: + server_cls = WebTestServer + + if use_ssl: + if not os.path.exists(key_file): + raise ValueError(f"SSL certificate not found: {key_file}") + if not os.path.exists(certificate): + raise ValueError(f"SSL key not found: {certificate}") + + try: + self.httpd = server_cls((host, port), + handler_cls, + self.router, + self.rewriter, + config=config, + bind_address=bind_address, + ws_doc_root=ws_doc_root, + use_ssl=use_ssl, + key_file=key_file, + certificate=certificate, + encrypt_after_connect=encrypt_after_connect, + latency=latency, + http2=http2) + self.started = False + + _host, self.port = self.httpd.socket.getsockname() + except Exception: + self.logger.critical("Failed to start HTTP server on port %s; " + "is something already using that port?" % port) + raise + + def start(self): + """Start the server. + + :param block: True to run the server on the current thread, blocking, + False to run on a separate thread.""" + http_type = "http2" if self.http2 else "https" if self.use_ssl else "http" + http_scheme = "https" if self.use_ssl else "http" + self.logger.info(f"Starting {http_type} server on {http_scheme}://{self.host}:{self.port}") + self.started = True + self.server_thread = threading.Thread(target=self.httpd.serve_forever) + self.server_thread.daemon = True # don't hang on exit + self.server_thread.start() + + def stop(self): + """ + Stops the server. + + If the server is not running, this method has no effect. + """ + if self.started: + try: + self.httpd.shutdown() + self.httpd.server_close() + self.server_thread.join() + self.server_thread = None + self.logger.info(f"Stopped http server on {self.host}:{self.port}") + except AttributeError: + pass + self.started = False + self.httpd = None + + def get_url(self, path="/", query=None, fragment=None): + if not self.started: + return None + + return urlunsplit(("http" if not self.use_ssl else "https", + f"{self.host}:{self.port}", + path, query, fragment)) + + +class _WebSocketConnection: + def __init__(self, request_handler, response): + """Mimic mod_python mp_conn. + + :param request_handler: A H2HandlerCopy instance. + + :param response: A H2Response instance. + """ + + self._request_handler = request_handler + self._response = response + + self.remote_addr = self._request_handler.client_address + + def write(self, data): + self._response.writer.write_data(data, False) + + def read(self, length): + return self._request_handler.rfile.read(length) + + +class _WebSocketRequest: + def __init__(self, request_handler, response): + """Mimic mod_python request. + + :param request_handler: A H2HandlerCopy instance. + + :param response: A H2Response instance. + """ + + self.connection = _WebSocketConnection(request_handler, response) + self.protocol = "HTTP/2" + self._response = response + + self.uri = request_handler.path + self.unparsed_uri = request_handler.path + self.method = request_handler.command + # read headers from request_handler + self.headers_in = request_handler.headers + # write headers directly into H2Response + self.headers_out = response.headers + + # proxies status to H2Response + @property + def status(self): + return self._response.status + + @status.setter + def status(self, status): + self._response.status = status diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/sslutils/__init__.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/sslutils/__init__.py new file mode 100644 index 0000000000..244faeadda --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/sslutils/__init__.py @@ -0,0 +1,16 @@ +# mypy: allow-untyped-defs + +from .base import NoSSLEnvironment +from .openssl import OpenSSLEnvironment +from .pregenerated import PregeneratedSSLEnvironment + +environments = {"none": NoSSLEnvironment, + "openssl": OpenSSLEnvironment, + "pregenerated": PregeneratedSSLEnvironment} + + +def get_cls(name): + try: + return environments[name] + except KeyError: + raise ValueError("%s is not a valid SSL type." % name) diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/sslutils/base.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/sslutils/base.py new file mode 100644 index 0000000000..d5f913735a --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/sslutils/base.py @@ -0,0 +1,19 @@ +# mypy: allow-untyped-defs + +class NoSSLEnvironment: + ssl_enabled = False + + def __init__(self, *args, **kwargs): + pass + + def __enter__(self): + return self + + def __exit__(self, *args, **kwargs): + pass + + def host_cert_path(self, hosts): + return None, None + + def ca_cert_path(self, hosts): + return None diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/sslutils/openssl.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/sslutils/openssl.py new file mode 100644 index 0000000000..2a388671b6 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/sslutils/openssl.py @@ -0,0 +1,422 @@ +# mypy: allow-untyped-defs + +import functools +import os +import random +import shutil +import subprocess +import tempfile +from datetime import datetime, timedelta, timezone +from email.utils import parsedate_to_datetime + +# Amount of time beyond the present to consider certificates "expired." This +# allows certificates to be proactively re-generated in the "buffer" period +# prior to their exact expiration time. +CERT_EXPIRY_BUFFER = dict(hours=6) + + +class OpenSSL: + def __init__(self, logger, binary, base_path, conf_path, hosts, duration, + base_conf_path=None): + """Context manager for interacting with OpenSSL. + Creates a config file for the duration of the context. + + :param logger: stdlib logger or python structured logger + :param binary: path to openssl binary + :param base_path: path to directory for storing certificates + :param conf_path: path for configuration file storing configuration data + :param hosts: list of hosts to include in configuration (or None if not + generating host certificates) + :param duration: Certificate duration in days""" + + self.base_path = base_path + self.binary = binary + self.conf_path = conf_path + self.base_conf_path = base_conf_path + self.logger = logger + self.proc = None + self.cmd = [] + self.hosts = hosts + self.duration = duration + + def __enter__(self): + with open(self.conf_path, "w") as f: + f.write(get_config(self.base_path, self.hosts, self.duration)) + return self + + def __exit__(self, *args, **kwargs): + os.unlink(self.conf_path) + + def log(self, line): + if hasattr(self.logger, "process_output"): + self.logger.process_output(self.proc.pid if self.proc is not None else None, + line.decode("utf8", "replace"), + command=" ".join(self.cmd)) + else: + self.logger.debug(line) + + def __call__(self, cmd, *args, **kwargs): + """Run a command using OpenSSL in the current context. + + :param cmd: The openssl subcommand to run + :param *args: Additional arguments to pass to the command + """ + self.cmd = [self.binary, cmd] + if cmd != "x509": + self.cmd += ["-config", self.conf_path] + self.cmd += list(args) + + # Copy the environment and add OPENSSL_CONF if available. + env = os.environ.copy() + if self.base_conf_path is not None: + env["OPENSSL_CONF"] = self.base_conf_path + + self.proc = subprocess.Popen(self.cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + env=env) + stdout, stderr = self.proc.communicate() + self.log(stdout) + if self.proc.returncode != 0: + raise subprocess.CalledProcessError(self.proc.returncode, self.cmd, + output=stdout) + + self.cmd = [] + self.proc = None + return stdout + + +def make_subject(common_name, + country=None, + state=None, + locality=None, + organization=None, + organization_unit=None): + args = [("country", "C"), + ("state", "ST"), + ("locality", "L"), + ("organization", "O"), + ("organization_unit", "OU"), + ("common_name", "CN")] + + rv = [] + + for var, key in args: + value = locals()[var] + if value is not None: + rv.append("/%s=%s" % (key, value.replace("/", "\\/"))) + + return "".join(rv) + +def make_alt_names(hosts): + return ",".join("DNS:%s" % host for host in hosts) + +def make_name_constraints(hosts): + return ",".join("permitted;DNS:%s" % host for host in hosts) + +def get_config(root_dir, hosts, duration=30): + if hosts is None: + san_line = "" + constraints_line = "" + else: + san_line = "subjectAltName = %s" % make_alt_names(hosts) + constraints_line = "nameConstraints = " + make_name_constraints(hosts) + + if os.path.sep == "\\": + # This seems to be needed for the Shining Light OpenSSL on + # Windows, at least. + root_dir = root_dir.replace("\\", "\\\\") + + rv = """[ ca ] +default_ca = CA_default + +[ CA_default ] +dir = %(root_dir)s +certs = $dir +new_certs_dir = $certs +crl_dir = $dir%(sep)scrl +database = $dir%(sep)sindex.txt +private_key = $dir%(sep)scacert.key +certificate = $dir%(sep)scacert.pem +serial = $dir%(sep)sserial +crldir = $dir%(sep)scrl +crlnumber = $dir%(sep)scrlnumber +crl = $crldir%(sep)scrl.pem +RANDFILE = $dir%(sep)sprivate%(sep)s.rand +x509_extensions = usr_cert +name_opt = ca_default +cert_opt = ca_default +default_days = %(duration)d +default_crl_days = %(duration)d +default_md = sha256 +preserve = no +policy = policy_anything +copy_extensions = copy + +[ policy_anything ] +countryName = optional +stateOrProvinceName = optional +localityName = optional +organizationName = optional +organizationalUnitName = optional +commonName = supplied +emailAddress = optional + +[ req ] +default_bits = 2048 +default_keyfile = privkey.pem +distinguished_name = req_distinguished_name +attributes = req_attributes +x509_extensions = v3_ca + +# Passwords for private keys if not present they will be prompted for +# input_password = secret +# output_password = secret +string_mask = utf8only +req_extensions = v3_req + +[ req_distinguished_name ] +countryName = Country Name (2 letter code) +countryName_default = AU +countryName_min = 2 +countryName_max = 2 +stateOrProvinceName = State or Province Name (full name) +stateOrProvinceName_default = +localityName = Locality Name (eg, city) +0.organizationName = Organization Name +0.organizationName_default = Web Platform Tests +organizationalUnitName = Organizational Unit Name (eg, section) +#organizationalUnitName_default = +commonName = Common Name (e.g. server FQDN or YOUR name) +commonName_max = 64 +emailAddress = Email Address +emailAddress_max = 64 + +[ req_attributes ] + +[ usr_cert ] +basicConstraints=CA:false +subjectKeyIdentifier=hash +authorityKeyIdentifier=keyid,issuer + +[ v3_req ] +basicConstraints = CA:FALSE +keyUsage = nonRepudiation, digitalSignature, keyEncipherment +extendedKeyUsage = serverAuth +%(san_line)s + +[ v3_ca ] +basicConstraints = CA:true +subjectKeyIdentifier=hash +authorityKeyIdentifier=keyid:always,issuer:always +keyUsage = keyCertSign +%(constraints_line)s +""" % {"root_dir": root_dir, + "san_line": san_line, + "duration": duration, + "constraints_line": constraints_line, + "sep": os.path.sep.replace("\\", "\\\\")} + + return rv + +class OpenSSLEnvironment: + ssl_enabled = True + + def __init__(self, logger, openssl_binary="openssl", base_path=None, + password="web-platform-tests", force_regenerate=False, + duration=30, base_conf_path=None): + """SSL environment that creates a local CA and host certificate using OpenSSL. + + By default this will look in base_path for existing certificates that are still + valid and only create new certificates if there aren't any. This behaviour can + be adjusted using the force_regenerate option. + + :param logger: a stdlib logging compatible logger or mozlog structured logger + :param openssl_binary: Path to the OpenSSL binary + :param base_path: Path in which certificates will be stored. If None, a temporary + directory will be used and removed when the server shuts down + :param password: Password to use + :param force_regenerate: Always create a new certificate even if one already exists. + """ + self.logger = logger + + self.temporary = False + if base_path is None: + base_path = tempfile.mkdtemp() + self.temporary = True + + self.base_path = os.path.abspath(base_path) + self.password = password + self.force_regenerate = force_regenerate + self.duration = duration + self.base_conf_path = base_conf_path + + self.path = None + self.binary = openssl_binary + self.openssl = None + + self._ca_cert_path = None + self._ca_key_path = None + self.host_certificates = {} + + def __enter__(self): + if not os.path.exists(self.base_path): + os.makedirs(self.base_path) + + path = functools.partial(os.path.join, self.base_path) + + with open(path("index.txt"), "w"): + pass + with open(path("serial"), "w") as f: + serial = "%x" % random.randint(0, 1000000) + if len(serial) % 2: + serial = "0" + serial + f.write(serial) + + self.path = path + + return self + + def __exit__(self, *args, **kwargs): + if self.temporary: + shutil.rmtree(self.base_path) + + def _config_openssl(self, hosts): + conf_path = self.path("openssl.cfg") + return OpenSSL(self.logger, self.binary, self.base_path, conf_path, hosts, + self.duration, self.base_conf_path) + + def ca_cert_path(self, hosts): + """Get the path to the CA certificate file, generating a + new one if needed""" + if self._ca_cert_path is None and not self.force_regenerate: + self._load_ca_cert() + if self._ca_cert_path is None: + self._generate_ca(hosts) + return self._ca_cert_path + + def _load_ca_cert(self): + key_path = self.path("cacert.key") + cert_path = self.path("cacert.pem") + + if self.check_key_cert(key_path, cert_path, None): + self.logger.info("Using existing CA cert") + self._ca_key_path, self._ca_cert_path = key_path, cert_path + + def check_key_cert(self, key_path, cert_path, hosts): + """Check that a key and cert file exist and are valid""" + if not os.path.exists(key_path) or not os.path.exists(cert_path): + return False + + with self._config_openssl(hosts) as openssl: + end_date_str = openssl("x509", + "-noout", + "-enddate", + "-in", cert_path).decode("utf8").split("=", 1)[1].strip() + # openssl outputs an RFC 822 date + end_date = parsedate_to_datetime(end_date_str) + time_buffer = timedelta(**CERT_EXPIRY_BUFFER) + if end_date < datetime.now(timezone.utc) + time_buffer: + return False + + #TODO: check the key actually signed the cert. + return True + + def _generate_ca(self, hosts): + path = self.path + self.logger.info("Generating new CA in %s" % self.base_path) + + key_path = path("cacert.key") + req_path = path("careq.pem") + cert_path = path("cacert.pem") + + with self._config_openssl(hosts) as openssl: + openssl("req", + "-batch", + "-new", + "-newkey", "rsa:2048", + "-keyout", key_path, + "-out", req_path, + "-subj", make_subject("web-platform-tests"), + "-passout", "pass:%s" % self.password) + + openssl("ca", + "-batch", + "-create_serial", + "-keyfile", key_path, + "-passin", "pass:%s" % self.password, + "-selfsign", + "-extensions", "v3_ca", + "-notext", + "-in", req_path, + "-out", cert_path) + + os.unlink(req_path) + + self._ca_key_path, self._ca_cert_path = key_path, cert_path + + def host_cert_path(self, hosts): + """Get a tuple of (private key path, certificate path) for a host, + generating new ones if necessary. + + hosts must be a list of all hosts to appear on the certificate, with + the primary hostname first.""" + hosts = tuple(sorted(hosts, key=lambda x:len(x))) + if hosts not in self.host_certificates: + if not self.force_regenerate: + key_cert = self._load_host_cert(hosts) + else: + key_cert = None + if key_cert is None: + key, cert = self._generate_host_cert(hosts) + else: + key, cert = key_cert + self.host_certificates[hosts] = key, cert + + return self.host_certificates[hosts] + + def _load_host_cert(self, hosts): + host = hosts[0] + key_path = self.path("%s.key" % host) + cert_path = self.path("%s.pem" % host) + + # TODO: check that this cert was signed by the CA cert + if self.check_key_cert(key_path, cert_path, hosts): + self.logger.info("Using existing host cert") + return key_path, cert_path + + def _generate_host_cert(self, hosts): + host = hosts[0] + if not self.force_regenerate: + self._load_ca_cert() + if self._ca_key_path is None: + self._generate_ca(hosts) + ca_key_path = self._ca_key_path + + assert os.path.exists(ca_key_path) + + path = self.path + + req_path = path("wpt.req") + cert_path = path("%s.pem" % host) + key_path = path("%s.key" % host) + + self.logger.info("Generating new host cert") + + with self._config_openssl(hosts) as openssl: + openssl("req", + "-batch", + "-newkey", "rsa:2048", + "-keyout", key_path, + "-in", ca_key_path, + "-nodes", + "-out", req_path) + + openssl("ca", + "-batch", + "-in", req_path, + "-passin", "pass:%s" % self.password, + "-subj", make_subject(host), + "-out", cert_path) + + os.unlink(req_path) + + return key_path, cert_path diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/sslutils/pregenerated.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/sslutils/pregenerated.py new file mode 100644 index 0000000000..5e9b1181a4 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/sslutils/pregenerated.py @@ -0,0 +1,28 @@ +# mypy: allow-untyped-defs + +class PregeneratedSSLEnvironment: + """SSL environment to use with existing key/certificate files + e.g. when running on a server with a public domain name + """ + ssl_enabled = True + + def __init__(self, logger, host_key_path, host_cert_path, + ca_cert_path=None): + self._ca_cert_path = ca_cert_path + self._host_key_path = host_key_path + self._host_cert_path = host_cert_path + + def __enter__(self): + return self + + def __exit__(self, *args, **kwargs): + pass + + def host_cert_path(self, hosts): + """Return the key and certificate paths for the host""" + return self._host_key_path, self._host_cert_path + + def ca_cert_path(self, hosts): + """Return the certificate path of the CA that signed the + host certificates, or None if that isn't known""" + return self._ca_cert_path diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/stash.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/stash.py new file mode 100644 index 0000000000..ed185c1756 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/stash.py @@ -0,0 +1,239 @@ +# mypy: allow-untyped-defs + +import base64 +import json +import os +import threading +import queue +import uuid + +from multiprocessing.managers import BaseManager, BaseProxy +# We also depend on some undocumented parts of multiprocessing.managers which +# don't have any type annotations. +from multiprocessing.managers import AcquirerProxy, DictProxy, public_methods # type: ignore +from typing import Dict + +from .utils import isomorphic_encode + + +class StashManager(BaseManager): + shared_data: Dict[str, object] = {} + lock = threading.Lock() + + +def _get_shared(): + return StashManager.shared_data + + +def _get_lock(): + return StashManager.lock + +StashManager.register("get_dict", + callable=_get_shared, + proxytype=DictProxy) +StashManager.register('Lock', + callable=_get_lock, + proxytype=AcquirerProxy) + + +# We have to create an explicit class here because the built-in +# AutoProxy has a bug with nested managers, and the MakeProxy +# method doesn't work with spawn-based multiprocessing, since the +# generated class can't be pickled for use in child processes. +class QueueProxy(BaseProxy): + _exposed_ = public_methods(queue.Queue) + + +for method in QueueProxy._exposed_: + + def impl_fn(method): + def _impl(self, *args, **kwargs): + return self._callmethod(method, args, kwargs) + _impl.__name__ = method + return _impl + + setattr(QueueProxy, method, impl_fn(method)) # type: ignore + + +StashManager.register("Queue", + callable=queue.Queue, + proxytype=QueueProxy) + + +class StashServer: + def __init__(self, address=None, authkey=None, mp_context=None): + self.address = address + self.authkey = authkey + self.manager = None + self.mp_context = mp_context + + def __enter__(self): + self.manager, self.address, self.authkey = start_server(self.address, + self.authkey, + self.mp_context) + store_env_config(self.address, self.authkey) + + def __exit__(self, *args, **kwargs): + if self.manager is not None: + self.manager.shutdown() + + +def load_env_config(): + address, authkey = json.loads(os.environ["WPT_STASH_CONFIG"]) + if isinstance(address, list): + address = tuple(address) + else: + address = str(address) + authkey = base64.b64decode(authkey) + return address, authkey + + +def store_env_config(address, authkey): + authkey = base64.b64encode(authkey) + os.environ["WPT_STASH_CONFIG"] = json.dumps((address, authkey.decode("ascii"))) + + +def start_server(address=None, authkey=None, mp_context=None): + if isinstance(authkey, str): + authkey = authkey.encode("ascii") + kwargs = {} + if mp_context is not None: + kwargs["ctx"] = mp_context + manager = StashManager(address, authkey, **kwargs) + manager.start() + + address = manager._address + if isinstance(address, bytes): + address = address.decode("ascii") + return (manager, address, manager._authkey) + + +class LockWrapper: + def __init__(self, lock): + self.lock = lock + + def acquire(self): + self.lock.acquire() + + def release(self): + self.lock.release() + + def __enter__(self): + self.acquire() + + def __exit__(self, *args, **kwargs): + self.release() + + +#TODO: Consider expiring values after some fixed time for long-running +#servers + +class Stash: + """Key-value store for persisting data across HTTP/S and WS/S requests. + + This data store is specifically designed for persisting data across server + requests. The synchronization is achieved by using the BaseManager from + the multiprocessing module so different processes can acccess the same data. + + Stash can be used interchangeably between HTTP, HTTPS, WS and WSS servers. + A thing to note about WS/S servers is that they require additional steps in + the handlers for accessing the same underlying shared data in the Stash. + This can usually be achieved by using load_env_config(). When using Stash + interchangeably between HTTP/S and WS/S request, the path part of the key + should be expliclitly specified if accessing the same key/value subset. + + The store has several unusual properties. Keys are of the form (path, + uuid), where path is, by default, the path in the HTTP request and + uuid is a unique id. In addition, the store is write-once, read-once, + i.e. the value associated with a particular key cannot be changed once + written and the read operation (called "take") is destructive. Taken together, + these properties make it difficult for data to accidentally leak + between different resources or different requests for the same + resource. + """ + + _proxy = None + lock = None + manager = None + _initializing = threading.Lock() + + def __init__(self, default_path, address=None, authkey=None): + self.default_path = default_path + self._get_proxy(address, authkey) + self.data = Stash._proxy + + def _get_proxy(self, address=None, authkey=None): + if address is None and authkey is None: + Stash._proxy = {} + Stash.lock = threading.Lock() + + # Initializing the proxy involves connecting to the remote process and + # retrieving two proxied objects. This process is not inherently + # atomic, so a lock must be used to make it so. Atomicity ensures that + # only one thread attempts to initialize the connection and that any + # threads running in parallel correctly wait for initialization to be + # fully complete. + with Stash._initializing: + if Stash.lock: + return + + Stash.manager = StashManager(address, authkey) + Stash.manager.connect() + Stash._proxy = self.manager.get_dict() + Stash.lock = LockWrapper(self.manager.Lock()) + + def get_queue(self): + return self.manager.Queue() + + def _wrap_key(self, key, path): + if path is None: + path = self.default_path + # This key format is required to support using the path. Since the data + # passed into the stash can be a DictProxy which wouldn't detect + # changes when writing to a subdict. + if isinstance(key, bytes): + # UUIDs are within the ASCII charset. + key = key.decode('ascii') + try: + my_uuid = uuid.UUID(key).bytes + except ValueError as e: + raise ValueError(f"""Invalid UUID "{key}" used as stash key""") from e + return (isomorphic_encode(path), my_uuid) + + def put(self, key, value, path=None): + """Place a value in the shared stash. + + :param key: A UUID to use as the data's key. + :param value: The data to store. This can be any python object. + :param path: The path that has access to read the data (by default + the current request path)""" + if value is None: + raise ValueError("SharedStash value may not be set to None") + internal_key = self._wrap_key(key, path) + if internal_key in self.data: + raise StashError("Tried to overwrite existing shared stash value " + "for key %s (old value was %s, new value is %s)" % + (internal_key, self.data[internal_key], value)) + else: + self.data[internal_key] = value + + def take(self, key, path=None): + """Remove a value from the shared stash and return it. + + :param key: A UUID to use as the data's key. + :param path: The path that has access to read the data (by default + the current request path)""" + internal_key = self._wrap_key(key, path) + value = self.data.get(internal_key, None) + if value is not None: + try: + self.data.pop(internal_key) + except KeyError: + # Silently continue when pop error occurs. + pass + + return value + + +class StashError(Exception): + pass diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/utils.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/utils.py new file mode 100644 index 0000000000..7bb1a4aa13 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/utils.py @@ -0,0 +1,207 @@ +import socket +from typing import AnyStr, Dict, List, TypeVar + +from .logger import get_logger + +KT = TypeVar('KT') +VT = TypeVar('VT') + + +def isomorphic_decode(s: AnyStr) -> str: + """Decodes a binary string into a text string using iso-8859-1. + + Returns `str`. The function is a no-op if the argument already has a text + type. iso-8859-1 is chosen because it is an 8-bit encoding whose code + points range from 0x0 to 0xFF and the values are the same as the binary + representations, so any binary string can be decoded into and encoded from + iso-8859-1 without any errors or data loss. Python 3 also uses iso-8859-1 + (or latin-1) extensively in http: + https://github.com/python/cpython/blob/273fc220b25933e443c82af6888eb1871d032fb8/Lib/http/client.py#L213 + """ + if isinstance(s, str): + return s + + if isinstance(s, bytes): + return s.decode("iso-8859-1") + + raise TypeError("Unexpected value (expecting string-like): %r" % s) + + +def isomorphic_encode(s: AnyStr) -> bytes: + """Encodes a text-type string into binary data using iso-8859-1. + + Returns `bytes`. The function is a no-op if the argument already has a + binary type. This is the counterpart of isomorphic_decode. + """ + if isinstance(s, bytes): + return s + + if isinstance(s, str): + return s.encode("iso-8859-1") + + raise TypeError("Unexpected value (expecting string-like): %r" % s) + + +def invert_dict(dict: Dict[KT, List[VT]]) -> Dict[VT, KT]: + rv = {} + for key, values in dict.items(): + for value in values: + if value in rv: + raise ValueError + rv[value] = key + return rv + + +class HTTPException(Exception): + def __init__(self, code: int, message: str = ""): + self.code = code + self.message = message + + +def _open_socket(host: str, port: int) -> socket.socket: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + if port != 0: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind((host, port)) + sock.listen(5) + return sock + + +def is_bad_port(port: int) -> bool: + """ + Bad port as per https://fetch.spec.whatwg.org/#port-blocking + """ + return port in [ + 0, + 1, # tcpmux + 7, # echo + 9, # discard + 11, # systat + 13, # daytime + 15, # netstat + 17, # qotd + 19, # chargen + 20, # ftp-data + 21, # ftp + 22, # ssh + 23, # telnet + 25, # smtp + 37, # time + 42, # name + 43, # nicname + 53, # domain + 69, # tftp + 77, # priv-rjs + 79, # finger + 87, # ttylink + 95, # supdup + 101, # hostriame + 102, # iso-tsap + 103, # gppitnp + 104, # acr-nema + 109, # pop2 + 110, # pop3 + 111, # sunrpc + 113, # auth + 115, # sftp + 117, # uucp-path + 119, # nntp + 123, # ntp + 135, # loc-srv / epmap + 137, # netbios-ns + 139, # netbios-ssn + 143, # imap2 + 161, # snmp + 179, # bgp + 389, # ldap + 427, # afp (alternate) + 465, # smtp (alternate) + 512, # print / exec + 513, # login + 514, # shell + 515, # printer + 526, # tempo + 530, # courier + 531, # chat + 532, # netnews + 540, # uucp + 548, # afp + 554, # rtsp + 556, # remotefs + 563, # nntp+ssl + 587, # smtp (outgoing) + 601, # syslog-conn + 636, # ldap+ssl + 989, # ftps-data + 999, # ftps + 993, # ldap+ssl + 995, # pop3+ssl + 1719, # h323gatestat + 1720, # h323hostcall + 1723, # pptp + 2049, # nfs + 3659, # apple-sasl + 4045, # lockd + 4190, # sieve + 5060, # sip + 5061, # sips + 6000, # x11 + 6566, # sane-port + 6665, # irc (alternate) + 6666, # irc (alternate) + 6667, # irc (default) + 6668, # irc (alternate) + 6669, # irc (alternate) + 6679, # osaut + 6697, # irc+tls + 10080, # amanda + ] + + +def get_port(host: str = '') -> int: + host = host or '127.0.0.1' + port = 0 + while True: + free_socket = _open_socket(host, 0) + port = free_socket.getsockname()[1] + free_socket.close() + if not is_bad_port(port): + break + return port + +def http2_compatible() -> bool: + # The HTTP/2 server requires OpenSSL 1.0.2+. + # + # For systems using other SSL libraries (e.g. LibreSSL), we assume they + # have the necessary support. + import ssl + if not ssl.OPENSSL_VERSION.startswith("OpenSSL"): + logger = get_logger() + logger.warning( + 'Skipping HTTP/2 compatibility check as system is not using ' + 'OpenSSL (found: %s)' % ssl.OPENSSL_VERSION) + return True + + # Note that OpenSSL's versioning scheme differs between 1.1.1 and + # earlier and 3.0.0. ssl.OPENSSL_VERSION_INFO returns a + # (major, minor, 0, patch, 0) + # tuple with OpenSSL 3.0.0 and later, and a + # (major, minor, fix, patch, status) + # tuple for older releases. + # Semantically, "patch" in 3.0.0+ is similar to "fix" in previous versions. + # + # What we do in the check below is allow OpenSSL 3.x.y+, 1.1.x+ and 1.0.2+. + ssl_v = ssl.OPENSSL_VERSION_INFO + return (ssl_v[0] > 1 or + (ssl_v[0] == 1 and + (ssl_v[1] == 1 or + (ssl_v[1] == 0 and ssl_v[2] >= 2)))) + + +def get_error_cause(exc: BaseException) -> BaseException: + """Get the parent cause/context from an exception""" + if exc.__cause__ is not None: + return exc.__cause__ + if exc.__context__ is not None: + return exc.__context__ + return exc diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/wptserve.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/wptserve.py new file mode 100755 index 0000000000..376196f669 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/wptserve.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +# mypy: allow-untyped-defs + +import argparse +import os + +from .server import WebTestHttpd + +def abs_path(path): + return os.path.abspath(path) + + +def parse_args(): + parser = argparse.ArgumentParser(description="HTTP server designed for extreme flexibility " + "required in testing situations.") + parser.add_argument("document_root", type=abs_path, + help="Root directory to serve files from") + parser.add_argument("--port", "-p", type=int, default=8000, + help="Port number to run server on") + parser.add_argument("--host", "-H", type=str, default="127.0.0.1", + help="Host to run server on") + return parser.parse_args() + + +def main(): + args = parse_args() + httpd = WebTestHttpd(host=args.host, port=args.port, + use_ssl=False, certificate=None, + doc_root=args.document_root) + httpd.start() + +if __name__ == "__main__": + main() # type: ignore diff --git a/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/ws_h2_handshake.py b/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/ws_h2_handshake.py new file mode 100644 index 0000000000..99d4dbeac4 --- /dev/null +++ b/Tests/LibWeb/Text/input/wpt-import/_wpttools/wptserve/wptserve/ws_h2_handshake.py @@ -0,0 +1,72 @@ +# mypy: allow-untyped-defs + +"""This file provides the opening handshake processor for the Bootstrapping +WebSockets with HTTP/2 protocol (RFC 8441). + +Specification: +https://tools.ietf.org/html/rfc8441 +""" + +from pywebsocket3 import common + +from pywebsocket3.handshake.base import get_mandatory_header +from pywebsocket3.handshake.base import HandshakeException +from pywebsocket3.handshake.base import validate_mandatory_header +from pywebsocket3.handshake.base import HandshakerBase + + +def check_connect_method(request): + if request.method != 'CONNECT': + raise HandshakeException('Method is not CONNECT: %r' % request.method) + + +class WsH2Handshaker(HandshakerBase): + def __init__(self, request, dispatcher): + """Bootstrapping handshake processor for the WebSocket protocol with HTTP/2 (RFC 8441). + + :param request: mod_python request. + + :param dispatcher: Dispatcher (dispatch.Dispatcher). + + WsH2Handshaker will add attributes such as ws_resource during handshake. + """ + + super().__init__(request, dispatcher) + + def _transform_header(self, header): + return header.lower() + + def _protocol_rfc(self): + return 'RFC 8441' + + def _validate_request(self): + check_connect_method(self._request) + validate_mandatory_header(self._request, ':protocol', 'websocket') + get_mandatory_header(self._request, 'authority') + + def _set_accept(self): + # irrelevant for HTTP/2 handshake + pass + + def _send_handshake(self): + # We are not actually sending the handshake, but just preparing it. It + # will be flushed by the caller. + self._request.status = 200 + + self._request.headers_out['upgrade'] = common.WEBSOCKET_UPGRADE_TYPE + self._request.headers_out[ + 'connection'] = common.UPGRADE_CONNECTION_TYPE + + if self._request.ws_protocol is not None: + self._request.headers_out[ + 'sec-websocket-protocol'] = self._request.ws_protocol + + if (self._request.ws_extensions is not None and + len(self._request.ws_extensions) != 0): + self._request.headers_out[ + 'sec-websocket-extensions'] = common.format_extensions( + self._request.ws_extensions) + + # Headers not specific for WebSocket + for name, value in self._request.extra_headers: + self._request.headers_out[name] = value