Inside Bitcoin

1#!/usr/bin/env python3
2# Copyright (c) The Bitcoin Core developers
3# Distributed under the MIT software license, see the accompanying
4# file COPYING or http://www.opensource.org/licenses/mit-license.php.
5"""Test the HTTP server basics."""
6
7from test_framework.test_framework import BitcoinTestFramework
8from test_framework.netutil import NETWORK_ERRORS
9from test_framework.util import (
10    assert_equal,
11    assert_raises,
12    mine_large_block,
13    str_to_b64str
14)
15from test_framework.wallet import MiniWallet
16
17import concurrent.futures
18import http.client
19import re
20import socket
21import threading
22import time
23import urllib.parse
24
25# Configuration option for some tests
26RPCSERVERTIMEOUT = 2
27# Set in httpserver.h
28MAX_HEADERS_SIZE = 8192
29MAX_BODY_SIZE = 32 * 1024 * 1024
30
31class BitcoinHTTPConnection:
32    def __init__(self, node):
33        self.url = urllib.parse.urlparse(node.url)
34        self.authpair = f'{self.url.username}:{self.url.password}'
35        self.headers = {"Authorization": f"Basic {str_to_b64str(self.authpair)}"}
36        self.reset_conn()
37
38    def reset_conn(self):
39        self.conn = http.client.HTTPConnection(self.url.hostname, self.url.port)
40        self.conn.connect()
41
42    def sock_closed(self):
43        if self.conn.sock is None:
44            return True
45        try:
46            self.conn.request('GET', '/')
47            self.conn.getresponse().read()
48            return False
49        except NETWORK_ERRORS:
50            return True
51
52    def close_sock(self):
53        self.conn.close()
54
55    def set_timeout(self, seconds):
56        self.conn.sock.settimeout(seconds)
57
58    def add_header(self, key, value):
59        self.headers.update({key: value})
60
61    def _request(self, method, path, data, connection_header, **kwargs):
62        headers = self.headers.copy()
63        if connection_header is not None:
64            headers["Connection"] = connection_header
65        self.conn.request(method, path, data, headers, **kwargs)
66        return self.conn.getresponse()
67
68    def post(self, path, data, connection_header=None, **kwargs):
69        return self._request('POST', path, data, connection_header, **kwargs)
70
71    def get(self, path, connection_header=None):
72        return self._request('GET', path, '', connection_header)
73
74    def send_raw(self, data):
75        self.conn.sock.sendall(data)
76
77    def post_raw(self, path, data):
78        data_bytes = data.encode("utf-8")
79        req = f"POST {path} HTTP/1.1\r\n"
80        req += f'Authorization: Basic {str_to_b64str(self.authpair)}\r\n'
81        req += f'Content-Length: {len(data_bytes)}\r\n\r\n'
82        self.send_raw(req.encode("ascii") + data_bytes)
83
84    def recv_raw(self):
85        '''
86        Blocking socket will wait until data is received and return up to 1024 bytes
87        '''
88        return self.conn.sock.recv(1024)
89
90    def expect_timeout(self, seconds):
91        # Wait for response, but expect a timeout disconnection
92        start = time.time()
93        response1 = self.recv_raw()
94        stop = time.time()
95        # Server disconnected with EOF
96        assert_equal(response1, b"")
97        # Server disconnected within an acceptable range of time:
98        # not immediately, and not too far over the configured duration.
99        # This allows for some jitter in the test between client and server.
100        duration = stop - start
101        assert duration <= seconds + 2, f"Server disconnected too slow: {duration} > {seconds}"
102        assert duration >= seconds - 1, f"Server disconnected too fast: {duration} < {seconds}"
103        # The connection is definitely closed.
104        assert self.sock_closed()
105
106class HTTPBasicsTest (BitcoinTestFramework):
107    def set_test_params(self):
108        self.num_nodes = 1
109
110    def setup_network(self):
111        self.setup_nodes()
112        self.node = self.nodes[0]
113
114    def send_bad_and_tolerate_disconnect(self, conn, predicate_fn):
115        '''
116        Tolerate a race condition when sending a malformed request that should result
117        in the server disconnecting the client. The server *should* be sending an error
118        response as well but in some conditions on some platforms (Windows) the python
119        client might encounter the socket error before processing the response.
120        '''
121        with self.node.assert_debug_log([f"HTTPResponse (status code: {http.client.BAD_REQUEST}"]):
122            try:
123                response = predicate_fn()
124                assert_equal(response.status, http.client.BAD_REQUEST)
125                self.log.info(f"Client received expected {http.client.BAD_REQUEST} response before connection was terminated")
126                # Drain server response
127                response.read()
128                conn.set_timeout(2)
129            except NETWORK_ERRORS:
130                self.log.info(f"Client did not receive expected {http.client.BAD_REQUEST} response before connection was terminated")
131        assert conn.sock_closed()
132
133
134    def run_test(self):
135        # The test framework typically reuses a single persistent HTTP connection
136        # for all RPCs to a TestNode. Because we are setting -rpcservertimeout
137        # so low on this one node, its connection will quickly timeout and get dropped by
138        # the server. Negating this setting will force the AuthServiceProxy
139        # for this node to create a fresh new HTTP connection for every command
140        # called for the remainder of this test.
141        self.node.reuse_http_connections = False
142
143        self.check_default_connection()
144        self.check_socket_exclusivity()
145        self.check_keepalive_connection()
146        self.check_close_connection()
147        self.check_excessive_request_size()
148        self.check_pipelining(with_invalid_second_request=False)
149        self.check_pipelining(with_invalid_second_request=True)
150        self.check_chunked_transfer()
151        self.check_idle_timeout()
152        self.check_server_busy_idle_timeout()
153        self.check_auth_required()
154        self.check_wrong_credentials()
155        self.check_malformed_auth_headers()
156        self.check_disallowed_http_methods()
157        self.check_path_traversal()
158        self.check_request_smuggling_cl_te()
159        self.check_duplicate_content_length()
160        self.check_null_byte_in_uri()
161        self.check_invalid_http_version()
162        self.check_whitespace_in_headers()
163        self.check_connection_limit()
164        self.check_pipelined_data_is_throttled()
165        self.check_slow_read_throttle()
166
167
168    def check_default_connection(self):
169        self.log.info("Checking default HTTP/1.1 connection persistence")
170        conn = BitcoinHTTPConnection(self.node)
171        # Make request without explicit "Connection" header
172        response1 = conn.post('/', '{"method": "getbestblockhash"}').read()
173        assert b'"error":null' in response1
174        # Connection still open after request
175        assert not conn.sock_closed()
176        # Make second request without explicit "Connection" header
177        response2 = conn.post('/', '{"method": "getchaintips"}').read()
178        assert b'"error":null' in response2
179        # Connection still open after second request
180        assert not conn.sock_closed()
181        # Close
182        conn.close_sock()
183        assert conn.sock_closed()
184
185
186    def check_socket_exclusivity(self):
187        self.log.info("Checking that another process cannot bind the HTTP listen port")
188        url = urllib.parse.urlparse(self.node.url)
189        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as competing_listener:
190            competing_listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
191            # Ill-configured sockets permit port reuse unless the original
192            # listener requested exclusive address use.
193            assert_raises(
194                OSError,
195                lambda: competing_listener.bind((url.hostname, url.port)))
196
197
198    def check_keepalive_connection(self):
199        self.log.info("Checking keep-alive connection persistence")
200        conn = BitcoinHTTPConnection(self.node)
201        # Make request with explicit "Connection: keep-alive" header
202        response1 = conn.post('/', '{"method": "getbestblockhash"}', connection_header='keep-alive').read()
203        assert b'"error":null' in response1
204        # Connection still open after request
205        assert not conn.sock_closed()
206        # Make second request with explicit "Connection" header
207        response2 = conn.post('/', '{"method": "getchaintips"}', connection_header='keep-alive').read()
208        assert b'"error":null' in response2
209        # Connection still open after second request
210        assert not conn.sock_closed()
211        # Close
212        conn.close_sock()
213        assert conn.sock_closed()
214
215
216    def check_close_connection(self):
217        self.log.info("Checking close connection after response")
218        conn = BitcoinHTTPConnection(self.node)
219        # Make request with explicit "Connection: close" header
220        response1 = conn.post('/', '{"method": "getbestblockhash"}', connection_header='close').read()
221        assert b'"error":null' in response1
222        # Connection closed after response
223        assert conn.sock_closed()
224
225
226    def check_excessive_request_size(self):
227        self.log.info("Checking excessive request size")
228
229        # Large URI plus up to 1000 bytes of default headers
230        # added by python's http.client still below total limit.
231        conn = BitcoinHTTPConnection(self.node)
232        response1 = conn.get(f'/{"x" * (MAX_HEADERS_SIZE - 1000)}')
233        assert_equal(response1.status, http.client.NOT_FOUND)
234
235        # Excessive URI size plus default headers breaks the limit.
236        conn = BitcoinHTTPConnection(self.node)
237        self.send_bad_and_tolerate_disconnect(conn, lambda: conn.get(f'/{"x" * MAX_HEADERS_SIZE}'))
238
239        # Compute how many short header lines need to be added to http.client
240        # default headers to make / break the total limit in a single request.
241        header_line_length = len("header_0000: foo\r\n")
242        headers_below_limit = (MAX_HEADERS_SIZE - 1000) // header_line_length
243        headers_above_limit = MAX_HEADERS_SIZE // header_line_length
244
245        # Many small header lines is ok
246        conn = BitcoinHTTPConnection(self.node)
247        for i in range(headers_below_limit):
248            conn.add_header(f"header_{i:04}", "foo")
249        response3 = conn.get('/x')
250        assert_equal(response3.status, http.client.NOT_FOUND)