Inside Bitcoin

251
252        # Too many small header lines exceeds total headers size allowed
253        conn = BitcoinHTTPConnection(self.node)
254        for i in range(headers_above_limit):
255            conn.add_header(f"header_{i:04}", "foo")
256        self.send_bad_and_tolerate_disconnect(conn, lambda: conn.get('/x'))
257
258        # Compute how much data we can add to a request message body
259        # to make / break the limit.
260        base_request_body_size = len('{"jsonrpc": "2.0", "id": "0", "method": "submitblock", "params": [""]}}')
261        bytes_below_limit = MAX_BODY_SIZE - base_request_body_size
262        bytes_above_limit = MAX_BODY_SIZE - base_request_body_size + 2
263
264        # Large request body size is ok
265        conn = BitcoinHTTPConnection(self.node)
266        response4 = conn.post('/', f'{{"jsonrpc": "2.0", "id": "0", "method": "submitblock", "params": ["{"0" * bytes_below_limit}"]}}')
267        assert_equal(response4.status, http.client.OK)
268
269        conn = BitcoinHTTPConnection(self.node)
270
271        # Split off the send into a background thread. When the server detects
272        # the excessive size it will stop reading from the socket, but the client
273        # will continue trying to write until the backpressure eventually
274        # drops the TCP window size to 0. While the send operation is blocking until
275        # it times out, we can still receive the server's response in the foreground.
276
277        def send_excessive_body(self, conn):
278            try:
279                # Excessive body size is invalid
280                conn.post_raw('/', f'{{"jsonrpc": "2.0", "id": "0", "method": "submitblock", "params": ["{"0" * bytes_above_limit}"]}}')
281                # On some platforms (e.g. Windows) the whole request may be
282                # accepted into the OS send buffer before the server disconnects.
283                # It's ok to allow that, the server-side behavior is asserted in
284                # the foreground thread via the 413 response.
285                self.log.info("Client finished sending request before connection was terminated")
286            except NETWORK_ERRORS:
287                self.log.info("Client did not finish sending request before connection was terminated")
288
289        send_thread = threading.Thread(target=send_excessive_body, args=(self, conn))
290        send_thread.start()
291
292        response5 = conn.recv_raw().decode()
293        assert "413 Content too large" in response5
294
295        try:
296            conn.conn.sock.shutdown(socket.SHUT_RDWR)
297            self.log.info("Send thread force-closed by test framework")
298        except OSError:
299            self.log.info("Send thread was already closed by RST from server")
300        send_thread.join()
301
302
303    def check_pipelining(self, with_invalid_second_request):
304        """
305        Requests are responded to in the order in which they were received
306        See https://www.rfc-editor.org/rfc/rfc7230#section-6.3.2
307        """
308        self.log.info("Check pipelining" + (" with invalid second request" if with_invalid_second_request else ""))
309        tip_height = self.node.getblockcount()
310        conn = BitcoinHTTPConnection(self.node)
311        conn.set_timeout(5)
312
313        # Send two requests in a row.
314        # The first request will block the second indefinitely
315        conn.post_raw('/', f'{{"method": "waitforblockheight", "params": [{tip_height + 1}]}}')
316        if with_invalid_second_request:
317            conn.post_raw(f'/{"x" * MAX_HEADERS_SIZE * 2}', '{"method": "getblockcount"}')
318        else:
319            conn.post_raw('/', '{"method": "getblockcount"}')
320
321        # The server should not respond to the second request until the first
322        # request has been handled. Since the server will not respond at all
323        # to the first request until we generate a block we expect a socket timeout.
324        assert_raises(TimeoutError, lambda: conn.recv_raw())
325
326        # Use a separate http connection to generate a block
327        self.generate(self.node, 1, sync_fun=self.no_op)
328
329        # Wait for responses to be received
330        if with_invalid_second_request:
331            OK = 1
332            BAD = 1
333        else:
334            OK = 2
335            BAD = 0
336        res = b""
337        while True:
338            res += conn.recv_raw()
339            if res.count(b"HTTP/1.1 200") == OK and res.count(b"HTTP/1.1 400") == BAD:
340                break
341
342        # waitforblockheight was responded to first, and then getblockcount
343        # which includes the block added after the request was made
344        chunks = res.split(b'"result":')
345        assert chunks[1].startswith(b'{"hash":')
346        if with_invalid_second_request:
347            # The response to the in-flight first request is sent before the
348            # error generated by parsing the second one, even though the second
349            # request could have been rejected much earlier.
350            assert res.index(b"HTTP/1.1 200") < res.index(b"HTTP/1.1 400")
351        else:
352            assert chunks[2].startswith(bytes(f'{tip_height + 1}', 'utf8'))
353
354
355    def check_chunked_transfer(self):
356        self.log.info("Check HTTP request encoded with chunked transfer")
357        conn = BitcoinHTTPConnection(self.node)
358        headers_chunked = conn.headers.copy()
359        headers_chunked.update({"Transfer-encoding": "chunked"})
360        body_chunked = [
361            b'{"method": "submitblock", "params": ["',
362            b'0' * 1000000,
363            b'1' * 1000000,
364            b'2' * 1000000,
365            b'3' * 1000000,
366            b'"]}'
367        ]
368        conn.conn.request(
369            method='POST',
370            url='/',
371            body=iter(body_chunked),
372            headers=headers_chunked,
373            encode_chunked=True)
374        response1 = conn.recv_raw()
375        assert b'{"result":"high-hash","error":null}\n' in response1
376
377        self.log.info("Check excessive size HTTP request encoded with chunked transfer")
378        conn = BitcoinHTTPConnection(self.node)
379        headers_chunked = conn.headers.copy()
380        headers_chunked.update({"Transfer-encoding": "chunked"})
381        body_chunked = [
382            b'{"method": "submitblock", "params": ["',
383            b'0' * 10000000,
384            b'1' * 10000000,
385            b'2' * 10000000,
386            b'3' * 10000000,
387            b'"]}'
388        ]
389
390        # Split off the send into a background thread. When the server detects
391        # the excessive size it will stop reading from the socket, but the client
392        # will continue trying to write until the backpressure eventually
393        # drops the TCP window size to 0. While the send operation is blocking until
394        # it times out, we can still receive the server's response in the foreground.
395
396        def send_excessive_chunked(self, conn):
397            try:
398                conn.conn.request(
399                    method='POST',
400                    url='/',
401                    body=iter(body_chunked),
402                    headers=headers_chunked,
403                    encode_chunked=True)
404                # On some platforms (e.g. Windows) the whole request may be
405                # accepted into the OS send buffer before the server disconnects.
406                # It's ok to allow that, the server-side behavior is asserted in
407                # the foreground thread via the 413 response.
408                self.log.info("Client finished sending request before connection was terminated")
409            except NETWORK_ERRORS:
410                self.log.info("Client did not finish sending request before connection was terminated")
411
412        send_thread = threading.Thread(target=send_excessive_chunked, args=(self, conn))
413        send_thread.start()
414
415        response2 = conn.recv_raw().decode()
416        assert "413 Content too large" in response2
417
418        try:
419            conn.conn.sock.shutdown(socket.SHUT_RDWR)
420            self.log.info("Send thread force-closed by test framework")
421        except OSError:
422            self.log.info("Send thread was already closed by RST from server")
423        send_thread.join()
424
425
426    def check_idle_timeout(self):
427        self.log.info("Check -rpcservertimeout")
428
429        # This is the amount of time the server will wait for a client to
430        # send a complete request. Test it by sending an incomplete but
431        # so-far otherwise well-formed HTTP request, and never finishing it.
432        self.restart_node(0, extra_args=[f"-rpcservertimeout={RPCSERVERTIMEOUT}"])
433
434        # Copied from http_incomplete_test_() in regress_http.c in libevent.
435        # A complete request would have an additional "\r\n" at the end.
436        bad_http_request = "GET /test1 HTTP/1.1\r\nHost: somehost\r\n"
437        conn = BitcoinHTTPConnection(self.node)
438        conn.send_raw(bad_http_request.encode("ascii"))
439
440        conn.expect_timeout(RPCSERVERTIMEOUT)
441
442        # Sanity check -- complete requests don't timeout waiting for completion
443        good_http_request = "GET /test2 HTTP/1.1\r\nHost: somehost\r\n\r\n"
444        conn.reset_conn()
445        conn.send_raw(good_http_request.encode("ascii"))
446        response = conn.recv_raw()
447        assert response.startswith(b"HTTP/1.1 404 Not Found")
448
449        # Still open
450        assert not conn.sock_closed()
451
452
453    def check_server_busy_idle_timeout(self):
454        self.log.info("Check that -rpcservertimeout won't close on a delayed response")
455
456        self.restart_node(0, extra_args=[f"-rpcservertimeout={RPCSERVERTIMEOUT}"])
457
458        tip_height = self.node.getblockcount()
459        conn = BitcoinHTTPConnection(self.node)
460        conn.post_raw('/', f'{{"method": "waitforblockheight", "params": [{tip_height + 1}]}}')
461
462        # Wait until after the timeout, then generate a block with a second HTTP connection
463        time.sleep(RPCSERVERTIMEOUT + 1)
464        generated_block = self.generate(self.node, 1, sync_fun=self.no_op)[0]
465
466        # The first connection gets the response it is patiently waiting for
467        response1 = conn.recv_raw().decode()
468        assert generated_block in response1
469        # The connection is still open
470        assert not conn.sock_closed()
471
472        # Now it will actually close due to idle timeout
473        conn.expect_timeout(RPCSERVERTIMEOUT)
474
475
476    def check_auth_required(self):
477        self.log.info("Check that requests without credentials return 401 Unauthorized with WWW-Authenticate")
478        conn = BitcoinHTTPConnection(self.node)
479        conn.headers = {}
480        response = conn.post('/', '{"method": "getbestblockhash"}')
481        assert_equal(response.status, http.client.UNAUTHORIZED)
482        assert response.getheader('WWW-Authenticate') is not None
483
484
485    def check_wrong_credentials(self):
486        self.log.info("Check that incorrect credentials return 401 Unauthorized")
487        conn = BitcoinHTTPConnection(self.node)
488        wrong_pair = f"{conn.url.username}:wrong_password"
489        conn.headers = {"Authorization": f"Basic {str_to_b64str(wrong_pair)}"}
490        response = conn.post('/', '{"method": "getbestblockhash"}')
491        assert_equal(response.status, http.client.UNAUTHORIZED)
492        assert response.getheader('WWW-Authenticate') is not None
493
494
495    def check_malformed_auth_headers(self):
496        self.log.info("Check that malformed Authorization headers return 401 Unauthorized")
497        cases = [
498            "Bearer sometoken123",
499            'Digest username="user", realm="test"',
500            "Basic !!!notbase64!!!",