Inside Bitcoin

251            extra_args = self.extra_args
252
253        # If listening and no -bind is given, then bitcoind would bind P2P ports on
254        # 0.0.0.0:P and 127.0.0.1:P+1 (for incoming Tor connections), where P is
255        # a unique port chosen by the test framework and configured as port=P in
256        # bitcoin.conf. To avoid collisions, change it to 127.0.0.1:tor_port().
257        will_listen = all(e != "-nolisten" and e != "-listen=0" for e in extra_args)
258        has_explicit_bind = self.has_explicit_bind or any(e.startswith("-bind=") for e in extra_args)
259        if will_listen and not has_explicit_bind:
260            extra_args.append(f"-bind=0.0.0.0:{p2p_port(self.index)}")
261            extra_args.append(f"-bind=127.0.0.1:{tor_port(self.index)}=onion")
262
263        self.use_v2transport = "-v2transport=1" in extra_args or (self.default_to_v2 and "-v2transport=0" not in extra_args)
264
265        # Add a new stdout and stderr file each time bitcoind is started
266        if stderr is None:
267            stderr = tempfile.NamedTemporaryFile(dir=self.stderr_dir, delete=False)
268        if stdout is None:
269            stdout = tempfile.NamedTemporaryFile(dir=self.stdout_dir, delete=False)
270        self.stderr = stderr
271        self.stdout = stdout
272
273        if cwd is None:
274            cwd = self.cwd
275
276        # Delete any existing cookie file -- if such a file exists (eg due to
277        # unclean shutdown), it will get overwritten anyway by bitcoind, and
278        # potentially interfere with our attempt to authenticate
279        delete_cookie_file(self.datadir_path, self.chain)
280
281        # add environment variable LIBC_FATAL_STDERR_=1 so that libc errors are written to stderr and not the terminal
282        subp_env = dict(os.environ, LIBC_FATAL_STDERR_="1")
283        if self.use_gui:
284            subp_env.setdefault("QT_QPA_PLATFORM", "minimal")
285            if platform.system() == "Darwin":
286                # QMacStyle assumes a Cocoa platform window, which the minimal platform
287                # does not provide. In particular, painting a QGroupBox can make Qt call
288                # addSubview: on an invalid native object (QTBUG-49686).
289                subp_env.setdefault("QT_STYLE_OVERRIDE", "fusion")
290            if platform.system() == "OpenBSD":
291                # The system Qt packages are built with GLib support, so Qt uses
292                # QEventDispatcherGlib, which pushes/pops the GLib thread-default
293                # main context in each thread. On OpenBSD the pop can run during
294                # thread exit after GLib's per-thread context stack has already
295                # been torn down, so shutdown emits messages like
296                #   (process:NNN): GLib-CRITICAL **: g_main_context_pop_thread_default:
297                #   assertion 'stack != NULL' failed
298                # on stderr, which the test framework treats as a failure.
299                # Fall back to Qt's poll-based event dispatcher instead.
300                subp_env.setdefault("QT_NO_GLIB", "1")
301            subp_env.setdefault("LC_ALL", "nl_NL.UTF-8") # Set language to try to trigger translation bugs
302            if sys.platform.startswith("linux") and "XDG_RUNTIME_DIR" not in subp_env:
303                # Qt prints warnings to stderr when XDG_RUNTIME_DIR is unset or has wrong
304                # permissions (e.g. in CI environments without a desktop session), which
305                # would cause tests to fail due to unexpected stderr output. The two
306                # warnings are:
307                #   "QStandardPaths: XDG_RUNTIME_DIR not set, defaulting to '/tmp/runtime-root'"
308                #   "QStandardPaths: wrong permissions on runtime directory /path, 0755 instead of 0700"
309                # Use a dedicated subdirectory with the required 0700 permissions.
310                xdg_runtime_dir = self.datadir_path / "xdg_runtime"
311                xdg_runtime_dir.mkdir(mode=0o700, exist_ok=True)
312                subp_env["XDG_RUNTIME_DIR"] = str(xdg_runtime_dir)
313        if env is not None:
314            subp_env.update(env)
315
316        self.process = subprocess.Popen(self.args + extra_args, env=subp_env, stdout=stdout, stderr=stderr, cwd=cwd, **kwargs)
317
318        self.running = True
319        self.log.debug("bitcoind started, waiting for RPC to come up")
320
321    def create_new_rpc_connection(self, *, mode="AUTO", client_timeout=None):
322        """Create an additional RPC connection, likely to be used in a new thread."""
323        mode = RPCConnectionType[mode]
324        if mode == RPCConnectionType.AUTO:
325            mode = RPCConnectionType.CLI if self.use_cli else RPCConnectionType.AUTHPROXY
326        client_timeout = client_timeout or (self.rpc_timeout // 2)  # Shorter timeout to allow for one retry in case of ETIMEDOUT
327        host = "127.0.0.1"
328        port = rpc_port(self.index)
329        if self.rpchost:
330            parts = self.rpchost.split(":")
331            if len(parts) == 2:
332                host, port = parts
333            else:
334                host = self.rpchost
335        if mode == RPCConnectionType.AUTHPROXY:
336            rpc_u, rpc_p = get_auth_cookie(self.datadir_path, self.chain)
337            url = f"http://{rpc_u}:{rpc_p}@{host}:{port}"
338            proxy = AuthServiceProxy(url, timeout=int(client_timeout))
339            coverage_logfile = coverage.get_filename(self.coverage_dir, self.index) if self.coverage_dir else None
340            rpc = coverage.AuthServiceProxyWrapper(proxy, url, coverage_logfile)
341            rpc.auth_service_proxy_instance.reuse_http_connections = self.reuse_http_connections
342            return rpc
343        else:  # mode==CLI
344            return TestNodeCLI(self.binaries)(
345                f"-datadir={self.datadir_path}",
346                f"-rpcclienttimeout={client_timeout}",
347                f"-rpcconnect={host}",
348                f"-rpcport={port}",
349            )
350
351    def wait_for_rpc_connection(self, *, wait_for_import=True):
352        """Sets up an RPC connection to the bitcoind process. Returns False if unable to connect."""
353        # Poll at a rate of four times per second
354        poll_per_s = 4
355
356        suppressed_errors = collections.defaultdict(int)
357        latest_error = None
358        def suppress_error(category: str, e: Exception):
359            suppressed_errors[category] += 1
360            return (category, repr(e))
361
362        for _ in range(poll_per_s * self.rpc_timeout):
363            if self.process.poll() is not None:
364                # Attach abrupt shutdown error/s to the exception message
365                self.stderr.seek(0)
366                str_error = ''.join(line.decode('utf-8') for line in self.stderr)
367                str_error += "************************\n" if str_error else ''
368
369                raise FailedToStartError(self._node_msg(
370                    f'bitcoind exited with status {self.process.returncode} during initialization. {str_error}'))
371            try:
372                rpc = self.create_new_rpc_connection(mode="AUTHPROXY")
373                rpc.getblockcount()
374                # If the call to getblockcount() succeeds then the RPC connection is up
375                if self.version_is_at_least(190000) and wait_for_import:
376                    # getmempoolinfo.loaded is available since commit
377                    # bb8ae2c (version 0.19.0)
378                    self.wait_until(lambda: rpc.getmempoolinfo()['loaded'])
379                    # Wait for the node to finish reindex, block import, and
380                    # loading the mempool. Usually importing happens fast or
381                    # even "immediate" when the node is started. However, there
382                    # is no guarantee and sometimes ImportBlocks might finish
383                    # later. This is going to cause intermittent test failures,
384                    # because generally the tests assume the node is fully
385                    # ready after being started.
386                    #
387                    # For example, the node will reject block messages from p2p
388                    # when it is still importing with the error "Unexpected
389                    # block message received"
390                    #
391                    # The wait is done here to make tests as robust as possible
392                    # and prevent racy tests and intermittent failures as much
393                    # as possible. Some tests might not need this, but the
394                    # overhead is trivial, and the added guarantees are worth
395                    # the minimal performance cost.
396                self.log.debug("RPC successfully started")
397                # Set rpc_connected even if we are in use_cli mode so that we know we can call self.stop() if needed.
398                self.rpc_connected = True
399                self.url = rpc.rpc_url
400                self.cli = self.create_new_rpc_connection(mode="CLI")
401                if self.use_cli:
402                    return
403                self._rpc = rpc
404                return
405            except JSONRPCException as e:
406                # Suppress these as they are expected during initialization.
407                # -28 RPC in warmup
408                # -342 Service unavailable, could be starting up or shutting down
409                if e.error['code'] not in [-28, -342]:
410                    raise  # unknown JSON RPC exception
411                latest_error = suppress_error(f"JSONRPCException {e.error['code']}", e)
412            except OSError as e:
413                error_num = e.errno
414                if error_num is None:
415                    # Work around issue where socket timeouts don't have errno set.
416                    # https://github.com/python/cpython/issues/109601
417                    if isinstance(e, TimeoutError):
418                        error_num = errno.ETIMEDOUT
419                    # http.client.RemoteDisconnected inherits from this type and
420                    # doesn't specify errno.
421                    elif isinstance(e, ConnectionResetError):
422                        error_num = errno.ECONNRESET
423                    # Windows can raise this while bitcoind shuts down during startup.
424                    elif isinstance(e, ConnectionAbortedError):
425                        error_num = errno.ECONNABORTED
426
427                # Suppress similarly to the above JSONRPCException errors.
428                if error_num not in [
429                    errno.ECONNABORTED, # Treat identical to ECONNRESET
430                    errno.ECONNRESET,   # This might happen when the RPC server is in warmup,
431                                        # but shut down before the call to getblockcount succeeds.
432                    errno.ETIMEDOUT,    # Treat identical to ECONNRESET
433                    errno.ECONNREFUSED  # Port not yet open?
434                ]:
435                    raise  # unknown OS error
436                latest_error = suppress_error(f"OSError {errno.errorcode[error_num]}", e)
437            except ValueError as e:
438                # Suppress if cookie file isn't generated yet and no rpcuser or rpcpassword; bitcoind may be starting.
439                if "No RPC credentials" not in str(e):
440                    raise
441                latest_error = suppress_error("missing_credentials", e)
442            time.sleep(1.0 / poll_per_s)
443        self._raise_assertion_error(f"Unable to connect to bitcoind after {self.rpc_timeout}s (ignored errors: {dict(suppressed_errors)!s}{'' if latest_error is None else f', latest: {latest_error[0]!r}/{latest_error[1]}'})")
444
445    def wait_for_cookie_credentials(self):
446        """Ensures auth cookie credentials can be read, e.g. for testing CLI with -rpcwait before RPC connection is up."""
447        self.log.debug("Waiting for cookie credentials")
448        # Poll at a rate of four times per second.
449        poll_per_s = 4
450        for _ in range(poll_per_s * self.rpc_timeout):
451            try:
452                get_auth_cookie(self.datadir_path, self.chain)
453                self.log.debug("Cookie credentials successfully retrieved")
454                return
455            except ValueError:  # cookie file not found and no rpcuser or rpcpassword; bitcoind is still starting
456                pass            # so we continue polling until RPC credentials are retrieved
457            time.sleep(1.0 / poll_per_s)
458        self._raise_assertion_error("Unable to retrieve cookie credentials after {}s".format(self.rpc_timeout))
459
460    def generate(self, nblocks, maxtries=1000000, **kwargs):
461        self.log.debug("TestNode.generate() dispatches `generate` call to `generatetoaddress`")
462        return self.generatetoaddress(nblocks=nblocks, address=self.get_deterministic_priv_key().address, maxtries=maxtries, **kwargs)
463
464    def generateblock(self, *args, called_by_framework, **kwargs):
465        assert called_by_framework, "Direct call of this mining RPC is discouraged. Please use one of the self.generate* methods on the test framework, which sync the nodes to avoid intermittent test issues. You may use sync_fun=self.no_op to disable the sync explicitly."
466        return self.__getattr__('generateblock')(*args, **kwargs)
467
468    def generatetoaddress(self, *args, called_by_framework, **kwargs):
469        assert called_by_framework, "Direct call of this mining RPC is discouraged. Please use one of the self.generate* methods on the test framework, which sync the nodes to avoid intermittent test issues. You may use sync_fun=self.no_op to disable the sync explicitly."
470        return self.__getattr__('generatetoaddress')(*args, **kwargs)
471
472    def generatetodescriptor(self, *args, called_by_framework, **kwargs):
473        assert called_by_framework, "Direct call of this mining RPC is discouraged. Please use one of the self.generate* methods on the test framework, which sync the nodes to avoid intermittent test issues. You may use sync_fun=self.no_op to disable the sync explicitly."
474        return self.__getattr__('generatetodescriptor')(*args, **kwargs)
475
476    def setmocktime(self, timestamp):
477        """Wrapper for setmocktime RPC, sets self.mocktime"""
478        if timestamp == 0:
479            # setmocktime(0) resets to system time.
480            self.mocktime = None
481        else:
482            self.mocktime = timestamp
483        return self.__getattr__('setmocktime')(timestamp)
484
485    def get_wallet_rpc(self, wallet_name):
486        if self.use_cli:
487            return self.cli("-rpcwallet={}".format(wallet_name))
488        else:
489            assert self.rpc_connected and self._rpc, self._node_msg("RPC not connected")
490            wallet_path = "wallet/{}".format(urllib.parse.quote(wallet_name))
491            return self._rpc / wallet_path
492
493    def version_is_at_least(self, ver):
494        return self.version is None or self.version >= ver
495
496    def stop_node(self, expected_stderr='', *, wait=0, wait_until_stopped=True):
497        """Stop the node."""
498        if not self.running:
499            return
500        assert self.rpc_connected, self._node_msg(