Inside Bitcoin

1#!/usr/bin/env python3
2# Copyright (c) 2013-present The Bitcoin Core developers
3# Distributed under the MIT software license, see the accompanying
4# file COPYING or http://www.opensource.org/licenses/mit-license.php.
5#
6# Generate seeds.txt from Pieter's DNS seeder
7#
8
9import argparse
10import collections
11import ipaddress
12from pathlib import Path
13import random
14import re
15import sys
16from typing import Union
17
18asmap_dir = Path(__file__).parent.parent / "asmap"
19sys.path.append(str(asmap_dir))
20from asmap import ASMap, net_to_prefix  # noqa: E402
21
22NSEEDS=512
23
24MAX_SEEDS_PER_ASN = {
25    'ipv4': 2,
26    'ipv6': 10,
27}
28
29MIN_BLOCKS = 910000
30
31PATTERN_IPV4 = re.compile(r"^(([0-2]?\d{1,2})\.([0-2]?\d{1,2})\.([0-2]?\d{1,2})\.([0-2]?\d{1,2})):(\d{1,5})$")
32PATTERN_IPV6 = re.compile(r"^\[([\da-f:]+)]:(\d{1,5})$", re.IGNORECASE)
33PATTERN_ONION = re.compile(r"^([a-z2-7]{56}\.onion):(\d+)$")
34PATTERN_I2P = re.compile(r"^([a-z2-7]{52}\.b32\.i2p):(\d{1,5})$")
35PATTERN_AGENT = re.compile(
36    r"^/Satoshi:("
37    r"0\.14\.(0|1|2|3|99)"
38    r"|0\.15\.(0|1|2|99)"
39    r"|0\.16\.(0|1|2|3|99)"
40    r"|0\.17\.(0|0\.1|1|2|99)"
41    r"|0\.18\.(0|1|99)"
42    r"|0\.19\.(0|1|2|99)"
43    r"|0\.20\.(0|1|2|99)"
44    r"|0\.21\.(0|1|2|99)"
45    r"|22\.(0|1|99)\.0"
46    r"|23\.(0|1|2|99)\.0"
47    r"|24\.(0|1|2|99)\.(0|1)"
48    r"|25\.(0|1|2|99)\.0"
49    r"|26\.(0|1|2|99)\.0"
50    r"|27\.(0|1|2|99)\.0"
51    r"|28\.(0|1|2|3|4|99)\.0"
52    r"|29\.(0|1|2|3|99)\.0"
53    r"|30\.(0|1|2|3|99)\.0"
54    r"|31\.(0|1|99)\.0"
55    r")")
56
57def parseline(line: str) -> Union[dict, None]:
58    """ Parses a line from `seeds_main.txt` into a dictionary of details for that line.
59    or `None`, if the line could not be parsed.
60    """
61    if line.startswith('#'):
62        # Ignore line that starts with comment
63        return None
64    sline = line.split()
65    if len(sline) < 11:
66        # line too short to be valid, skip it.
67        return None
68    # Skip bad results.
69    if int(sline[1]) == 0:
70        return None
71    m = PATTERN_IPV4.match(sline[0])
72    sortkey = None
73    ip = None
74    if m is None:
75        m = PATTERN_IPV6.match(sline[0])
76        if m is None:
77            m = PATTERN_ONION.match(sline[0])
78            if m is None:
79                m = PATTERN_I2P.match(sline[0])
80                if m is None:
81                    return None
82                else:
83                    net = 'i2p'
84                    ipstr = sortkey = m.group(1)
85                    port = int(m.group(2))
86            else:
87                net = 'onion'
88                ipstr = sortkey = m.group(1)
89                port = int(m.group(2))
90        else:
91            net = 'ipv6'
92            if m.group(1) in ['::']: # Not interested in localhost
93                return None
94            ipstr = m.group(1)
95            if ipstr.startswith("fc"): # cjdns looks like ipv6 but always begins with fc
96                net = "cjdns"
97            sortkey = ipstr # XXX parse IPv6 into number, could use name_to_ipv6 from generate-seeds
98            port = int(m.group(2))
99    else:
100        # Do IPv4 sanity check
101        ip = 0
102        for i in range(0,4):
103            if int(m.group(i+2)) < 0 or int(m.group(i+2)) > 255:
104                return None
105            ip = ip + (int(m.group(i+2)) << (8*(3-i)))
106        if ip == 0:
107            return None
108        net = 'ipv4'
109        sortkey = ip
110        ipstr = m.group(1)
111        port = int(m.group(6))
112    # Extract uptime %.
113    uptime30 = float(sline[7][:-1])
114    # Extract Unix timestamp of last success.
115    lastsuccess = int(sline[2])
116    # Extract protocol version.
117    version = int(sline[10])
118    # Extract user agent.
119    agent = sline[11][1:-1]
120    # Extract service flags.
121    service = int(sline[9], 16)
122    # Extract blocks.
123    blocks = int(sline[8])
124    # Construct result.
125    return {
126        'net': net,
127        'ip': ipstr,
128        'port': port,
129        'ipnum': ip,
130        'uptime': uptime30,
131        'lastsuccess': lastsuccess,
132        'version': version,
133        'agent': agent,
134        'service': service,
135        'blocks': blocks,
136        'sortkey': sortkey,
137    }
138
139def dedup(ips: list[dict]) -> list[dict]:
140    """ Remove duplicates from `ips` where multiple ips share address and port. """
141    d = {}
142    for ip in ips:
143        ip_port = (ip["ip"], ip["port"])
144        if ip_port not in d or ip["lastsuccess"] > d[ip_port]["lastsuccess"]:
145            d[ip_port] = ip
146    return list(d.values())
147
148def filtermultiport(ips: list[dict]) -> list[dict]:
149    """ Filter out hosts with more nodes per IP"""
150    hist = collections.defaultdict(list)
151    for ip in ips:
152        hist[ip['sortkey']].append(ip)
153    return [value[0] for (key,value) in list(hist.items()) if len(value)==1]
154
155# Based on Greg Maxwell's seed_filter.py
156def filterbyasn(asmap: ASMap, ips: list[dict], max_per_asn: dict, max_per_net: int) -> list[dict]:
157    """ Prunes `ips` by
158    (a) trimming ips to have at most `max_per_net` ips from each net (e.g. ipv4, ipv6); and
159    (b) trimming ips to have at most `max_per_asn` ips from each asn in each net.
160    """
161    # Sift out ips by type
162    ips_ipv46 = [ip for ip in ips if ip['net'] in ['ipv4', 'ipv6']]
163    ips_onion = [ip for ip in ips if ip['net'] == 'onion']
164    ips_i2p = [ip for ip in ips if ip['net'] == 'i2p']
165    ips_cjdns = [ip for ip in ips if ip["net"] == "cjdns"]
166
167    # Filter IPv46 by ASN, and limit to max_per_net per network
168    result = []
169    net_count: dict[str, int] = collections.defaultdict(int)
170    asn_count: dict[int, int] = collections.defaultdict(int)
171
172    for i, ip in enumerate(ips_ipv46):
173        if net_count[ip['net']] == max_per_net:
174            # do not add this ip as we already too many
175            # ips from this network
176            continue
177        asn = asmap.lookup(net_to_prefix(ipaddress.ip_network(ip['ip'])))
178        if not asn or asn_count[ip['net'], asn] == max_per_asn[ip['net']]:
179            # do not add this ip as we already have too many
180            # ips from this ASN on this network
181            continue
182        asn_count[ip['net'], asn] += 1
183        net_count[ip['net']] += 1
184        ip['asn'] = asn
185        result.append(ip)
186
187    # Add back Onions (up to max_per_net)
188    result.extend(ips_onion[0:max_per_net])
189    result.extend(ips_i2p[0:max_per_net])
190    result.extend(ips_cjdns[0:max_per_net])
191    return result
192
193def ip_stats(ips: list[dict]) -> str:
194    """ Format and return pretty string from `ips`. """
195    hist: dict[str, int] = collections.defaultdict(int)
196    for ip in ips:
197        if ip is not None:
198            hist[ip['net']] += 1
199
200    return f"{hist['ipv4']:6d} {hist['ipv6']:6d} {hist['onion']:6d} {hist['i2p']:6d} {hist['cjdns']:6d}"
201
202def parse_args():
203    argparser = argparse.ArgumentParser(description='Generate a list of bitcoin node seed ip addresses.')
204    argparser.add_argument("-a","--asmap", help='the location of the asmap asn database file (required)', required=True)
205    argparser.add_argument("-s","--seeds", help='the location of the DNS seeds file (required)', required=True)
206    argparser.add_argument("-m", "--minblocks", help="The minimum number of blocks each node must have", default=MIN_BLOCKS, type=int)
207    return argparser.parse_args()
208
209def main():
210    args = parse_args()
211
212    print(f'Loading asmap database "{args.asmap}"…', end='', file=sys.stderr, flush=True)
213    with open(args.asmap, 'rb') as f:
214        asmap = ASMap.from_binary(f.read())
215    print('Done.', file=sys.stderr)
216
217    print('Loading and parsing DNS seeds…', end='', file=sys.stderr, flush=True)
218    with open(args.seeds, 'r') as f:
219        lines = f.readlines()
220    ips = [parseline(line) for line in lines]
221    random.shuffle(ips)
222    print('Done.', file=sys.stderr)
223
224    print('\x1b[7m  IPv4   IPv6  Onion    I2P  CJDNS Pass                                               \x1b[0m', file=sys.stderr)
225    print(f'{ip_stats(ips):s} Initial', file=sys.stderr)
226    # Skip entries with invalid address.
227    ips = [ip for ip in ips if ip is not None]
228    print(f'{ip_stats(ips):s} Skip entries with invalid address', file=sys.stderr)
229    # Skip duplicates (in case multiple seeds files were concatenated)
230    ips = dedup(ips)
231    print(f'{ip_stats(ips):s} After removing duplicates', file=sys.stderr)
232    # Enforce minimal number of blocks.
233    ips = [ip for ip in ips if ip['blocks'] >= args.minblocks]
234    print(f'{ip_stats(ips):s} Enforce minimal number of blocks', file=sys.stderr)
235    # Require service bit 1.
236    ips = [ip for ip in ips if (ip['service'] & 1) == 1]
237    print(f'{ip_stats(ips):s} Require service bit 1', file=sys.stderr)
238    # Require at least 50% 30-day uptime for clearnet, onion and i2p; 10% for cjdns
239    req_uptime = {
240        'ipv4': 50,
241        'ipv6': 50,
242        'onion': 50,
243        'i2p': 50,
244        'cjdns': 10,
245    }
246    ips = [ip for ip in ips if ip['uptime'] > req_uptime[ip['net']]]
247    print(f'{ip_stats(ips):s} Require minimum uptime', file=sys.stderr)
248    # Require a known and recent user agent.
249    ips = [ip for ip in ips if PATTERN_AGENT.match(ip['agent'])]
250    print(f'{ip_stats(ips):s} Require a known and recent user agent', file=sys.stderr)