Inside Bitcoin

1#!/usr/bin/env python3
2# Copyright (c) 2022 Pieter Wuille
3# Distributed under the MIT software license, see the accompanying
4# file LICENSE or http://www.opensource.org/licenses/mit-license.php.
5
6import argparse
7import sys
8import ipaddress
9import json
10import math
11from collections import defaultdict
12
13import asmap
14
15def load_file(input_file):
16    try:
17        contents = input_file.read()
18    except OSError as err:
19        sys.exit(f"Input file '{input_file.name}' cannot be read: {err.strerror}.")
20    try:
21        bin_asmap = asmap.ASMap.from_binary(contents)
22    except ValueError:
23        bin_asmap = None
24    txt_error = None
25    entries = None
26    try:
27        txt_contents = str(contents, encoding="utf-8")
28    except UnicodeError:
29        txt_error = "invalid UTF-8"
30        txt_contents = None
31    if txt_contents is not None:
32        entries = []
33        for line in txt_contents.split("\n"):
34            idx = line.find('#')
35            if idx >= 0:
36                line = line[:idx]
37            line = line.lstrip(' ').rstrip(' \t\r\n')
38            if len(line) == 0:
39                continue
40            fields = line.split(' ')
41            if len(fields) != 2:
42                txt_error = f"unparseable line '{line}'"
43                entries = None
44                break
45            prefix, asn = fields
46            if len(asn) <= 2 or asn[:2] != "AS" or any(c < '0' or c > '9' for c in asn[2:]):
47                txt_error = f"invalid ASN '{asn}'"
48                entries = None
49                break
50            try:
51                net = ipaddress.ip_network(prefix)
52            except ValueError:
53                txt_error = f"invalid network '{prefix}'"
54                entries = None
55                break
56            entries.append((asmap.net_to_prefix(net), int(asn[2:])))
57    if entries is not None and bin_asmap is not None and len(contents) > 0:
58        sys.exit(f"Input file '{input_file.name}' is ambiguous.")
59    if entries is not None:
60        state = asmap.ASMap()
61        state.update_multi(entries)
62        return state
63    if bin_asmap is not None:
64        return bin_asmap
65    sys.exit(f"Input file '{input_file.name}' is neither a valid binary asmap file nor valid text input ({txt_error}).")
66
67
68def save_binary(output_file, state, fill):
69    contents = state.to_binary(fill=fill)
70    try:
71        output_file.write(contents)
72        output_file.close()
73    except OSError as err:
74        sys.exit(f"Output file '{output_file.name}' cannot be written to: {err.strerror}.")
75
76def save_text(output_file, state, fill, overlapping):
77    for prefix, asn in state.to_entries(fill=fill, overlapping=overlapping):
78        net = asmap.prefix_to_net(prefix)
79        try:
80            print(f"{net} AS{asn}", file=output_file)
81        except OSError as err:
82            sys.exit(f"Output file '{output_file.name}' cannot be written to: {err.strerror}.")
83    try:
84        output_file.close()
85    except OSError as err:
86        sys.exit(f"Output file '{output_file.name}' cannot be written to: {err.strerror}.")
87
88def main():
89    parser = argparse.ArgumentParser(description="Tool for performing various operations on textual and binary asmap files.")
90    subparsers = parser.add_subparsers(title="valid subcommands", dest="subcommand")
91
92    parser_encode = subparsers.add_parser("encode", help="convert asmap data to binary format")
93    parser_encode.add_argument('-f', '--fill', dest="fill", default=False, action="store_true",
94                               help="permit reassigning undefined network ranges arbitrarily to reduce size")
95    parser_encode.add_argument('infile', nargs='?', type=argparse.FileType('rb'), default=sys.stdin.buffer,
96                               help="input asmap file (text or binary); default is stdin")
97    parser_encode.add_argument('outfile', nargs='?', type=argparse.FileType('wb'), default=sys.stdout.buffer,
98                               help="output binary asmap file; default is stdout")
99
100    parser_decode = subparsers.add_parser("decode", help="convert asmap data to text format")
101    parser_decode.add_argument('-f', '--fill', dest="fill", default=False, action="store_true",
102                               help="permit reassigning undefined network ranges arbitrarily to reduce length")
103    parser_decode.add_argument('-n', '--nonoverlapping', dest="overlapping", default=True, action="store_false",
104                               help="output strictly non-overall ping network ranges (increases output size)")
105    parser_decode.add_argument('infile', nargs='?', type=argparse.FileType('rb'), default=sys.stdin.buffer,
106                               help="input asmap file (text or binary); default is stdin")
107    parser_decode.add_argument('outfile', nargs='?', type=argparse.FileType('w'), default=sys.stdout,
108                               help="output text file; default is stdout")
109
110    parser_diff = subparsers.add_parser("diff", help="compute the difference between two asmap files")
111    parser_diff.add_argument('-i', '--ignore-unassigned', dest="ignore_unassigned", default=False, action="store_true",
112                             help="ignore unassigned ranges in the first input (useful when second input is filled)")
113    parser_diff.add_argument('infile1', type=argparse.FileType('rb'),
114                             help="first file to compare (text or binary)")
115    parser_diff.add_argument('infile2', type=argparse.FileType('rb'),
116                             help="second file to compare (text or binary)")
117
118    parser_diff_addrs = subparsers.add_parser("diff_addrs",
119                                              help="compute difference between two asmap files for a set of addresses")
120    parser_diff_addrs.add_argument('-s', '--show-addresses', dest="show_addresses", default=False, action="store_true",
121                                   help="include reassigned addresses in the output")
122    parser_diff_addrs.add_argument("infile1", type=argparse.FileType("rb"),
123                                   help="first file to compare (text or binary)")
124    parser_diff_addrs.add_argument("infile2", type=argparse.FileType("rb"),
125                                   help="second file to compare (text or binary)")
126    parser_diff_addrs.add_argument("addrs_file", type=argparse.FileType("r"),
127                                   help="address file containing getnodeaddresses output to use in the comparison "
128                                   "(make sure to set the count parameter to zero to get all node addresses, "
129                                   "e.g. 'bitcoin-cli getnodeaddresses 0 > addrs.json')")
130    args = parser.parse_args()
131    if args.subcommand is None:
132        parser.print_help()
133    elif args.subcommand == "encode":
134        if args.outfile.isatty():
135            sys.exit("Not much use in writing binary to a TTY. Please specify an output file or pipe output to another process.")
136        state = load_file(args.infile)
137        save_binary(args.outfile, state, fill=args.fill)
138    elif args.subcommand == "decode":
139        state = load_file(args.infile)
140        save_text(args.outfile, state, fill=args.fill, overlapping=args.overlapping)
141    elif args.subcommand == "diff":
142        state1 = load_file(args.infile1)
143        state2 = load_file(args.infile2)
144        ipv4_changed = 0
145        ipv4_entries_changed = 0
146        ipv6_changed = 0
147        ipv6_entries_changed = 0
148        for prefix, old_asn, new_asn in state1.diff(state2):
149            if args.ignore_unassigned and old_asn == 0:
150                continue
151            net = asmap.prefix_to_net(prefix)
152            if isinstance(net, ipaddress.IPv4Network):
153                ipv4_changed += net.num_addresses
154                ipv4_entries_changed += 1
155            elif isinstance(net, ipaddress.IPv6Network):
156                ipv6_changed += net.num_addresses
157                ipv6_entries_changed += 1
158            if new_asn == 0:
159                print(f"# {net} was AS{old_asn}")
160            elif old_asn == 0:
161                print(f"{net} AS{new_asn} # was unassigned")
162            else:
163                print(f"{net} AS{new_asn} # was AS{old_asn}")
164        ipv4_change_str = "" if ipv4_changed == 0 else f" (2^{math.log2(ipv4_changed):.2f})"
165        ipv6_change_str = "" if ipv6_changed == 0 else f" (2^{math.log2(ipv6_changed):.2f})"
166
167        print(
168f"""# Summary
169IPv4: {ipv4_entries_changed} entries with {ipv4_changed}{ipv4_change_str} addresses changed
170IPv6: {ipv6_entries_changed} entries with {ipv6_changed}{ipv6_change_str} addresses changed"""
171        )
172    elif args.subcommand == "diff_addrs":
173        state1 = load_file(args.infile1)
174        state2 = load_file(args.infile2)
175        address_info = json.load(args.addrs_file)
176        addrs = {a["address"] for a in address_info if a["network"] in ["ipv4", "ipv6"]}
177        reassignments = defaultdict(list)
178        for addr in addrs:
179            net = ipaddress.ip_network(addr)
180            prefix = asmap.net_to_prefix(net)
181            old_asn = state1.lookup(prefix)
182            new_asn = state2.lookup(prefix)
183            if new_asn != old_asn:
184                reassignments[(old_asn, new_asn)].append(addr)
185        reassignments = sorted(reassignments.items(), key=lambda item: len(item[1]), reverse=True)
186        num_reassignment_type = defaultdict(int)
187        for (old_asn, new_asn), reassigned_addrs in reassignments:
188            num_reassigned = len(reassigned_addrs)
189            num_reassignment_type[(bool(old_asn), bool(new_asn))] += num_reassigned
190            old_asn_str = f"AS{old_asn}" if old_asn else "unassigned"
191            new_asn_str = f"AS{new_asn}" if new_asn else "unassigned"
192            opt = ": " + ", ".join(reassigned_addrs) if args.show_addresses else ""
193            print(f"{num_reassigned} address(es) reassigned from {old_asn_str} to {new_asn_str}{opt}")
194        num_reassignments = sum(len(addrs) for _, addrs in reassignments)
195        share = num_reassignments / len(addrs) if len(addrs) > 0 else 0
196        print(f"Summary: {num_reassignments:,} ({share:.2%}) of {len(addrs):,} addresses were reassigned "
197              f"(migrations={num_reassignment_type[True, True]}, assignments={num_reassignment_type[False, True]}, "
198              f"unassignments={num_reassignment_type[True, False]})")
199    else:
200        parser.print_help()
201        sys.exit("No command provided.")
202
203if __name__ == '__main__':
204    main()
205