{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": { "collapsed": true, "scrolled": false }, "outputs": [], "source": [ "# Written by Constantijn Bicker Caarten\n", "# Last updated: 07-07-2017\n", "#\n", "#\n", "# This code gathers data on TLDs in the DNS. This data \n", "# consists out of A, AAAA, DNSKEY, DS and NS records, \n", "# as well as TCP and UDP support, response time and\n", "# anycast support." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Imports" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "collapsed": true }, "outputs": [], "source": [ "from dns.resolver import Resolver, NXDOMAIN, query\n", "from socket import error as socket_error\n", "from urllib.request import urlopen\n", "from dns.query import udp, tcp\n", "from bs4 import BeautifulSoup\n", "from dns.resolver import dns\n", "from uuid import uuid4\n", "from tqdm import tqdm\n", "\n", "import matplotlib as plt\n", "import pandas as pd\n", "import numpy as np\n", "\n", "import datetime\n", "import requests\n", "import socket\n", "import copy\n", "import json\n", "import os" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Constants" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "collapsed": true }, "outputs": [], "source": [ "ZSK = 256\n", "KSK = 257\n", "\n", "ATLAS_API_KEY = '' # Add your Atlas API key\n", "ATLAS_BILL_TO = '' # Add your Atlas account email\n", "\n", "URL_DNS_MEASUREMENT_CREATE = 'https://atlas.ripe.net:443/api/v2/measurements/dns/'\n", "URL_DNS_MEASUREMENT_GET = 'https://atlas.ripe.net:443/api/v2/measurements/dns/'\n", "\n", "HEADERS = {'Content-type': 'application/json', 'Accept': 'text/plain'}\n", "\n", "NEWLINE = '\\n'" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "collapsed": true }, "outputs": [], "source": [ "payload = {\n", " 'bill_to': ATLAS_BILL_TO,\n", " 'is_oneoff': True,\n", " 'definitions': [],\n", " 'probes': []\n", "}\n", " \n", "definition = {\n", " 'af': 4,\n", " 'query_class': 'IN',\n", " 'query_type': '',\n", " 'query_argument': '',\n", " 'description': '',\n", " 'use_probe_resolver': True,\n", " 'resolve_on_probe': False,\n", " 'set_nsid_bit': True,\n", " 'protocol': 'UDP',\n", " 'udp_payload_size': 512,\n", " 'retry': 0,\n", " 'skip_dns_check': False,\n", " 'include_qbuf': False,\n", " 'include_abuf': True,\n", " 'prepend_probe_id': False,\n", " 'set_rd_bit': False,\n", " 'set_do_bit': False,\n", " 'set_cd_bit': False,\n", " 'type': 'dns',\n", " 'is_public': True\n", "}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Functions" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "collapsed": true, "scrolled": false }, "outputs": [], "source": [ "def write_list(fn, data):\n", " '''Writes a list to a file with each value on a new line'''\n", " with open(fn, 'w') as f:\n", " for datum in data:\n", " f.write(datum + NEWLINE)\n", " \n", "def append_list(fn, data):\n", " '''Appends a list to a file with each value on a new line'''\n", " with open(fn, 'a') as f:\n", " for datum in data:\n", " f.write(datum + NEWLINE)\n", " \n", "def read_list(fn):\n", " '''Reads a file and '''\n", " with open(fn, 'r') as f:\n", " return [line.strip(NEWLINE) for line in f]\n", " \n", "def write_json(fn, data):\n", " with open(fn, 'w') as f:\n", " f.write(json.dumps(data)) \n", " \n", "def read_json(fn):\n", " '''Read a json file (fn) and returns it as a dictionary'''\n", " with open(fn, 'r') as f:\n", " return json.loads(f.read())" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "collapsed": true }, "outputs": [], "source": [ "def write_data(fn, data):\n", " \"\"\"Backs up the previous version of the data if it exists and writes the new data to a file.\"\"\"\n", " # Backs up the previous data if it exists.\n", " try:\n", " now = datetime.datetime.now().strftime('%H:%M-%d-%m-%Y')\n", " write_json(\"data/backup/{}_{}.json \".format(fn, now), \n", " read_json(\"data/{}.json\".format(fn)))\n", " except:\n", " pass\n", "\n", " write_json(\"data/{}.json\".format(fn), data)" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "collapsed": true }, "outputs": [], "source": [ "def find(lst, key, value):\n", " '''Finds the first index of a list \n", " lst where the key matches the value'''\n", " for index, dic in enumerate(lst):\n", " if dic[key] == value:\n", " return index\n", " \n", " return None" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "collapsed": true }, "outputs": [], "source": [ "def ns_ips(fn):\n", " ns_ips = {}\n", "\n", " with open(fn, 'r') as f:\n", " for line in f:\n", " if not line.startswith(';;'):\n", " ns, _, _, _, ip = line.split()\n", " ns = ns[:-1]\n", "\n", " if ns in ns_ips and ip not in ns_ips[ns]:\n", " ns_ips[ns].append(ip)\n", " else:\n", " ns_ips[ns] = [ip]\n", " \n", " return ns_ips" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "collapsed": true, "scrolled": true }, "outputs": [], "source": [ "class CustomDNSException(Exception):\n", " pass\n", "\n", "def test_tcp_udp(data, timeout = 5):\n", " data_copy = copy.deepcopy(data)\n", " \n", " pbar = tqdm(total=len(data_copy))\n", "\n", " for datum in data_copy:\n", " protocols = []\n", " \n", " if 'tcp' in datum and not datum['tcp']:\n", " protocols.append(udp)\n", " \n", " if ('udp' in datum and not datum['udp']):\n", " protocols.append(tcp)\n", " \n", " for p in protocols:\n", " # Create SOA query\n", " m = dns.message.make_query(datum['tld'], dns.rdatatype.SOA)\n", " \n", " try: \n", " a = p(m, datum['ip'], timeout = timeout)\n", " \n", " # We expect NOERROR RCODE (0) and an answer\n", " if a.rcode() == 0 and len(a.answer) > 0:\n", " datum[p.__name__] = True\n", " else:\n", " raise CustomDNSException('failed')\n", " \n", " except (dns.exception.Timeout, socket_error, CustomDNSException):\n", " datum[p.__name__] = False\n", "\n", " pbar.update(1)\n", " pbar.close()\n", " \n", " return data_copy" ] }, { "cell_type": "code", "execution_count": 10, "metadata": { "collapsed": true }, "outputs": [], "source": [ "def find_nxdomain(tld, max_tries = 3):\n", " for _ in range(max_tries):\n", " domain = '{}.{}'.format(str(uuid4()), tld)\n", " \n", " try:\n", " query(domain)\n", " except NXDOMAIN:\n", " return domain\n", " except:\n", " pass\n", " \n", " return None\n", "\n", "def find_nxdomain_wildcard(tld, max_tries = 3):\n", " for _ in range(max_tries):\n", " domain = '{}.{}'.format(str(uuid4()), tld)\n", "\n", " response = !dig soa +noall +authority +noidn {domain}\n", "\n", " if len(response) > 0 and response[0].startswith(tld):\n", " return domain\n", " \n", " return None" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Init" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# First time run\n", "!mkdir data\n", "\n", "!mkdir data/dig\n", "!mkdir data/lists\n", "!mkdir data/cymru\n", "!mkdir data/whois\n", "!mkdir data/atlas\n", "!mkdir data/atlas/ns\n", "!mkdir data/atlas/soa\n", "!mkdir data/backup" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Top-Level Domains" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "--2017-07-12 20:37:59-- https://data.iana.org/TLD/tlds-alpha-by-domain.txt\n", "Loaded CA certificate '/etc/ssl/certs/ca-certificates.crt'\n", "Resolving data.iana.org... 72.21.81.189, 2606:2800:11f:bb5:f27:227f:1bbf:a0e\n", "Connecting to data.iana.org|72.21.81.189|:443... connected.\n", "HTTP request sent, awaiting response... 200 OK\n", "Length: 10433 (10K) [text/plain]\n", "Saving to: ‘data/lists/tlds’\n", "\n", "data/lists/tlds 100%[===================>] 10.19K --.-KB/s in 0s \n", "\n", "2017-07-12 20:37:59 (189 MB/s) - ‘data/lists/tlds’ saved [10433/10433]\n", "\n" ] } ], "source": [ "!wget https://data.iana.org/TLD/tlds-alpha-by-domain.txt -O data/lists/tlds\n", "!sed -i '1d' data/lists/tlds # remove header" ] }, { "cell_type": "code", "execution_count": 12, "metadata": { "collapsed": true }, "outputs": [], "source": [ "tlds = read_list('data/lists/tlds')" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# Gathers WHOIS records for each TLD.\n", "\n", "pbar = tqdm(total=len(tlds))\n", "\n", "for tld in tlds:\n", " !whois -h whois.iana.org:43 {tld} > data/whois/{tld} \n", " \n", " pbar.update(1)\n", "pbar.close()" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Gathering empty WHOIS records.\n", "Gathering missing WHOIS records.\n" ] } ], "source": [ "# Gathers empty or missing WHOIS records.\n", "\n", "indir = 'data/whois/'\n", "\n", "for root, dirs, filenames in os.walk(indir):\n", " print(\"Gathering empty WHOIS records.\")\n", " for f in filenames:\n", " stat = os.stat(indir + f)\n", " \n", " if stat.st_size == 0:\n", " !whois -h whois.iana.org:43 {f} > data/whois/{f}\n", " \n", " print(\"Gathering missing WHOIS records.\")\n", " for tld in tlds:\n", " if tld not in filenames:\n", " !whois -h whois.iana.org:43 {tld} > data/whois/{tld}" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [], "source": [ "# Extracts the creation date and organisations for each TLD from the WHOIS record.\n", "\n", "data_tlds = [{'tld': tld, 'organisations': []} for tld in tlds]\n", "\n", "for root, dirs, filenames in os.walk(indir):\n", " for fn in filenames:\n", " with open(indir + fn, 'r') as f:\n", " for line in f:\n", " if line.startswith('created'): \n", " index = find(data_tlds, 'tld', fn)\n", " data_tlds[index]['creation_date'] = line.split()[-1]\n", " elif line.startswith('organisation'):\n", " _, org = line.split('rganisation:')\n", " index = find(data_tlds, 'tld', fn)\n", " data_tlds[index]['organisations'].append(org.strip(NEWLINE))" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [], "source": [ "# Gets the type of each TLD listed in the table of the url.\n", "\n", "url = \"https://www.iana.org/domains/root/db/\"\n", "html = urlopen(url)\n", "soup = BeautifulSoup(html, 'html5lib')\n", "\n", "for table in soup.find_all(attrs={'class': 'iana-table'}):\n", " values = [td.get_text(strip=True) for td in table.find_all('td')]\n", " values = [td for td in table.find_all('td')]\n", "\n", "for i in range(0, len(values), 3):\n", " tld = str(values[i].findAll('a', href=True)[0]).split('.html')[0][26:].upper()\n", " index = find(data_tlds, 'tld', tld)\n", " \n", " if index != None:\n", " data_tlds[index]['type'] = values[i + 1].get_text(strip = True)" ] }, { "cell_type": "code", "execution_count": 16, "metadata": { "collapsed": true }, "outputs": [], "source": [ "write_data('data_tlds', data_tlds)" ] }, { "cell_type": "code", "execution_count": 17, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# Special TLDs are the same as record types or classes which do not work in some bulk operations.\n", "special_tlds = ['CH', 'IN', 'MD', 'MG', 'MR', 'MX']\n", "write_list('data/lists/tlds', [tld for tld in tlds if tld not in special_tlds])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Name Servers" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# Gathers the name servers of every TLD using dig.\n", "print('Gathering name servers.')\n", "!dig +noall +answer +noidn -t NS -f data/lists/tlds > data/dig/tld_nss\n", "\n", "for tld in special_tlds:\n", " !dig +noall +answer +noidn -t NS {tld} >> data/dig/tld_nss\n", "\n", "print('Done.')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# Parses the NS records.\n", "\n", "data_ns = []\n", "\n", "with open('data/dig/tld_nss', 'r') as f:\n", " for line in f:\n", " if not line.startswith('.'):\n", " tld, _, _, _, ns = line.split()\n", " data_ns.append({'tld': tld[:-1], 'ns': ns.lower()[:-1]})\n", " \n", "write_data('data_ns', data_ns)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# IP Addresses" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "write_list('data/lists/nss', set([datum['ns'] for datum in data_ns]))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "print('Gathering IPv4 addresses.')\n", "!dig +noall +answer +noidn A -f data/lists/nss > data/dig/ns_ipv4s\n", "print('Done.')\n", "\n", "print('Gathering IPv6 addresses.')\n", "!dig +noall +answer +noidn AAAA -f data/lists/nss > data/dig/ns_ipv6s\n", "print('Done.')" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "ns_ipv4s = ns_ips('data/dig/ns_ipv4s')\n", "ns_ipv6s = ns_ips('data/dig/ns_ipv6s')\n", "\n", "data_ips = []\n", "\n", "# Adds the IP adress to the dictionary and creates a copy \n", "# in case a name server has multiple IP addresses or has \n", "# both a IPv4 and IPv6 address.\n", "for datum in data_ns:\n", " if datum['ns'] in ns_ipv4s:\n", " for ip in ns_ipv4s[datum['ns']]:\n", " new_datum = copy.deepcopy(datum)\n", " new_datum['ip'] = ip\n", " data_ips.append(new_datum)\n", " \n", " if datum['ns'] in ns_ipv6s:\n", " for ip in ns_ipv6s[datum['ns']]:\n", " new_datum = copy.deepcopy(datum)\n", " new_datum['ip'] = ip\n", " data_ips.append(new_datum)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "write_data('data_ips', data_ips)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Autonomous System Number" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "write_list('data/lists/ips', ['begin'])\n", "append_list('data/lists/ips', set([datum['ip'] for datum in data_ips]))\n", "append_list('data/lists/ips', ['end'])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true, "scrolled": false }, "outputs": [], "source": [ "!netcat whois.cymru.com 43 < data/lists/ips | sort -n > data/cymru/ip_asns" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true, "scrolled": false }, "outputs": [], "source": [ "# Makes a dictionary with the IP addresses as key and a list of ASNs as value.\n", "\n", "ip_asns = {}\n", "\n", "with open('data/cymru/ip_asns', 'r') as f:\n", " for line in f:\n", " if not line.startswith('Bulk') and not line.startswith('NA'):\n", " \n", " asn, ip, org = [value.strip() for value in line.split('|')]\n", " \n", " if ip in ip_asns and asn not in ip_asns[ip]:\n", " ip_asns[ip].append(asn)\n", " else:\n", " ip_asns[ip] = [asn]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "data_asns = []\n", "\n", "for datum in data_ips:\n", " if datum['ip'] in ip_asns:\n", " for asn in ip_asns[datum['ip']]:\n", " new_datum = copy.deepcopy(datum)\n", " new_datum['asn'] = asn\n", " data_asns.append(new_datum)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "write_data('data_asns', data_asns)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Reachability" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "data_ips = read_json('data/data_ips.json')" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "data_reach = test_tcp_udp(data_ips)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "scrolled": true }, "outputs": [], "source": [ "# Retries testing TCP or UDP\n", "data_reach = test_tcp_udp(data_reach, timeout = 15)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "write_data('data_reach', data_reach)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Credibility" ] }, { "cell_type": "code", "execution_count": 20, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Gathering DNSKEY records.\n", "Done\n", "Gathering DS records.\n", "Done.\n", "Gathering DNSKEY and DS records for special TLDs.\n", "Done\n" ] } ], "source": [ "print('Gathering DNSKEY records.')\n", "!dig +noall +answer +noidn -t DNSKEY -f data/lists/tlds > data/dig/tld_dnskeys\n", "print('Done')\n", "\n", "print('Gathering DS records.')\n", "!dig +noall +answer +noidn -t DS -f data/lists/tlds > data/dig/tld_dss\n", "print('Done.')\n", "\n", "print('Gathering DNSKEY and DS records for special TLDs.')\n", "for tld in special_tlds:\n", " !dig +noall +answer +noidn -t DNSKEY {tld} >> data/dig/tld_dnskeys\n", " !dig +noall +answer +noidn -t DS {tld} >> data/dig/tld_dss\n", "print('Done')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# Parses the DNSKEY records.\n", "\n", "data_cred = [{'tld': tld, 'ds': False, 'dnskey': False, 'jsj': None} for tld in tlds]\n", "\n", "for answer in read_list('data/dig/tld_dnskeys'):\n", " answer_fields = answer.split()\n", " tld = answer_fields[0][:-1].upper()\n", " index = find(data_cred, 'tld', tld)\n", " \n", " try:\n", " data_cred[index]['dnskey'] = True\n", " \n", " if int(answer_fields[4]) == KSK:\n", " data_cred[index]['ksk'] = answer_fields[6]\n", " elif int(answer_fields[4]) == ZSK:\n", " data_cred[index]['zsk'] = answer_fields[6]\n", " except:\n", " print(tld)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# Parses the DS records.\n", "\n", "for answer in read_list('data/dig/tld_dss'):\n", " v = answer.split()\n", " tld = v[0][:-1].upper()\n", " \n", " index = find(data_cred, 'tld', tld)\n", " \n", " try:\n", " data_cred[index]['ds'] = True\n", " except:\n", " print(tld)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "write_data('data_cred', data_cred)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Performance" ] }, { "cell_type": "code", "execution_count": 22, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 1547/1547 [01:51<00:00, 13.85it/s]\n" ] } ], "source": [ "# Generates domains that result in a NXDOMAIN response for each TLD.\n", "data_test_perf = []\n", "pbar = tqdm(total=len(tlds))\n", "\n", "for tld in tlds:\n", " data_test_perf.append({'tld': tld, 'domain': find_nxdomain(tld)})\n", " \n", " pbar.update(1)\n", "pbar.close()" ] }, { "cell_type": "code", "execution_count": 23, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 38/38 [00:05<00:00, 3.85it/s]\n" ] } ], "source": [ "# Generates domains that result in a NXDOMAIN response for each TLD that uses wildcards.\n", "pbar = tqdm(total=len([datum for datum in data_test_perf if not datum['domain']]))\n", "\n", "for datum in [datum for datum in data_test_perf if not datum['domain']]:\n", " datum['domain'] = find_nxdomain_wildcard(datum['tld'])\n", " \n", " pbar.update(1)\n", "pbar.close()" ] }, { "cell_type": "code", "execution_count": 24, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# Set the probes.\n", "probe_ids = [10262, 10287, 11040, 11429, 12515, 12873, 12956, 13623, 13728, 13769, 13788, 13799, 13804, \n", " 13805, 13810, 14237, 26057, 14564, 15156, 14691, 15594, 15799, 4205, 18131, 18195, 18691, \n", " 19326, 19740, 20111, 20353, 20493, 20531, 20621, 21003, 21035, 21122, 21251, 21345, 21703, \n", " 22286, 22695, 23031, 23085, 28240, 27972, 23697, 24807, 25011, 25148, 25323, 26936, 26378, \n", " 26627, 4155, 26823, 28355, 30676, 4829, 29006, 29183, 29405, 30225, 30324, 31201, 19306, \n", " 19634, 6025, 11660, 22388, 25182, 4123, 3812, 20923, 14384, 12389]\n", "\n", "probes = [\n", " {\n", " \"value\": str(probe_ids)[1:-1],\n", " \"type\": \"probes\",\n", " \"requested\": len(probe_ids)\n", " }\n", "]" ] }, { "cell_type": "code", "execution_count": 28, "metadata": {}, "outputs": [], "source": [ "payloads = []\n", "payload_size = 100 # max 100\n", "step_size = int(payload_size / 2)\n", "\n", "for i in range(0, len(data_test_perf), step_size):\n", " defintions = []\n", " \n", " for datum in data_test_perf[i:i + step_size]:\n", " # Create caching measurement\n", " definition_caching = definition.copy()\n", " definition_caching['query_type'] = \"NS\"\n", " definition_caching['query_argument'] = datum['tld']\n", " definition_caching['description'] = \"caching \" + datum['tld']\n", " defintions.append(definition_caching)\n", " \n", " # Create response time measurement\n", " definition_measuring = definition.copy()\n", " definition_measuring['query_type'] = \"SOA\"\n", " definition_measuring['query_argument'] = datum['domain']\n", " definition_measuring['description'] = \"measuring \" + datum['tld']\n", " defintions.append(definition_measuring) \n", "\n", " new_payload = payload.copy()\n", " new_payload['probes'] = probes\n", " new_payload['definitions'] = defintions\n", "\n", " payloads.append(new_payload)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "write_data('payloads', payloads)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "measurement_ids = []\n", "measurement_responses = []\n", "url = URL_DNS_MEASUREMENT_CREATE + '?key=' + ATLAS_API_KEY" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# Only 10 payloads can be sent per day.\n", "# Manually set this slice each day.\n", "for payload in payloads[:10]:\n", " \n", " request = requests.post(url, data = json.dumps(payload), headers = HEADERS)\n", " print(request.status_code)\n", " \n", " while request.status_code == 400:\n", " print(request.json())\n", " request = requests.post(url, data = json.dumps(payload), headers = HEADERS)\n", " time.sleep(300)\n", " print(request.status_code)\n", " \n", " measurement_ids += measurement_ids + request.json()\n", " measurement_ids = [min(measurement_ids), max(measurement_ids)]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# Retrieves the result of each measurement and writes it to a file.\n", "\n", "next_result = True\n", "url = '{}?id__gte={}&id__lte={}&mine=true'.format(URL_DNS_MEASUREMENT_GET, min(measurement_ids), max(measurement_ids))\n", "measurements = requests.get(url).json()\n", "pbar = tqdm(total=len(tlds))\n", "\n", "while next_result:\n", " for result in measurements['results']:\n", " if len(result['description'].split()) == 2:\n", " measurement_type, tld = result['description'].split()\n", " request = requests.get(result['result']).json()\n", "\n", " if measurement_type == 'measuring':\n", " with open('data/atlas/soa/{}.json'.format(tld.upper()), 'w') as f:\n", " temp = copy.deepcopy(result)\n", " temp['result'] = request\n", "\n", " f.write(json.dumps(temp))\n", "\n", " elif measurement_type == 'caching':\n", " with open('data/atlas/ns/{}.json'.format(tld.upper()), 'w') as f:\n", " temp = copy.deepcopy(result)\n", " temp['result'] = request\n", "\n", " f.write(json.dumps(temp))\n", " \n", " if measurements['next']:\n", " measurements = requests.get(measurements['next']).json() \n", " else:\n", " next_result = False\n", "\n", " pbar.update(1)\n", "pbar.close()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# Extracts the response time from the measurement results.\n", "\n", "indir = 'data/atlas/soa/'\n", "\n", "data_perf = []\n", "\n", "for root, dirs, filenames in os.walk(indir):\n", " for f in filenames:\n", " tld, _ = f.split('.')\n", " \n", " datum = {'tld': tld, 'rt': [], 'timeouts': 0}\n", " \n", " with open(indir + f, 'r') as f:\n", " tld_results = json.loads(f.read())\n", " \n", " for probe in tld_results['result']:\n", " for result in probe['resultset']:\n", " if 'result' in result:\n", " datum['rt'].append(result['result']['rt']) \n", " elif 'error' in result and 'timeout' in result['error']:\n", " datum['timeouts'] += 1\n", " \n", " datum['rt'] = np.mean(datum['rt'])\n", " data_perf.append(datum)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "write_data('data_perf', data_perf)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Anycast" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "test_set = [\n", " {'target': '185.49.140.60', 'argument': 'nlnetlabs.nl', 'ns': 'ns.nlnetlabs.nl'},\n", " {'target': '192.16.197.229', 'argument': 'nlnet.nl', 'ns': 'mcvax.nlnet.nl'},\n", " {'target': '194.0.28.53', 'argument': 'nl', 'ns': 'ns5.dns.nl'},\n", " {'target': '204.61.216.4', 'argument': 'nlnetlabs.nl', 'ns': 'anyns.pch.net'}\n", "]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "definitions = []\n", "n = 3 # number of measurements\n", "\n", "for test in test_set:\n", " for i in range(1, n + 1): \n", " new_definition = copy.deepcopy(definition)\n", " new_definition['type'] = 'SOA'\n", " new_definition['query_argument'] = test['argument']\n", " new_definition['use_probe_resolver'] = False\n", " new_definition['target'] = test['target']\n", " new_definition['description'] = 'anycast {} {}'.format(i, test['ns'])\n", "\n", " definitions.append(new_definition)\n", "\n", "payload['definitions'] = definitions" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "anycast_probes = [\n", " # North America\n", " {'id': 22447, 'country-code': 'US', 'city': 'San Francisco'},\n", " {'id': 14233, 'country-code': 'US', 'city': 'Denver'},\n", " {'id': 25081, 'country-code': 'US', 'city': 'Washington'},\n", " # South America\n", " {'id': 31450, 'country-code': 'CR', 'city': 'San Jose'},\n", " {'id': 30185, 'country-code': 'BR', 'city': 'Sao Paulo'},\n", " {'id': 30123, 'country-code': 'CL', 'city': 'Santiago'},\n", " # Europe\n", " {'id': 32669, 'country-code': 'GR', 'city': 'Athens'},\n", " {'id': 31479, 'country-code': 'RU', 'city': 'Moscow'},\n", " {'id': 29762, 'country-code': 'ES', 'city': 'Madrid'},\n", " {'id': 26610, 'country-code': 'NL', 'city': 'Utrecht'},\n", " # Africa\n", " {'id': 22458, 'country-code': 'ZA', 'city': 'Cape Town'},\n", " {'id': 13258, 'country-code': 'AE', 'city': 'Dubai'},\n", " {'id': 28493, 'country-code': 'SN', 'city': 'Dakar'},\n", " # Asia\n", " {'id': 28819, 'country-code': 'JP', 'city': 'Tokyo'},\n", " {'id': 28964, 'country-code': 'KR', 'city': 'Seoul'},\n", " {'id': 6107, 'country-code': 'IN', 'city': 'Mumbai'},\n", " {'id': 25047, 'country-code': 'HK', 'city': 'Hong Kong'},\n", " {'id': 26378, 'country-code': 'KG', 'city': 'Bishkek'},\n", " # Oceania\n", " {'id': 25208, 'country-code': 'AU', 'city': 'Sydney'},\n", " {'id': 28226, 'country-code': 'NZ', 'city': 'Welington'}\n", "]\n", "\n", "probes = [\n", " {\n", " \"value\": str([probe['id'] for probe in anycast_probes])[1:-1],\n", " \"type\": \"probes\",\n", " \"requested\": len(anycast_probes)\n", " }\n", "]\n", "\n", "payload['probes'] = probes" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "url = '{}?key={}'.format(URL_DNS_MEASUREMENT_CREATE, atlas_api_key)\n", "request = requests.post(url, data = json.dumps(payload), headers = HEADERS)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "measurement_ids = request.json()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# Only 10 payloads can be sent per day.\n", "# Manually set this slice each day.\n", "for payload in payloads[:10]:\n", " \n", " request = requests.post(url, data = json.dumps(payload), headers = HEADERS)\n", " print(request.status_code)\n", " \n", " while request.status_code == 400:\n", " print(request.json())\n", " request = requests.post(url, data = json.dumps(payload), headers = HEADERS)\n", " time.sleep(300)\n", " print(request.status_code)\n", " \n", " measurement_ids += measurement_ids + request.json()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "next_result = True\n", "url = '{}?id__in={}&mine=true'.format(URL_DNS_MEASUREMENT_GET, str(measurement_ids)[1:-1])\n", "measurements = requests.get(url).json()\n", "\n", "while next_result:\n", " for result in measurements['results']:\n", " if len(result['description'].split()) == 3:\n", " measurement_type, i, ns = result['description'].split()\n", " request = requests.get(result['result']).json()\n", "\n", " if measurement_type == 'anycast':\n", " temp = copy.deepcopy(result)\n", " temp['result'] = request\n", " \n", " write_json('data/atlas/anycast/{}.{}.json'.format(ns, i), temp)\n", " \n", " if measurements['next']:\n", " measurements = requests.get(measurements['next']).json() \n", " else:\n", " next_result = False" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "indir = 'data/atlas/anycast/'\n", "probe_data = {k:v for k, v in [(probe_id, []) for probe_id in [probe['id'] for probe in anycast_probes]]}\n", "data_ac = [{'ns': test['ns'], 'probes': copy.deepcopy(probe_data)} for test in test_set]\n", "\n", "for root, dirs, filenames in os.walk(indir):\n", " for f in filenames:\n", " result = read_json(indir + f)\n", " _, _, ns = result['description'].split()\n", " index = find(data_ac, 'ns', ns)\n", " \n", " for probe in result['result']:\n", " try:\n", " data_ac[index]['probes'][probe['prb_id']].append(probe['result']['rt'])\n", " except:\n", " print('x')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "write_data('data_ac', data_ac)" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.1" } }, "nbformat": 4, "nbformat_minor": 2 }