first commit

1 files changed, 1369 insertions, 0 deletions

diff --git a/data_gathering.ipynb b/data_gathering.ipynb
new file mode 100644
index 0000000..9070f80
--- /dev/null
+++ b/data_gathering.ipynb
@@ -0,0 +1,1369 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {
+    "collapsed": true,
+    "scrolled": false
+   },
+   "outputs": [],
+   "source": [
+    "# Constantijn Bicker Caarten\n",
+    "# Last updated: 13-06-2017"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "from socket import error as socket_error\n",
+    "from urllib.request import urlopen\n",
+    "from dns.query import udp, tcp\n",
+    "from bs4 import BeautifulSoup\n",
+    "from dns.resolver import dns\n",
+    "from uuid import uuid4\n",
+    "from tqdm import tqdm\n",
+    "\n",
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "import matplotlib as plt\n",
+    "\n",
+    "import subprocess\n",
+    "import socket\n",
+    "import copy\n",
+    "import time\n",
+    "import json\n",
+    "import os\n",
+    "\n",
+    "%matplotlib inline"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 76,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "mkdir: cannot create directory ‘data’: File exists\n",
+      "mkdir: cannot create directory ‘data/whois’: File exists\n",
+      "mkdir: cannot create directory ‘data/dig’: File exists\n",
+      "mkdir: cannot create directory ‘data/backup’: File exists\n",
+      "mkdir: cannot create directory ‘data/lists’: File exists\n"
+     ]
+    }
+   ],
+   "source": [
+    "# First time run\n",
+    "!mkdir data\n",
+    "!mkdir data/whois\n",
+    "!mkdir data/cymru\n",
+    "!mkdir data/dig\n",
+    "!mkdir data/backup\n",
+    "!mkdir data/lists"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 199,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "ZSK = 256\n",
+    "KSK = 257\n",
+    "\n",
+    "pie = (6, 6)\n",
+    "\n",
+    "newline = '\\n'"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Functions"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 157,
+   "metadata": {
+    "collapsed": true,
+    "scrolled": false
+   },
+   "outputs": [],
+   "source": [
+    "def write_list(fn, data):\n",
+    "    '''Writes a list to a file with each value on a new line'''\n",
+    "    with open(fn, 'w') as f:\n",
+    "        for datum in data:\n",
+    "            f.write(datum + newline)\n",
+    "        \n",
+    "def append_list(fn, data):\n",
+    "    '''Appends a list to a file with each value on a new line'''\n",
+    "    with open(fn, 'a') as f:\n",
+    "        for datum in data:\n",
+    "            f.write(datum + newline)\n",
+    "    \n",
+    "def read_list(fn):\n",
+    "    '''Reads a file and '''\n",
+    "    with open(fn, 'r') as f:\n",
+    "        return [line.strip(newline) for line in f]\n",
+    "    \n",
+    "def write_json(fn, data):\n",
+    "    with open(fn, 'w') as f:\n",
+    "        f.write(json.dumps(data))    \n",
+    "            \n",
+    "def read_json(fn):\n",
+    "    '''Read a json file (fn) and returns it as a dictionary'''\n",
+    "    with open(fn, 'r') as f:\n",
+    "        return json.dumps(f.read())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 134,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def write_data(fn, data):\n",
+    "    \"\"\"Backs up the previous version of the data if it exists and writes the new data to a file.\"\"\"\n",
+    "    # Backs up the previous data if it exists.\n",
+    "    try:\n",
+    "        write_json(\"data/backup/{}.json \".format(fn) + time.ctime().replace(' ', '-'), \n",
+    "                   read_json(\"data/{}.json\".format(fn)))\n",
+    "    except:\n",
+    "        pass\n",
+    "\n",
+    "    write_json(\"data/{}.json\".format(fn), data)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "def find(lst, key, value):\n",
+    "    for i, dic in enumerate(lst):\n",
+    "        if dic[key] == value:\n",
+    "            return i\n",
+    "    return None\n",
+    "\n",
+    "def sort_dict_list(data, x):\n",
+    "    return sorted(data, key=lambda k: k[x]) "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "def ns_ips(fn):\n",
+    "    ns_ips = {}\n",
+    "\n",
+    "    with open(fn, 'r') as f:\n",
+    "        for line in f:\n",
+    "            if not line.startswith(';;'):\n",
+    "                ns, _, _, _, ip = line.split()\n",
+    "                ns = ns[:-1]\n",
+    "\n",
+    "                if ns in ns_ips and ip not in ns_ips[ns]:\n",
+    "                    ns_ips[ns].append(ip)\n",
+    "                else:\n",
+    "                    ns_ips[ns] = [ip]\n",
+    "                \n",
+    "    return ns_ips"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 119,
+   "metadata": {
+    "collapsed": true,
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "class CustomDNSException(Exception):\n",
+    "    pass\n",
+    "\n",
+    "def test_tcp_udp(data, timeout = 5):\n",
+    "    pbar = tqdm(total=len(data))\n",
+    "\n",
+    "    for datum in data:\n",
+    "        for p in (udp, tcp):\n",
+    "            # Create SOA query\n",
+    "            m = dns.message.make_query(datum['tld'], dns.rdatatype.SOA)\n",
+    "            try: \n",
+    "                a = p(m, datum['ip'], timeout = timeout)\n",
+    "                # We expect NOERROR RCODE (0) and an answer\n",
+    "                if a.rcode() == 0 and len(a.answer) > 0:\n",
+    "                    datum[p.__name__] = True\n",
+    "\n",
+    "                else:\n",
+    "                    raise CustomDNSException('failed')\n",
+    "            except (dns.exception.Timeout, socket_error, CustomDNSException):\n",
+    "                datum[p.__name__] = False\n",
+    "\n",
+    "        pbar.update(1)\n",
+    "    pbar.close()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Top-Level Domains"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "--2017-06-13 13:09:02--  https://data.iana.org/TLD/tlds-alpha-by-domain.txt\n",
+      "Loaded CA certificate '/etc/ssl/certs/ca-certificates.crt'\n",
+      "Resolving data.iana.org... 2606:2800:11f:bb5:f27:227f:1bbf:a0e, 72.21.81.189\n",
+      "Connecting to data.iana.org|2606:2800:11f:bb5:f27:227f:1bbf:a0e|:443... connected.\n",
+      "HTTP request sent, awaiting response... 200 OK\n",
+      "Length: 10295 (10K) [text/plain]\n",
+      "Saving to: ‘data/lists/tlds’\n",
+      "\n",
+      "data/lists/tlds     100%[===================>]  10.05K  --.-KB/s    in 0s      \n",
+      "\n",
+      "2017-06-13 13:09:08 (128 MB/s) - ‘data/lists/tlds’ saved [10295/10295]\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "!wget https://data.iana.org/TLD/tlds-alpha-by-domain.txt -O data/lists/tlds\n",
+    "!sed -i '1d' data/lists/tlds # remove header"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "tlds = read_list('data/lists/tlds')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "pbar = tqdm(total=len(tlds))\n",
+    "\n",
+    "for tld in tlds:\n",
+    "    !whois -h whois.iana.org:43 {tld} > data/whois/{tld}  \n",
+    "        \n",
+    "    pbar.update(1)\n",
+    "pbar.close()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 92,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Gathering empty WHOIS records.\n",
+      "Gathering missing WHOIS records.\n"
+     ]
+    }
+   ],
+   "source": [
+    "indir = 'data/whois/'\n",
+    "\n",
+    "for root, dirs, filenames in os.walk(indir):\n",
+    "    print(\"Gathering empty WHOIS records.\")\n",
+    "    for f in filenames:\n",
+    "        stat = os.stat(indir + f)\n",
+    "            \n",
+    "        if stat.st_size == 0:\n",
+    "            !whois -h whois.iana.org:43 {f} > data/whois/{f}\n",
+    "    \n",
+    "    print(\"Gathering missing WHOIS records.\")\n",
+    "    for tld in tlds:\n",
+    "        if tld not in filenames:\n",
+    "            !whois -h whois.iana.org:43 {tld} > data/whois/{tld}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 126,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data_tlds = [{'tld': tld, 'organisations': []} for tld in tlds]\n",
+    "\n",
+    "for root, dirs, filenames in os.walk(indir):\n",
+    "    for fn in filenames:\n",
+    "        with open(indir + fn, 'r') as f:\n",
+    "            for line in f:\n",
+    "                if line.startswith('created'):                    \n",
+    "                    index = find(data_tlds, 'tld', fn)\n",
+    "                    data_tlds[index]['creation_date'] = line.split()[-1]\n",
+    "                elif line.startswith('organisation'):\n",
+    "                    _, org = line.split('rganisation:')\n",
+    "                    index = find(data_tlds, 'tld', fn)\n",
+    "                    data_tlds[index]['organisations'].append(org.strip(newline))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 136,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "write_data('data_tlds', data_tlds)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 152,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "# Special TLDs are the same as record types or classes which do not work in bulk.\n",
+    "special_tlds = ['CH', 'IN', 'MD', 'MG', 'MR', 'MX']\n",
+    "write_list('data/lists/tlds', [tld for tld in tlds if tld not in special_tlds])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Name Servers"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Gathering name servers.\n",
+      "Done.\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Gathers the name servers of every TLD using dig.\n",
+    "print('Gathering name servers.')\n",
+    "!dig +noall +answer +noidn -t NS -f data/lists/tlds > data/dig/tld_nss\n",
+    "\n",
+    "for tld in special_tlds:\n",
+    "    !dig +noall +answer +noidn -t NS {tld} >> data/dig/tld_nss\n",
+    "\n",
+    "print('Done.')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 139,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "data_ns = []\n",
+    "\n",
+    "# Parses the answers of dig.\n",
+    "with open('data/dig/tld_nss', 'r') as f:\n",
+    "    for line in f:\n",
+    "        if not line.startswith('.'):\n",
+    "            tld, _, _, _, ns = line.split()\n",
+    "            data_ns.append({'tld': tld[:-1], 'ns': ns.lower()[:-1]})\n",
+    "            \n",
+    "write_data('data_ns', data_ns)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# IP Addresses"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "write_list('data/lists/nss', set([datum['ns'] for datum in data]))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Gathering IPv4 addresses.\n",
+      "Done.\n",
+      "Gathering IPv6 addresses.\n",
+      "Done.\n"
+     ]
+    }
+   ],
+   "source": [
+    "print('Gathering IPv4 addresses.')\n",
+    "!dig +noall +answer +noidn A -f data/lists/nss > data/dig/ns_ipv4s\n",
+    "print('Done.')\n",
+    "\n",
+    "print('Gathering IPv6 addresses.')\n",
+    "!dig +noall +answer +noidn AAAA -f data/lists/nss > data/dig/ns_ipv6s\n",
+    "print('Done.')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 143,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "ns_ipv4s = ns_ips('data/dig/ns_ipv4s')\n",
+    "ns_ipv6s = ns_ips('data/dig/ns_ipv6s')\n",
+    "\n",
+    "data_ips = []\n",
+    "\n",
+    "for datum in data:\n",
+    "    if datum['ns'] in ns_ipv4s:\n",
+    "        for ip in ns_ipv4s[datum['ns']]:\n",
+    "            new_datum = copy.deepcopy(datum)\n",
+    "            new_datum['ip'] = ip\n",
+    "            data_ips.append(new_datum)\n",
+    "    \n",
+    "    if datum['ns'] in ns_ipv6s:\n",
+    "        for ip in ns_ipv6s[datum['ns']]:\n",
+    "            new_datum = copy.deepcopy(datum)\n",
+    "            new_datum['ip'] = ip\n",
+    "            data_ips.append(new_datum)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 144,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "write_data('data_ips', data_ips)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Autonomous System Number"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 73,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "write_list('data/lists/ips', ['begin'])\n",
+    "append_list('data/lists/ips', set([datum['ip'] for datum in data_ips]))\n",
+    "append_list('data/lists/ips', ['end'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 77,
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [],
+   "source": [
+    "!netcat whois.cymru.com 43 < data/lists/ips | sort -n > data/cymru/ip_asns"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 145,
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [],
+   "source": [
+    "ip_asns = {}\n",
+    "\n",
+    "with open('data/cymru/ip_asns', 'r') as f:\n",
+    "    for line in f:\n",
+    "        if not line.startswith('Bulk') and not line.startswith('NA'):\n",
+    "            \n",
+    "            asn, ip, org = [value.strip() for value in line.split('|')]\n",
+    "            \n",
+    "            if ip in ip_asns and asn not in ip_asns[ip]:\n",
+    "                ip_asns[ip].append(asn)\n",
+    "            else:\n",
+    "                ip_asns[ip] = [asn]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 146,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "for datum in data_ips:\n",
+    "    if datum['ip'] in ip_asns:\n",
+    "        datum['asn'] = ip_asns[datum['ip']]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 147,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data_asns = []\n",
+    "\n",
+    "for datum in data_ips:\n",
+    "    if datum['ip'] in ip_asns:\n",
+    "        for asn in ip_asns[datum['ip']]:\n",
+    "            new_datum = copy.deepcopy(datum)\n",
+    "            new_datum['asn'] = asn\n",
+    "            data_asns.append(new_datum)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 148,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "write_data('data_asns', data_asns)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Reachability"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 149,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "test_tcp_udp(data_ips)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "data_no_tcp_and_udp = [datum for datum in data_ips if not datum['tcp'] and not datum['udp']]\n",
+    "\n",
+    "test_tcp_udp(data_no_tcp_and_udp, timeout = 10)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "len([datum for datum in data_ips if not datum['tcp'] and not datum['udp']]), len(data_ips)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 142,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "write_data('data_prot', data_ips)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "len(set([datum['tld'] for datum in data_ips if not datum['tcp'] and not datum['udp']]))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "with open('data/backup/data_tcp_udp', 'w') as f:\n",
+    "    f.write(json.dumps(data_ips))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "# with open('data/backup/data_tcp_udp', 'r') as f:\n",
+    "#     data_ips = json.loads(f.read())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true,
+    "scrolled": false
+   },
+   "outputs": [],
+   "source": [
+    "df = pd.DataFrame(data_ips)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "xtlds = []\n",
+    "xnss = []\n",
+    "xips = []\n",
+    "xasns = []\n",
+    "xtcp = []\n",
+    "xudp = []\n",
+    "\n",
+    "for datum in data_ips:\n",
+    "    xtlds.append(datum['tld'])\n",
+    "    xnss.append(datum['ns'])\n",
+    "    xips.append(datum['ip'])\n",
+    "    xtcp.append(datum['tcp'])\n",
+    "    xudp.append(datum['udp'])\n",
+    "    if 'asn' in datum:\n",
+    "        xasns.append(datum['asn'])\n",
+    "    else:\n",
+    "        xasns.append([])\n",
+    "        \n",
+    "print(len(data_ips), len(xtlds), len(xnss), len(xips))\n",
+    "    \n",
+    "ix = pd.MultiIndex.from_arrays([xtlds, xnss, xips], names=['tld', 'ns', 'ip'])\n",
+    "dg = pd.DataFrame({'asn': xasns, 'tcp': xtcp, 'udp': xudp}, index = ix)\n",
+    "# dg.head(10)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "# dage = {}\n",
+    "\n",
+    "# for datum in data_age:\n",
+    "#     dage[datum['tld']] = datum['age']\n",
+    "    \n",
+    "# for datum in data_ips:\n",
+    "#     if datum['tld'][:-1].upper() in dage:\n",
+    "#         if dage[datum['tld'][:-1].upper()] == 'new':\n",
+    "#             datum['age'] = 'new'\n",
+    "#         else:\n",
+    "#             datum['age'] = 'old'\n",
+    "#     else:\n",
+    "#         datum['age'] = None\n",
+    "\n",
+    "# xtlds = []\n",
+    "# xnss = []\n",
+    "# xips = []\n",
+    "# xasns = []\n",
+    "# xtcp = []\n",
+    "# xudp = []\n",
+    "\n",
+    "# for datum in data_ips:\n",
+    "#     if datum['age'] == 'old':\n",
+    "    \n",
+    "#         xtlds.append(datum['tld'])\n",
+    "#         xnss.append(datum['ns'])\n",
+    "#         xips.append(datum['ip'])\n",
+    "#         xtcp.append(datum['tcp'])\n",
+    "#         xudp.append(datum['udp'])\n",
+    "#         if 'asn' in datum:\n",
+    "#             xasns.append(datum['asn'])\n",
+    "#         else:\n",
+    "#             xasns.append([])\n",
+    "        \n",
+    "# print(len(data_ips), len(xtlds), len(xnss), len(xips))\n",
+    "    \n",
+    "# ix = pd.MultiIndex.from_arrays([xtlds, xnss, xips], names=['tld', 'ns', 'ip'])\n",
+    "# dg = pd.DataFrame({'asn': xasns, 'tcp': xtcp, 'udp': xudp}, index = ix)\n",
+    "# # dg.head(10)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "# data_ips[0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "ns_tcp_udp = [{'ns': datum['ns'], 'tcp': False, 'udp': False} for datum in data_ips if datum['tld'] in dtype and dtype[datum['tld']] == 'country-code']\n",
+    "\n",
+    "for ns in ns_tcp_udp:\n",
+    "    for datum in data_ips:\n",
+    "        if datum['ns'] == ns['ns']:\n",
+    "            if datum['tcp']:\n",
+    "                ns['tcp'] = True\n",
+    "                \n",
+    "            if datum['udp']:\n",
+    "                ns['udp'] = True       "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "ns_tcp_udp = [{'ns': datum['ns'], 'tcp': False, 'udp': False} for datum in data_ips]\n",
+    "\n",
+    "for ns in ns_tcp_udp:\n",
+    "    for datum in data_ips:\n",
+    "        if datum['ns'] == ns['ns']:\n",
+    "            if datum['tcp']:\n",
+    "                ns['tcp'] = True\n",
+    "                \n",
+    "            if datum['udp']:\n",
+    "                ns['udp'] = True       "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "ip = ':'\n",
+    "\n",
+    "ns_tcp_udp = [{'ns': datum['ns'], 'tcp': False, 'udp': False} for datum in data_ips if ip in datum['ip']]\n",
+    "\n",
+    "for ns in ns_tcp_udp:\n",
+    "    for datum in data_ips:\n",
+    "        if datum['ns'] == ns['ns'] and ip in datum['ip']:\n",
+    "            if datum['tcp']:\n",
+    "                ns['tcp'] = True\n",
+    "                \n",
+    "            if datum['udp']:\n",
+    "                ns['udp'] = True"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "dg = pd.DataFrame(ns_tcp_udp)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "ax = dg.tcp.value_counts().plot.pie(autopct=lambda p : '{:.2f}% ({:.0f})'.format(p, p * dg.count().udp / 100), figsize = pie_size)\n",
+    "# ax = dg.tcp.plot.bar()\n",
+    "ax.set_ylabel('')\n",
+    "fig = ax.get_figure()\n",
+    "fig.savefig(\"imgs/tcp.pdf\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "ax = dg.udp.value_counts().plot.pie(autopct=lambda p : '{:.2f}% ({:.0f})'.format(p, p * dg.count().udp / 100), \n",
+    "                                    figsize = pie_size)\n",
+    "ax.set_ylabel('')\n",
+    "# ax.set_title('UDP')\n",
+    "fig = ax.get_figure()\n",
+    "fig.savefig(\"imgs/udp.pdf\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "df_ftcp = dg.loc[dg.tcp == False]\n",
+    "df_fudp = dg.loc[dg.udp == False]\n",
+    "df_ttcp = dg.loc[dg.tcp == True]\n",
+    "\n",
+    "ff = df_ftcp.loc[df_ftcp.udp == False].count().tcp\n",
+    "ft = df_ftcp.loc[df_ftcp.udp == True].count().tcp\n",
+    "tf = df_fudp.loc[df_fudp.tcp == True].count().tcp\n",
+    "tt = df_ttcp.loc[df_ttcp.udp == True].count().tcp\n",
+    "\n",
+    "print(ff, ft, tf, tt)\n",
+    "\n",
+    "ut_data = [{'name': 'none', 'count': ff},       \n",
+    "           {'name': 'tcp', 'count': tf},     \n",
+    "           {'name': 'udp', 'count': ft},           \n",
+    "           {'name': 'tcp + udp', 'count': tt}\n",
+    "          ]\n",
+    "\n",
+    "total = ff + ft + tf + tt\n",
+    "\n",
+    "dfgh = pd.DataFrame(ut_data)\n",
+    "dfgh.index = dfgh['name']\n",
+    "del dfgh['name']\n",
+    "# ax = dfgh.plot.pie('count',\n",
+    "# #                    autopct='s(%.2f)',\n",
+    "#                    autopct=lambda p : '{:.2f}% ({:.0f})'.format(p, p * total / 100),\n",
+    "# #                    radius = 2.5,\n",
+    "# #                    pctdistance=1.2,\n",
+    "# #                    labeldistance=1.2,\n",
+    "# #                    explode = True,\n",
+    "#                    figsize = pie_size, \n",
+    "#                    legend=False, \n",
+    "#                    labels=['','','',''])\n",
+    "\n",
+    "ax = dfgh.plot.barh()\n",
+    "# ax.set_xlim([0,10000])\n",
+    "\n",
+    "ax.legend(loc='best', labels=dfgh.index)\n",
+    "ax.set_xlabel('Number of name servers')\n",
+    "ax.set_ylabel('Protocol(s) supported')\n",
+    "# ax.set_title('name server udp/tcp support')\n",
+    "ax.legend_.remove()\n",
+    "ax.set_xscale('log')\n",
+    "fig = ax.get_figure()\n",
+    "fig.tight_layout()\n",
+    "fig.savefig(\"imgs/tcp_udp_generic.pdf\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "dfgh.index"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "df.loc[df.tld == 'actor.']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "# def write_to_file(fn, indir, content):\n",
+    "#     with open(indir + fn, 'w') as f:\n",
+    "#         f.write(content)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "# write_to_file('tcp_udp_not_working', 'data/temp/', df_ftcp.loc[df_ftcp.udp == False].to_csv())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "# df_ftcp.loc[df_ftcp.udp == False]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "# df.loc[df.ip.str.contains(':')].count().tcp #ipv6\n",
+    "# df[~df[\"ip\"].str.contains(\":\")].count().tcp #ipv4"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "# ax = df[~df[\"ip\"].str.contains(\":\")].tcp.value_counts().plot.pie(autopct='%.2f', figsize = pie_size)\n",
+    "# ax.set_ylabel('')\n",
+    "# fig = ax.get_figure()\n",
+    "# fig.savefig(\"imgs/tcp_ipv4.pdf\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "# ax = df[~df[\"ip\"].str.contains(\":\")].udp.value_counts().plot.pie(autopct='%.2f', figsize = pie_size)\n",
+    "# ax.set_ylabel('')\n",
+    "# fig = ax.get_figure()\n",
+    "# fig.savefig(\"imgs/udp_ipv4.pdf\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "# ax = df.loc[df.ip.str.contains(':')].tcp.value_counts().plot.pie(autopct='%.2f', figsize = pie_size)\n",
+    "# ax.set_ylabel('')\n",
+    "# fig = ax.get_figure()\n",
+    "# fig.savefig(\"imgs/tcp_ipv6.pdf\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "# ax = df.loc[df.ip.str.contains(':')].udp.value_counts().plot.pie(autopct='%.2f', figsize = pie_size)\n",
+    "# ax.set_ylabel('')\n",
+    "# fig = ax.get_figure()\n",
+    "# fig.savefig(\"imgs/udp_ipv6.pdf\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "df.head()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Credibility"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 201,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "!dig +noall +answer +noidn -t DNSKEY -f data/lists/tlds > data/dig/tld_dnskeys\n",
+    "!dig +noall +answer +noidn -t DS -f data/lists/tlds > data/dig/tld_dss\n",
+    "\n",
+    "for tld in special_tlds:\n",
+    "    !dig +noall +answer +noidn -t DNSKEY {tld} >> data/dig/tld_dnskeys\n",
+    "    !dig +noall +answer +noidn -t DS {tld} >> data/dig/tld_dss"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 235,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data_cred = [{'tld': tld, 'ds': False, 'dnskey': False, 'algorithm': None} for tld in tlds]\n",
+    "\n",
+    "temp = []\n",
+    "\n",
+    "for answer in read_list('data/dig/tld_dnskeys'):\n",
+    "    v = answer.split()\n",
+    "    tld = v[0][:-1].upper()\n",
+    "    index = find(data_cred, 'tld', tld)\n",
+    "    \n",
+    "    try:\n",
+    "        data_cred[index]['dnskey'] = True\n",
+    "        data_cred[index]['algorithm'] = v[6]\n",
+    "    except:\n",
+    "        print(tld)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 236,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for answer in read_list('data/dig/tld_dss'):\n",
+    "    v = answer.split()\n",
+    "    tld = v[0][:-1].upper()\n",
+    "    \n",
+    "    index = find(data_cred, 'tld', tld)\n",
+    "    \n",
+    "    try:\n",
+    "        data_cred[index]['ds'] = True\n",
+    "    except:\n",
+    "        print(tld)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 237,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "write_data('data_cred', data_cred)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Organisations per TLD"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "# tld_orgs"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "df_orgs = pd.DataFrame(tld_orgs)\n",
+    "df_orgs.index = df_orgs['tld']\n",
+    "del df_orgs['tld']\n",
+    "df_orgs.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "df_orgs.reset_index(inplace=True)\n",
+    "rows = []\n",
+    "_ = df_orgs.apply(lambda row: [rows.append([row['tld'], nn]) \n",
+    "                         for nn in row.organisations], axis=1)\n",
+    "df_orgs_new = pd.DataFrame(rows, columns=df_orgs.columns).set_index(['tld'])\n",
+    "\n",
+    "df_orgs_new.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "df_orgs_new.organisations.value_counts(ascending=False).head(80).plot.barh(figsize = (10,20))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "bins = df_orgs_new.organisations.value_counts().nunique() - 1\n",
+    "ax = df_orgs_new.organisations.value_counts().hist(bins = bins)\n",
+    "ax.set_yscale('log')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "df_orgs_new.organisations.value_counts().value_counts().plot.pie()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "# df_orgs_new.organisations.value_counts()\n",
+    "\n",
+    "# df2[df2['rr_quality'] > 0]].groupby([df2.index.hour,'sleep_summary_id')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "ax = df_tld_orgs.organisation.value_counts().head(20).plot.barh(figsize = bar_size, fontsize=12)\n",
+    "ax.set_xlabel('Number of TLDs', fontsize = 16)\n",
+    "ax.set_ylabel('Organisation',fontsize = 16)\n",
+    "fig = ax.get_figure()\n",
+    "fig.savefig(\"imgs/orgs.png\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "ax = df_tld_orgs.type.value_counts().plot.pie(figsize = pie_size, legend=True)\n",
+    "ax.set_ylabel('')\n",
+    "fig = ax.get_figure()\n",
+    "fig.savefig(\"imgs/types.png\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Growth"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "data_age = []\n",
+    "dage = {}\n",
+    "\n",
+    "for datum in tld_creation:\n",
+    "    y, m, d = datum['date_created'].split('-')\n",
+    "    if y in ['2014', '2015', '2016', '2017'] or y == '2013' and int(m) >= 10:\n",
+    "        data_age.append({'tld': datum['tld'], 'age': 'new'})\n",
+    "        dage[datum['tld']] = 'new'\n",
+    "    else:\n",
+    "        data_age.append({'tld': datum['tld'], 'age': 'old'})\n",
+    "        dage[datum['tld']] = 'old'"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.1"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}