{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Constantijn Bicker Caarten\n",
    "# Last updated: 13-06-2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from socket import error as socket_error\n",
    "from urllib.request import urlopen\n",
    "from dns.query import udp, tcp\n",
    "from bs4 import BeautifulSoup\n",
    "from dns.resolver import dns\n",
    "from uuid import uuid4\n",
    "from tqdm import tqdm\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib as plt\n",
    "\n",
    "import subprocess\n",
    "import socket\n",
    "import copy\n",
    "import time\n",
    "import json\n",
    "import os\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mkdir: cannot create directory ‘data’: File exists\n",
      "mkdir: cannot create directory ‘data/whois’: File exists\n",
      "mkdir: cannot create directory ‘data/dig’: File exists\n",
      "mkdir: cannot create directory ‘data/backup’: File exists\n",
      "mkdir: cannot create directory ‘data/lists’: File exists\n"
     ]
    }
   ],
   "source": [
    "# First time run\n",
    "!mkdir data\n",
    "!mkdir data/whois\n",
    "!mkdir data/cymru\n",
    "!mkdir data/dig\n",
    "!mkdir data/backup\n",
    "!mkdir data/lists"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ZSK = 256\n",
    "KSK = 257\n",
    "\n",
    "pie = (6, 6)\n",
    "\n",
    "newline = '\\n'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "def write_list(fn, data):\n",
    "    '''Writes a list to a file with each value on a new line'''\n",
    "    with open(fn, 'w') as f:\n",
    "        for datum in data:\n",
    "            f.write(datum + newline)\n",
    "        \n",
    "def append_list(fn, data):\n",
    "    '''Appends a list to a file with each value on a new line'''\n",
    "    with open(fn, 'a') as f:\n",
    "        for datum in data:\n",
    "            f.write(datum + newline)\n",
    "    \n",
    "def read_list(fn):\n",
    "    '''Reads a file and '''\n",
    "    with open(fn, 'r') as f:\n",
    "        return [line.strip(newline) for line in f]\n",
    "    \n",
    "def write_json(fn, data):\n",
    "    with open(fn, 'w') as f:\n",
    "        f.write(json.dumps(data))    \n",
    "            \n",
    "def read_json(fn):\n",
    "    '''Read a json file (fn) and returns it as a dictionary'''\n",
    "    with open(fn, 'r') as f:\n",
    "        return json.dumps(f.read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "def write_data(fn, data):\n",
    "    \"\"\"Backs up the previous version of the data if it exists and writes the new data to a file.\"\"\"\n",
    "    # Backs up the previous data if it exists.\n",
    "    try:\n",
    "        write_json(\"data/backup/{}.json \".format(fn) + time.ctime().replace(' ', '-'), \n",
    "                   read_json(\"data/{}.json\".format(fn)))\n",
    "    except:\n",
    "        pass\n",
    "\n",
    "    write_json(\"data/{}.json\".format(fn), data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def find(lst, key, value):\n",
    "    for i, dic in enumerate(lst):\n",
    "        if dic[key] == value:\n",
    "            return i\n",
    "    return None\n",
    "\n",
    "def sort_dict_list(data, x):\n",
    "    return sorted(data, key=lambda k: k[x]) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def ns_ips(fn):\n",
    "    ns_ips = {}\n",
    "\n",
    "    with open(fn, 'r') as f:\n",
    "        for line in f:\n",
    "            if not line.startswith(';;'):\n",
    "                ns, _, _, _, ip = line.split()\n",
    "                ns = ns[:-1]\n",
    "\n",
    "                if ns in ns_ips and ip not in ns_ips[ns]:\n",
    "                    ns_ips[ns].append(ip)\n",
    "                else:\n",
    "                    ns_ips[ns] = [ip]\n",
    "                \n",
    "    return ns_ips"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "class CustomDNSException(Exception):\n",
    "    pass\n",
    "\n",
    "def test_tcp_udp(data, timeout = 5):\n",
    "    pbar = tqdm(total=len(data))\n",
    "\n",
    "    for datum in data:\n",
    "        for p in (udp, tcp):\n",
    "            # Create SOA query\n",
    "            m = dns.message.make_query(datum['tld'], dns.rdatatype.SOA)\n",
    "            try: \n",
    "                a = p(m, datum['ip'], timeout = timeout)\n",
    "                # We expect NOERROR RCODE (0) and an answer\n",
    "                if a.rcode() == 0 and len(a.answer) > 0:\n",
    "                    datum[p.__name__] = True\n",
    "\n",
    "                else:\n",
    "                    raise CustomDNSException('failed')\n",
    "            except (dns.exception.Timeout, socket_error, CustomDNSException):\n",
    "                datum[p.__name__] = False\n",
    "\n",
    "        pbar.update(1)\n",
    "    pbar.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Top-Level Domains"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2017-06-13 13:09:02--  https://data.iana.org/TLD/tlds-alpha-by-domain.txt\n",
      "Loaded CA certificate '/etc/ssl/certs/ca-certificates.crt'\n",
      "Resolving data.iana.org... 2606:2800:11f:bb5:f27:227f:1bbf:a0e, 72.21.81.189\n",
      "Connecting to data.iana.org|2606:2800:11f:bb5:f27:227f:1bbf:a0e|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 10295 (10K) [text/plain]\n",
      "Saving to: ‘data/lists/tlds’\n",
      "\n",
      "data/lists/tlds     100%[===================>]  10.05K  --.-KB/s    in 0s      \n",
      "\n",
      "2017-06-13 13:09:08 (128 MB/s) - ‘data/lists/tlds’ saved [10295/10295]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!wget https://data.iana.org/TLD/tlds-alpha-by-domain.txt -O data/lists/tlds\n",
    "!sed -i '1d' data/lists/tlds # remove header"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tlds = read_list('data/lists/tlds')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pbar = tqdm(total=len(tlds))\n",
    "\n",
    "for tld in tlds:\n",
    "    !whois -h whois.iana.org:43 {tld} > data/whois/{tld}  \n",
    "        \n",
    "    pbar.update(1)\n",
    "pbar.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gathering empty WHOIS records.\n",
      "Gathering missing WHOIS records.\n"
     ]
    }
   ],
   "source": [
    "indir = 'data/whois/'\n",
    "\n",
    "for root, dirs, filenames in os.walk(indir):\n",
    "    print(\"Gathering empty WHOIS records.\")\n",
    "    for f in filenames:\n",
    "        stat = os.stat(indir + f)\n",
    "            \n",
    "        if stat.st_size == 0:\n",
    "            !whois -h whois.iana.org:43 {f} > data/whois/{f}\n",
    "    \n",
    "    print(\"Gathering missing WHOIS records.\")\n",
    "    for tld in tlds:\n",
    "        if tld not in filenames:\n",
    "            !whois -h whois.iana.org:43 {tld} > data/whois/{tld}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_tlds = [{'tld': tld, 'organisations': []} for tld in tlds]\n",
    "\n",
    "for root, dirs, filenames in os.walk(indir):\n",
    "    for fn in filenames:\n",
    "        with open(indir + fn, 'r') as f:\n",
    "            for line in f:\n",
    "                if line.startswith('created'):                    \n",
    "                    index = find(data_tlds, 'tld', fn)\n",
    "                    data_tlds[index]['creation_date'] = line.split()[-1]\n",
    "                elif line.startswith('organisation'):\n",
    "                    _, org = line.split('rganisation:')\n",
    "                    index = find(data_tlds, 'tld', fn)\n",
    "                    data_tlds[index]['organisations'].append(org.strip(newline))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [],
   "source": [
    "write_data('data_tlds', data_tlds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Special TLDs are the same as record types or classes which do not work in bulk.\n",
    "special_tlds = ['CH', 'IN', 'MD', 'MG', 'MR', 'MX']\n",
    "write_list('data/lists/tlds', [tld for tld in tlds if tld not in special_tlds])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Name Servers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gathering name servers.\n",
      "Done.\n"
     ]
    }
   ],
   "source": [
    "# Gathers the name servers of every TLD using dig.\n",
    "print('Gathering name servers.')\n",
    "!dig +noall +answer +noidn -t NS -f data/lists/tlds > data/dig/tld_nss\n",
    "\n",
    "for tld in special_tlds:\n",
    "    !dig +noall +answer +noidn -t NS {tld} >> data/dig/tld_nss\n",
    "\n",
    "print('Done.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_ns = []\n",
    "\n",
    "# Parses the answers of dig.\n",
    "with open('data/dig/tld_nss', 'r') as f:\n",
    "    for line in f:\n",
    "        if not line.startswith('.'):\n",
    "            tld, _, _, _, ns = line.split()\n",
    "            data_ns.append({'tld': tld[:-1], 'ns': ns.lower()[:-1]})\n",
    "            \n",
    "write_data('data_ns', data_ns)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# IP Addresses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "write_list('data/lists/nss', set([datum['ns'] for datum in data]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gathering IPv4 addresses.\n",
      "Done.\n",
      "Gathering IPv6 addresses.\n",
      "Done.\n"
     ]
    }
   ],
   "source": [
    "print('Gathering IPv4 addresses.')\n",
    "!dig +noall +answer +noidn A -f data/lists/nss > data/dig/ns_ipv4s\n",
    "print('Done.')\n",
    "\n",
    "print('Gathering IPv6 addresses.')\n",
    "!dig +noall +answer +noidn AAAA -f data/lists/nss > data/dig/ns_ipv6s\n",
    "print('Done.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ns_ipv4s = ns_ips('data/dig/ns_ipv4s')\n",
    "ns_ipv6s = ns_ips('data/dig/ns_ipv6s')\n",
    "\n",
    "data_ips = []\n",
    "\n",
    "for datum in data:\n",
    "    if datum['ns'] in ns_ipv4s:\n",
    "        for ip in ns_ipv4s[datum['ns']]:\n",
    "            new_datum = copy.deepcopy(datum)\n",
    "            new_datum['ip'] = ip\n",
    "            data_ips.append(new_datum)\n",
    "    \n",
    "    if datum['ns'] in ns_ipv6s:\n",
    "        for ip in ns_ipv6s[datum['ns']]:\n",
    "            new_datum = copy.deepcopy(datum)\n",
    "            new_datum['ip'] = ip\n",
    "            data_ips.append(new_datum)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "write_data('data_ips', data_ips)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Autonomous System Number"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "write_list('data/lists/ips', ['begin'])\n",
    "append_list('data/lists/ips', set([datum['ip'] for datum in data_ips]))\n",
    "append_list('data/lists/ips', ['end'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "!netcat whois.cymru.com 43 < data/lists/ips | sort -n > data/cymru/ip_asns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "ip_asns = {}\n",
    "\n",
    "with open('data/cymru/ip_asns', 'r') as f:\n",
    "    for line in f:\n",
    "        if not line.startswith('Bulk') and not line.startswith('NA'):\n",
    "            \n",
    "            asn, ip, org = [value.strip() for value in line.split('|')]\n",
    "            \n",
    "            if ip in ip_asns and asn not in ip_asns[ip]:\n",
    "                ip_asns[ip].append(asn)\n",
    "            else:\n",
    "                ip_asns[ip] = [asn]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for datum in data_ips:\n",
    "    if datum['ip'] in ip_asns:\n",
    "        datum['asn'] = ip_asns[datum['ip']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_asns = []\n",
    "\n",
    "for datum in data_ips:\n",
    "    if datum['ip'] in ip_asns:\n",
    "        for asn in ip_asns[datum['ip']]:\n",
    "            new_datum = copy.deepcopy(datum)\n",
    "            new_datum['asn'] = asn\n",
    "            data_asns.append(new_datum)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "write_data('data_asns', data_asns)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Reachability"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "test_tcp_udp(data_ips)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_no_tcp_and_udp = [datum for datum in data_ips if not datum['tcp'] and not datum['udp']]\n",
    "\n",
    "test_tcp_udp(data_no_tcp_and_udp, timeout = 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "len([datum for datum in data_ips if not datum['tcp'] and not datum['udp']]), len(data_ips)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "write_data('data_prot', data_ips)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "len(set([datum['tld'] for datum in data_ips if not datum['tcp'] and not datum['udp']]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('data/backup/data_tcp_udp', 'w') as f:\n",
    "    f.write(json.dumps(data_ips))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# with open('data/backup/data_tcp_udp', 'r') as f:\n",
    "#     data_ips = json.loads(f.read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "df = pd.DataFrame(data_ips)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "xtlds = []\n",
    "xnss = []\n",
    "xips = []\n",
    "xasns = []\n",
    "xtcp = []\n",
    "xudp = []\n",
    "\n",
    "for datum in data_ips:\n",
    "    xtlds.append(datum['tld'])\n",
    "    xnss.append(datum['ns'])\n",
    "    xips.append(datum['ip'])\n",
    "    xtcp.append(datum['tcp'])\n",
    "    xudp.append(datum['udp'])\n",
    "    if 'asn' in datum:\n",
    "        xasns.append(datum['asn'])\n",
    "    else:\n",
    "        xasns.append([])\n",
    "        \n",
    "print(len(data_ips), len(xtlds), len(xnss), len(xips))\n",
    "    \n",
    "ix = pd.MultiIndex.from_arrays([xtlds, xnss, xips], names=['tld', 'ns', 'ip'])\n",
    "dg = pd.DataFrame({'asn': xasns, 'tcp': xtcp, 'udp': xudp}, index = ix)\n",
    "# dg.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# dage = {}\n",
    "\n",
    "# for datum in data_age:\n",
    "#     dage[datum['tld']] = datum['age']\n",
    "    \n",
    "# for datum in data_ips:\n",
    "#     if datum['tld'][:-1].upper() in dage:\n",
    "#         if dage[datum['tld'][:-1].upper()] == 'new':\n",
    "#             datum['age'] = 'new'\n",
    "#         else:\n",
    "#             datum['age'] = 'old'\n",
    "#     else:\n",
    "#         datum['age'] = None\n",
    "\n",
    "# xtlds = []\n",
    "# xnss = []\n",
    "# xips = []\n",
    "# xasns = []\n",
    "# xtcp = []\n",
    "# xudp = []\n",
    "\n",
    "# for datum in data_ips:\n",
    "#     if datum['age'] == 'old':\n",
    "    \n",
    "#         xtlds.append(datum['tld'])\n",
    "#         xnss.append(datum['ns'])\n",
    "#         xips.append(datum['ip'])\n",
    "#         xtcp.append(datum['tcp'])\n",
    "#         xudp.append(datum['udp'])\n",
    "#         if 'asn' in datum:\n",
    "#             xasns.append(datum['asn'])\n",
    "#         else:\n",
    "#             xasns.append([])\n",
    "        \n",
    "# print(len(data_ips), len(xtlds), len(xnss), len(xips))\n",
    "    \n",
    "# ix = pd.MultiIndex.from_arrays([xtlds, xnss, xips], names=['tld', 'ns', 'ip'])\n",
    "# dg = pd.DataFrame({'asn': xasns, 'tcp': xtcp, 'udp': xudp}, index = ix)\n",
    "# # dg.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# data_ips[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ns_tcp_udp = [{'ns': datum['ns'], 'tcp': False, 'udp': False} for datum in data_ips if datum['tld'] in dtype and dtype[datum['tld']] == 'country-code']\n",
    "\n",
    "for ns in ns_tcp_udp:\n",
    "    for datum in data_ips:\n",
    "        if datum['ns'] == ns['ns']:\n",
    "            if datum['tcp']:\n",
    "                ns['tcp'] = True\n",
    "                \n",
    "            if datum['udp']:\n",
    "                ns['udp'] = True       "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ns_tcp_udp = [{'ns': datum['ns'], 'tcp': False, 'udp': False} for datum in data_ips]\n",
    "\n",
    "for ns in ns_tcp_udp:\n",
    "    for datum in data_ips:\n",
    "        if datum['ns'] == ns['ns']:\n",
    "            if datum['tcp']:\n",
    "                ns['tcp'] = True\n",
    "                \n",
    "            if datum['udp']:\n",
    "                ns['udp'] = True       "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ip = ':'\n",
    "\n",
    "ns_tcp_udp = [{'ns': datum['ns'], 'tcp': False, 'udp': False} for datum in data_ips if ip in datum['ip']]\n",
    "\n",
    "for ns in ns_tcp_udp:\n",
    "    for datum in data_ips:\n",
    "        if datum['ns'] == ns['ns'] and ip in datum['ip']:\n",
    "            if datum['tcp']:\n",
    "                ns['tcp'] = True\n",
    "                \n",
    "            if datum['udp']:\n",
    "                ns['udp'] = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dg = pd.DataFrame(ns_tcp_udp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ax = dg.tcp.value_counts().plot.pie(autopct=lambda p : '{:.2f}% ({:.0f})'.format(p, p * dg.count().udp / 100), figsize = pie_size)\n",
    "# ax = dg.tcp.plot.bar()\n",
    "ax.set_ylabel('')\n",
    "fig = ax.get_figure()\n",
    "fig.savefig(\"imgs/tcp.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ax = dg.udp.value_counts().plot.pie(autopct=lambda p : '{:.2f}% ({:.0f})'.format(p, p * dg.count().udp / 100), \n",
    "                                    figsize = pie_size)\n",
    "ax.set_ylabel('')\n",
    "# ax.set_title('UDP')\n",
    "fig = ax.get_figure()\n",
    "fig.savefig(\"imgs/udp.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_ftcp = dg.loc[dg.tcp == False]\n",
    "df_fudp = dg.loc[dg.udp == False]\n",
    "df_ttcp = dg.loc[dg.tcp == True]\n",
    "\n",
    "ff = df_ftcp.loc[df_ftcp.udp == False].count().tcp\n",
    "ft = df_ftcp.loc[df_ftcp.udp == True].count().tcp\n",
    "tf = df_fudp.loc[df_fudp.tcp == True].count().tcp\n",
    "tt = df_ttcp.loc[df_ttcp.udp == True].count().tcp\n",
    "\n",
    "print(ff, ft, tf, tt)\n",
    "\n",
    "ut_data = [{'name': 'none', 'count': ff},       \n",
    "           {'name': 'tcp', 'count': tf},     \n",
    "           {'name': 'udp', 'count': ft},           \n",
    "           {'name': 'tcp + udp', 'count': tt}\n",
    "          ]\n",
    "\n",
    "total = ff + ft + tf + tt\n",
    "\n",
    "dfgh = pd.DataFrame(ut_data)\n",
    "dfgh.index = dfgh['name']\n",
    "del dfgh['name']\n",
    "# ax = dfgh.plot.pie('count',\n",
    "# #                    autopct='s(%.2f)',\n",
    "#                    autopct=lambda p : '{:.2f}% ({:.0f})'.format(p, p * total / 100),\n",
    "# #                    radius = 2.5,\n",
    "# #                    pctdistance=1.2,\n",
    "# #                    labeldistance=1.2,\n",
    "# #                    explode = True,\n",
    "#                    figsize = pie_size, \n",
    "#                    legend=False, \n",
    "#                    labels=['','','',''])\n",
    "\n",
    "ax = dfgh.plot.barh()\n",
    "# ax.set_xlim([0,10000])\n",
    "\n",
    "ax.legend(loc='best', labels=dfgh.index)\n",
    "ax.set_xlabel('Number of name servers')\n",
    "ax.set_ylabel('Protocol(s) supported')\n",
    "# ax.set_title('name server udp/tcp support')\n",
    "ax.legend_.remove()\n",
    "ax.set_xscale('log')\n",
    "fig = ax.get_figure()\n",
    "fig.tight_layout()\n",
    "fig.savefig(\"imgs/tcp_udp_generic.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dfgh.index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.loc[df.tld == 'actor.']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# def write_to_file(fn, indir, content):\n",
    "#     with open(indir + fn, 'w') as f:\n",
    "#         f.write(content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# write_to_file('tcp_udp_not_working', 'data/temp/', df_ftcp.loc[df_ftcp.udp == False].to_csv())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# df_ftcp.loc[df_ftcp.udp == False]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# df.loc[df.ip.str.contains(':')].count().tcp #ipv6\n",
    "# df[~df[\"ip\"].str.contains(\":\")].count().tcp #ipv4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# ax = df[~df[\"ip\"].str.contains(\":\")].tcp.value_counts().plot.pie(autopct='%.2f', figsize = pie_size)\n",
    "# ax.set_ylabel('')\n",
    "# fig = ax.get_figure()\n",
    "# fig.savefig(\"imgs/tcp_ipv4.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# ax = df[~df[\"ip\"].str.contains(\":\")].udp.value_counts().plot.pie(autopct='%.2f', figsize = pie_size)\n",
    "# ax.set_ylabel('')\n",
    "# fig = ax.get_figure()\n",
    "# fig.savefig(\"imgs/udp_ipv4.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# ax = df.loc[df.ip.str.contains(':')].tcp.value_counts().plot.pie(autopct='%.2f', figsize = pie_size)\n",
    "# ax.set_ylabel('')\n",
    "# fig = ax.get_figure()\n",
    "# fig.savefig(\"imgs/tcp_ipv6.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# ax = df.loc[df.ip.str.contains(':')].udp.value_counts().plot.pie(autopct='%.2f', figsize = pie_size)\n",
    "# ax.set_ylabel('')\n",
    "# fig = ax.get_figure()\n",
    "# fig.savefig(\"imgs/udp_ipv6.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Credibility"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!dig +noall +answer +noidn -t DNSKEY -f data/lists/tlds > data/dig/tld_dnskeys\n",
    "!dig +noall +answer +noidn -t DS -f data/lists/tlds > data/dig/tld_dss\n",
    "\n",
    "for tld in special_tlds:\n",
    "    !dig +noall +answer +noidn -t DNSKEY {tld} >> data/dig/tld_dnskeys\n",
    "    !dig +noall +answer +noidn -t DS {tld} >> data/dig/tld_dss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 235,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_cred = [{'tld': tld, 'ds': False, 'dnskey': False, 'algorithm': None} for tld in tlds]\n",
    "\n",
    "temp = []\n",
    "\n",
    "for answer in read_list('data/dig/tld_dnskeys'):\n",
    "    v = answer.split()\n",
    "    tld = v[0][:-1].upper()\n",
    "    index = find(data_cred, 'tld', tld)\n",
    "    \n",
    "    try:\n",
    "        data_cred[index]['dnskey'] = True\n",
    "        data_cred[index]['algorithm'] = v[6]\n",
    "    except:\n",
    "        print(tld)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 236,
   "metadata": {},
   "outputs": [],
   "source": [
    "for answer in read_list('data/dig/tld_dss'):\n",
    "    v = answer.split()\n",
    "    tld = v[0][:-1].upper()\n",
    "    \n",
    "    index = find(data_cred, 'tld', tld)\n",
    "    \n",
    "    try:\n",
    "        data_cred[index]['ds'] = True\n",
    "    except:\n",
    "        print(tld)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "write_data('data_cred', data_cred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Organisations per TLD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# tld_orgs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_orgs = pd.DataFrame(tld_orgs)\n",
    "df_orgs.index = df_orgs['tld']\n",
    "del df_orgs['tld']\n",
    "df_orgs.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_orgs.reset_index(inplace=True)\n",
    "rows = []\n",
    "_ = df_orgs.apply(lambda row: [rows.append([row['tld'], nn]) \n",
    "                         for nn in row.organisations], axis=1)\n",
    "df_orgs_new = pd.DataFrame(rows, columns=df_orgs.columns).set_index(['tld'])\n",
    "\n",
    "df_orgs_new.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_orgs_new.organisations.value_counts(ascending=False).head(80).plot.barh(figsize = (10,20))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "bins = df_orgs_new.organisations.value_counts().nunique() - 1\n",
    "ax = df_orgs_new.organisations.value_counts().hist(bins = bins)\n",
    "ax.set_yscale('log')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_orgs_new.organisations.value_counts().value_counts().plot.pie()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# df_orgs_new.organisations.value_counts()\n",
    "\n",
    "# df2[df2['rr_quality'] > 0]].groupby([df2.index.hour,'sleep_summary_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ax = df_tld_orgs.organisation.value_counts().head(20).plot.barh(figsize = bar_size, fontsize=12)\n",
    "ax.set_xlabel('Number of TLDs', fontsize = 16)\n",
    "ax.set_ylabel('Organisation',fontsize = 16)\n",
    "fig = ax.get_figure()\n",
    "fig.savefig(\"imgs/orgs.png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ax = df_tld_orgs.type.value_counts().plot.pie(figsize = pie_size, legend=True)\n",
    "ax.set_ylabel('')\n",
    "fig = ax.get_figure()\n",
    "fig.savefig(\"imgs/types.png\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Growth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_age = []\n",
    "dage = {}\n",
    "\n",
    "for datum in tld_creation:\n",
    "    y, m, d = datum['date_created'].split('-')\n",
    "    if y in ['2014', '2015', '2016', '2017'] or y == '2013' and int(m) >= 10:\n",
    "        data_age.append({'tld': datum['tld'], 'age': 'new'})\n",
    "        dage[datum['tld']] = 'new'\n",
    "    else:\n",
    "        data_age.append({'tld': datum['tld'], 'age': 'old'})\n",
    "        dage[datum['tld']] = 'old'"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}