{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from dns.resolver import Resolver, NXDOMAIN, NoNameservers, Timeout, NoAnswer, query\n",
    "from uuid import uuid4\n",
    "from tqdm import tqdm\n",
    "\n",
    "import matplotlib as plt\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import requests\n",
    "import time\n",
    "import json\n",
    "import copy\n",
    "import os\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "atlas_api_key = 'e057e19f-53ae-4b66-9b5e-c8bc00d7b4fe'\n",
    "url_dns_measurements_create = 'https://atlas.ripe.net:443/api/v2/measurements/dns/'\n",
    "url_dns_measurements_get = 'https://atlas.ripe.net:443/api/v2/measurements/dns/'\n",
    "\n",
    "newline = '\\n'\n",
    "\n",
    "figsize = (6, 4) #default\n",
    "figsize = (15, 10)\n",
    "\n",
    "min_meas_id = 8759930\n",
    "max_meas_id = 8770979\n",
    "\n",
    "headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def write_list(fn, data):\n",
    "    '''Writes a list to a file with each value on a new line'''\n",
    "    with open(fn, 'w') as f:\n",
    "        for datum in data:\n",
    "            f.write(datum + newline)\n",
    "        \n",
    "def append_list(fn, data):\n",
    "    '''Appends a list to a file with each value on a new line'''\n",
    "    with open(fn, 'a') as f:\n",
    "        for datum in data:\n",
    "            f.write(datum + newline)\n",
    "    \n",
    "def read_list(fn):\n",
    "    '''Reads a file and '''\n",
    "    with open(fn, 'r') as f:\n",
    "        return [line.strip(newline) for line in f]\n",
    "    \n",
    "def write_json(fn, data):\n",
    "    with open(fn, 'w') as f:\n",
    "        f.write(json.dumps(data))\n",
    "            \n",
    "def read_json(fn):\n",
    "    '''Read a json file (fn) and returns it as a dictionary'''\n",
    "    with open(fn, 'r') as f:\n",
    "        return json.loads(f.read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def write_data(fn, data):\n",
    "    \"\"\"Backs up the previous version of the data if it exists and writes the new data to a file.\"\"\"\n",
    "    # Backs up the previous data if it exists.\n",
    "    try:\n",
    "        write_json(\"data/backup/{}.json \".format(fn) + time.ctime().replace(' ', '-'), \n",
    "                   read_json(\"data/{}.json\".format(fn)))\n",
    "    except:\n",
    "        pass\n",
    "\n",
    "    write_json(\"data/{}.json\".format(fn), data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = []\n",
    "\n",
    "with open('data/tlds', 'r') as f:\n",
    "    next(f)\n",
    "    \n",
    "    for line in f:\n",
    "        data.append({'tld': line[:-1].lower()})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Test tld set\n",
    "# data = [{'tld': 'nl'}, {'tld': 'audi'}]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def find_nxdomain(tld, max_tries = 3):\n",
    "    for _ in range(max_tries):\n",
    "        domain = '{}.{}'.format(str(uuid4()), tld)\n",
    "        \n",
    "        try:\n",
    "            query(domain)\n",
    "        except:\n",
    "            return domain\n",
    "    \n",
    "    return None\n",
    "\n",
    "def find_nxdomain_wildcard(tld, max_tries = 3):\n",
    "    for _ in range(max_tries):\n",
    "        domain = '{}.{}'.format(str(uuid4()), tld)\n",
    "\n",
    "        response = !dig soa +noall +authority +noidn {domain}\n",
    "\n",
    "        if response[0].startswith(tld):\n",
    "            return domain\n",
    "    \n",
    "    return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "for datum in data:\n",
    "    domain = str(uuid4()) + '.' + datum['tld']\n",
    "        \n",
    "    try:\n",
    "        query(domain)\n",
    "        print(datum['tld'], 'DOMAIN EXISTS')\n",
    "    except NXDOMAIN:\n",
    "        datum['domain'] = domain\n",
    "    except NoNameservers:\n",
    "        print(datum['tld'], 'NO NAMESERVERS')\n",
    "    except Timeout:\n",
    "        print(datum['tld'], 'TIME OUT')\n",
    "    except NoAnswer:\n",
    "        print(datum['tld'], 'NO ANSWER')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "for datum in data:\n",
    "    if not 'domain' in datum:\n",
    "        domain = str(uuid4()) + '.' + datum['tld']\n",
    "\n",
    "        try:\n",
    "            query(domain)\n",
    "            print(datum['tld'], 'DOMAIN EXISTS')\n",
    "        except NXDOMAIN:\n",
    "            datum['domain'] = domain\n",
    "        except NoNameservers:\n",
    "            print(datum['tld'], 'NO NAMESERVERS')\n",
    "        except Timeout:\n",
    "            print(datum['tld'], 'TIME OUT')\n",
    "        except NoAnswer:\n",
    "            print(datum['tld'], 'NO ANSWER')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# wildcard check\n",
    "for item in data:\n",
    "    if len(item) == 1:\n",
    "        domain = str(uuid4()) + '.' + item['tld']\n",
    "        print(domain)\n",
    "        \n",
    "        bashCommand = \"dig soa +noall +authority \" + domain\n",
    "        process = subprocess.Popen(bashCommand, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)\n",
    "        out = str(process.stdout.read())\n",
    "        \n",
    "        print(out)\n",
    "        \n",
    "        if out.startswith(\"b'\" + item['tld']):\n",
    "            item['domain'] = domain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_2 = data.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# data = [{'tld': 'nl'}]\n",
    "for datum in data:\n",
    "    if not 'domain' in datum:\n",
    "        print(datum)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = pd.DataFrame(data)\n",
    "df.loc[df.domain.isnull()].tld"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "payload = {\n",
    "    \"bill_to\": \"bickerkards@gmail.com\",\n",
    "    \"is_oneoff\": True,\n",
    "    \"definitions\": [],\n",
    "    \"probes\": []\n",
    "}\n",
    "    \n",
    "definition = {\n",
    "    \"af\":4,\n",
    "    \"query_class\":\"IN\",\n",
    "    \"query_type\":\"A\",\n",
    "    \"query_argument\": \"nlnetlabs.nl\",\n",
    "    \"description\":\"Test getting probes\",\n",
    "    \"use_probe_resolver\":True,\n",
    "    \"resolve_on_probe\":False,\n",
    "    \"set_nsid_bit\":True,\n",
    "    \"protocol\":\"UDP\",\n",
    "    \"udp_payload_size\":512,\n",
    "    \"retry\":0,\n",
    "    \"skip_dns_check\":False,\n",
    "    \"include_qbuf\":False,\n",
    "    \"include_abuf\":True,\n",
    "    \"prepend_probe_id\":False,\n",
    "    \"set_rd_bit\":False,\n",
    "    \"set_do_bit\":False,\n",
    "    \"set_cd_bit\":False,\n",
    "    \"type\":\"dns\",\n",
    "    \"is_public\":True\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "probe_ids = [10262, 10287, 11040, 11429, 12515, 12873, 12956, 13623, 13728, 13769, 13788, 13799, 13804, 13805, 13810, 14237, 26057, 14564, 15156, 14691, 15594, 15799, 4205, 18131, 18195, 18691, 19326, 19740, 20111, 20353, 20493, 20531, 20621, 21003, 21035, 21122, 21251, 21345, 21703, 22286, 22695, 23031, 23085, 28240, 27972, 23697, 24807, 25011, 25148, 25323, 26936, 26378, 26627, 4155, 26823, 28355, 30676, 4829, 29006, 29183, 29405, 30225, 30324, 31201, 19306, 19634, 6025, 11660, 22388, 25182, 4123, 3812, 20923, 14384, 12389]\n",
    "\n",
    "probes = [\n",
    "    {\n",
    "        \"value\": str(probe_ids)[1:-1],\n",
    "        \"type\": \"probes\",\n",
    "        \"requested\": len(probe_ids)\n",
    "    }\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "payloads = []\n",
    "step_size = 50\n",
    "    \n",
    "for i in range(0, len(data), step_size):\n",
    "    defintions = []\n",
    "    \n",
    "    for datum in data[i:i + step_size]:\n",
    "        definition_caching = definition.copy()\n",
    "        definition_caching['query_type'] = \"NS\"\n",
    "        definition_caching['query_argument'] = datum['tld']\n",
    "        definition_caching['description'] = \"caching \" + datum['tld']\n",
    "        defintions.append(definition_caching)\n",
    "\n",
    "        definition_measuring = definition.copy()\n",
    "        definition_measuring['query_type'] = \"SOA\"\n",
    "        definition_measuring['query_argument'] = datum['domain']\n",
    "        definition_measuring['description'] = \"measuring \" + datum['tld']\n",
    "        defintions.append(definition_measuring)        \n",
    "\n",
    "    new_payload = payload.copy()\n",
    "    new_payload['probes'] = probes\n",
    "    new_payload['definitions'] = defintions\n",
    "\n",
    "    payloads.append(new_payload)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "measurement_ids = []\n",
    "measurement_responses = []\n",
    "url = url_dns_measurements_create + '?key=' + atlas_api_key"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('data/payloads', 'r') as f:\n",
    "    payloads = json.loads(f.read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# measurement_ids.append(requests.post(url, data = json.dumps(payloads[0]), headers = headers))\n",
    "# [x['description'] for x in payloads[1]['definitions']]\n",
    "# request.status_code\n",
    "# measurement_ids\n",
    "# len(payloads)\n",
    "# request.json()\n",
    "payloads[23]['definitions'][-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true,
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "# split into 3\n",
    "for payload in payloads[23:]:\n",
    "    \n",
    "    request = requests.post(url, data = json.dumps(payload), headers = headers)\n",
    "    print(request.status_code)\n",
    "    \n",
    "    while request.status_code == 400:\n",
    "        print(request.json())\n",
    "        request = requests.post(url, data = json.dumps(payload), headers = headers)\n",
    "        time.sleep(300)\n",
    "        print(request.status_code)\n",
    "    \n",
    "    measurement_ids += measurement_ids + request.json()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for id in measurement_ids:\n",
    "    print(requests.get(url_dns_measurements_get + '?id__in=' + str(id)).json())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# with open('data/measurement_ids', 'w') as f:\n",
    "#     f.write(json.dumps(measurement_ids))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "temp_url = url_dns_measurements_get + '?id__lte=' + str(max(measurement_ids)) + '&id__gte=' + str(min(measurement_ids)) + '&description__startswith=measuring&mine=true'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "temp_url = url_dns_measurements_get + '?id__lte=' + str(max_meas_id) + '&id__gte=' + str(min_meas_id) + '&description__startswith=measuring&mine=true'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# # measurements = [requests.get(url_dns_measurements_get + '?id__in=' + str(id)).json() for id in measurement_ids]\n",
    "# measurements = []\n",
    "# pbar = tqdm(total=len(measurement_ids))\n",
    "\n",
    "# for id in measurement_ids:\n",
    "#     measurements.append(requests.get(url_dns_measurements_get + '?id__in=' + str(id)).json()) \n",
    "        \n",
    "#     pbar.update(1)\n",
    "# pbar.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# xyz = probe_ids.copy()\n",
    "\n",
    "# for probe_idsd in [r['prb_id'] for r in request]:\n",
    "#     xyz.remove(probe_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tlds = []\n",
    "\n",
    "with open('data/tlds') as f:\n",
    "#     next(f)\n",
    "    \n",
    "    for line in f:\n",
    "        tlds.append(line.strip('\\n').lower())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tld_timeouts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "results = {}\n",
    "xprobes = {}\n",
    "\n",
    "next = True\n",
    "measurements = requests.get(temp_url).json()\n",
    "\n",
    "while next:\n",
    "    for result in measurements['results']:\n",
    "        measurement_type, tld = result['description'].split()\n",
    "\n",
    "        if measurement_type == 'measuring':\n",
    "            request = requests.get(result['result']).json()\n",
    "\n",
    "            for probe in request:\n",
    "                for result2 in probe['resultset']:\n",
    "                    if 'result' in result2:\n",
    "                        if tld in results:\n",
    "                            results[tld].append(result2['result']['rt'])\n",
    "                        else:\n",
    "                            results[tld] = [result2['result']['rt']]\n",
    "                    else:\n",
    "                        print(tld, probe['prb_id'], result2['error'])\n",
    "                        \n",
    "                        if tld in tld_timeouts:\n",
    "                            tld_timeouts[tld] += 1\n",
    "                        else:\n",
    "                            tld_timeouts[tld] = 1\n",
    "\n",
    "                        if probe['prb_id'] in xprobes:\n",
    "                            xprobes[probe['prb_id']] += 1\n",
    "                        else:\n",
    "                            xprobes[probe['prb_id']] = 1\n",
    "                            \n",
    "    if measurements['next']:\n",
    "        print('\\n' + measurements['next'].split('=')[-1] + '\\n')\n",
    "        measurements = requests.get(measurements['next']).json() \n",
    "    else:\n",
    "        next = False\n",
    "        print('Done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "!mkdir data/atlas\n",
    "!mkdir data/atlas/ns\n",
    "!mkdir data/atlas/soa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "temp_url = url_dns_measurements_get + '?id__lte=' + str(max_meas_id) + '&id__gte=' + str(min_meas_id) + '&mine=true'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  0%|          | 0/1531 [00:00<?, ?it/s]\u001b[A\n",
      "  0%|          | 1/1531 [00:00<05:10,  4.92it/s]\u001b[A\n",
      "  0%|          | 2/1531 [00:00<05:05,  5.00it/s]\u001b[A\n",
      "  0%|          | 3/1531 [00:00<04:56,  5.15it/s]\u001b[A\n",
      "  0%|          | 4/1531 [00:00<04:41,  5.42it/s]\u001b[A\n",
      "  0%|          | 5/1531 [00:03<27:53,  1.10s/it]\u001b[A\n",
      "  0%|          | 6/1531 [00:04<21:40,  1.17it/s]\u001b[A\n",
      "  0%|          | 7/1531 [00:04<19:31,  1.30it/s]\u001b[A\n",
      "  1%|          | 8/1531 [00:05<16:34,  1.53it/s]\u001b[A\n",
      "  1%|          | 9/1531 [00:05<13:47,  1.84it/s]\u001b[A\n",
      "  1%|          | 10/1531 [00:05<11:49,  2.14it/s]\u001b[A\n",
      "  1%|          | 11/1531 [00:05<09:56,  2.55it/s]\u001b[A\n",
      "  1%|          | 12/1531 [00:06<08:49,  2.87it/s]\u001b[A\n",
      "  1%|          | 13/1531 [00:06<07:36,  3.33it/s]\u001b[A\n",
      "  1%|          | 14/1531 [00:06<07:00,  3.60it/s]\u001b[A\n",
      "  1%|          | 15/1531 [00:06<06:32,  3.86it/s]\u001b[A\n",
      "  1%|          | 16/1531 [00:07<05:58,  4.23it/s]\u001b[A\n",
      "  1%|          | 17/1531 [00:07<05:58,  4.23it/s]\u001b[A\n",
      "  1%|          | 18/1531 [00:07<05:45,  4.38it/s]\u001b[A\n",
      "  1%|          | 19/1531 [00:07<05:42,  4.42it/s]\u001b[A\n",
      "  7%|▋         | 100/1531 [00:24<04:47,  4.98it/s]\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m/usr/lib/python3.6/site-packages/urllib3/connectionpool.py\u001b[0m in \u001b[0;36m_make_request\u001b[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[1;32m    378\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# Python 2.7, use buffering of HTTP responses\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 379\u001b[0;31m                 \u001b[0mhttplib_response\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetresponse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbuffering\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    380\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# Python 2.6 and older, Python 3\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: getresponse() got an unexpected keyword argument 'buffering'",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-35-f1320228216d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      8\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'description'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m             \u001b[0mmeasurement_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtld\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'description'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m             \u001b[0mrequest\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrequests\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'result'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjson\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mmeasurement_type\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'measuring'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.6/site-packages/requests/api.py\u001b[0m in \u001b[0;36mget\u001b[0;34m(url, params, **kwargs)\u001b[0m\n\u001b[1;32m     68\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     69\u001b[0m     \u001b[0mkwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msetdefault\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'allow_redirects'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 70\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mrequest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'get'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     71\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     72\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.6/site-packages/requests/api.py\u001b[0m in \u001b[0;36mrequest\u001b[0;34m(method, url, **kwargs)\u001b[0m\n\u001b[1;32m     54\u001b[0m     \u001b[0;31m# cases, and look like a memory leak in others.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     55\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0msessions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSession\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 56\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmethod\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmethod\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     57\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     58\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.6/site-packages/requests/sessions.py\u001b[0m in \u001b[0;36mrequest\u001b[0;34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[0m\n\u001b[1;32m    486\u001b[0m         }\n\u001b[1;32m    487\u001b[0m         \u001b[0msend_kwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msettings\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 488\u001b[0;31m         \u001b[0mresp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprep\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0msend_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    489\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    490\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mresp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.6/site-packages/requests/sessions.py\u001b[0m in \u001b[0;36msend\u001b[0;34m(self, request, **kwargs)\u001b[0m\n\u001b[1;32m    607\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    608\u001b[0m         \u001b[0;31m# Send the request\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 609\u001b[0;31m         \u001b[0mr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0madapter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    610\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    611\u001b[0m         \u001b[0;31m# Total elapsed time of the request (approximately)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.6/site-packages/requests/adapters.py\u001b[0m in \u001b[0;36msend\u001b[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[1;32m    421\u001b[0m                     \u001b[0mdecode_content\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    422\u001b[0m                     \u001b[0mretries\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_retries\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 423\u001b[0;31m                     \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    424\u001b[0m                 )\n\u001b[1;32m    425\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.6/site-packages/urllib3/connectionpool.py\u001b[0m in \u001b[0;36murlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001b[0m\n\u001b[1;32m    598\u001b[0m                                                   \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout_obj\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    599\u001b[0m                                                   \u001b[0mbody\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbody\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mheaders\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mheaders\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 600\u001b[0;31m                                                   chunked=chunked)\n\u001b[0m\u001b[1;32m    601\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    602\u001b[0m             \u001b[0;31m# If we're going to release the connection in ``finally:``, then\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.6/site-packages/urllib3/connectionpool.py\u001b[0m in \u001b[0;36m_make_request\u001b[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[1;32m    380\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# Python 2.6 and older, Python 3\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    381\u001b[0m                 \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 382\u001b[0;31m                     \u001b[0mhttplib_response\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetresponse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    383\u001b[0m                 \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    384\u001b[0m                     \u001b[0;31m# Remove the TypeError from the exception chain in Python 3;\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.6/http/client.py\u001b[0m in \u001b[0;36mgetresponse\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1329\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1330\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1331\u001b[0;31m                 \u001b[0mresponse\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbegin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1332\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mConnectionError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1333\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.6/http/client.py\u001b[0m in \u001b[0;36mbegin\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    295\u001b[0m         \u001b[0;31m# read until we get a non-100 response\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    296\u001b[0m         \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 297\u001b[0;31m             \u001b[0mversion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstatus\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreason\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_read_status\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    298\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mstatus\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mCONTINUE\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    299\u001b[0m                 \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.6/http/client.py\u001b[0m in \u001b[0;36m_read_status\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    256\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    257\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_read_status\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 258\u001b[0;31m         \u001b[0mline\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadline\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_MAXLINE\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"iso-8859-1\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    259\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mline\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0m_MAXLINE\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    260\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mLineTooLong\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"status line\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.6/socket.py\u001b[0m in \u001b[0;36mreadinto\u001b[0;34m(self, b)\u001b[0m\n\u001b[1;32m    584\u001b[0m         \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    585\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 586\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecv_into\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    587\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    588\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_timeout_occurred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.6/ssl.py\u001b[0m in \u001b[0;36mrecv_into\u001b[0;34m(self, buffer, nbytes, flags)\u001b[0m\n\u001b[1;32m   1000\u001b[0m                   \u001b[0;34m\"non-zero flags not allowed in calls to recv_into() on %s\"\u001b[0m \u001b[0;34m%\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1001\u001b[0m                   self.__class__)\n\u001b[0;32m-> 1002\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnbytes\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbuffer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1003\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1004\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0msocket\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecv_into\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbuffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnbytes\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mflags\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.6/ssl.py\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, len, buffer)\u001b[0m\n\u001b[1;32m    863\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Read on closed or unwrapped SSL socket.\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    864\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 865\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sslobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbuffer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    866\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mSSLError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    867\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mSSL_ERROR_EOF\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msuppress_ragged_eofs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.6/ssl.py\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, len, buffer)\u001b[0m\n\u001b[1;32m    623\u001b[0m         \"\"\"\n\u001b[1;32m    624\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mbuffer\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 625\u001b[0;31m             \u001b[0mv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sslobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbuffer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    626\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    627\u001b[0m             \u001b[0mv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sslobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "next_result = True\n",
    "measurements = requests.get(temp_url).json()\n",
    "pbar = tqdm(total=1531)\n",
    "# print('1')\n",
    "\n",
    "while next_result:\n",
    "    for result in measurements['results']:\n",
    "        if len(result['description'].split()) == 2:\n",
    "            measurement_type, tld = result['description'].split()\n",
    "            request = requests.get(result['result']).json()\n",
    "\n",
    "            if measurement_type == 'measuring':\n",
    "                with open('data/atlas/soa/{}.json'.format(tld.upper()), 'w') as f:\n",
    "                    temp = copy.deepcopy(result)\n",
    "                    temp['result'] = request\n",
    "\n",
    "                    f.write(json.dumps(temp))\n",
    "\n",
    "            elif measurement_type == 'caching':\n",
    "                with open('data/atlas/ns/{}.json'.format(tld.upper()), 'w') as f:\n",
    "                    temp = copy.deepcopy(result)\n",
    "                    temp['result'] = request\n",
    "\n",
    "                    f.write(json.dumps(temp))\n",
    "\n",
    "        pbar.update(1)\n",
    "    pbar.close()\n",
    "    \n",
    "    if measurements['next']:\n",
    "#         print(measurements['next'].split('=')[-1])\n",
    "        measurements = requests.get(measurements['next']).json() \n",
    "    else:\n",
    "        next_result = False\n",
    "        print('Done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "indir = 'data/atlas/soa/'\n",
    "\n",
    "data_perf = []\n",
    "\n",
    "for root, dirs, filenames in os.walk(indir):\n",
    "    for f in filenames:\n",
    "#         tld, _ = f.split('.')\n",
    "        tld = f\n",
    "        \n",
    "        datum = {'tld': tld, 'rt': [], 'timeouts': 0}\n",
    "        \n",
    "        with open(indir + f, 'r') as f:\n",
    "            tld_results = json.loads(f.read())\n",
    "            \n",
    "            for probe in tld_results['result']:\n",
    "                for result in probe['resultset']:\n",
    "                    if 'result' in result:\n",
    "                        datum['rt'].append(result['result']['rt'])                    \n",
    "                    elif 'error' in result and 'timeout' in result['error']:\n",
    "                        datum['timeouts'] += 1\n",
    "        \n",
    "        datum['rt'] = np.mean(datum['rt'])\n",
    "        data_perf.append(datum)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "write_data('data_perf', data_perf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# df2.sort_values('ntimeouts', ascending=False).head()\n",
    "# df2.ntimeouts.hist(bins=8, align='right')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2 = pd.DataFrame(list(tld_timeouts.items()), columns=['tld', 'ntimeouts'])\n",
    "ax = df2.hist(bins=8, align='left', color='grey')\n",
    "\n",
    "for a in ax:\n",
    "    for b in a:\n",
    "        b.set_xlabel(\"Number of timeouts\")\n",
    "        b.set_ylabel(\"Number of TLDs\")\n",
    "        b.set_title('')\n",
    "        b.set_yscale('log')\n",
    "        b.set_facecolor('lightgrey')\n",
    "        fig = b.get_figure()\n",
    "        fig.savefig(\"imgs/per_timeouts.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "[x for x in tld_timeouts if tld_timeouts[x] > 7]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tld_timeouts['xn--ygbi2ammx']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "c = 0\n",
    "for tld in tld_timeouts:\n",
    "    if tld_timeouts[tld] > 0:\n",
    "        c+=1\n",
    "c / len(tld_timeouts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df2.loc[df2.ntimeouts > 5].sort_values('ntimeouts', ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tld_timeouts['fk']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "xdata = {'tld': [tld for tld in results], 'rt': [np.mean(results[tld]) for tld in results]}\n",
    "df = pd.DataFrame(xdata, columns = ['tld', 'rt'])\n",
    "df.index = df['tld']\n",
    "del df['tld']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "edata = {'tld': [tld for tld in results], 'rt': [np.std(results[tld]) for tld in results]}\n",
    "dfe = pd.DataFrame(edata, columns = ['tld', 'rt'])\n",
    "dfe.index = dfe['tld']\n",
    "del dfe['tld']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ax = df.hist(bins=160, range=(0,1400))\n",
    "\n",
    "for a in ax:\n",
    "    for b in a:\n",
    "        b.set_xlim(0,1400)\n",
    "        b.set_xlabel(\"Response time (ms)\")\n",
    "        b.set_ylabel(\"Number of TLDs\")\n",
    "        b.set_title('')\n",
    "        fig = b.get_figure()\n",
    "        fig.savefig(\"imgs/per.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.loc[df.rt > 600]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "caching_ids = [response.json() for response in chaching_ids]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "chaching_ids2 = []\n",
    "\n",
    "for i in range(202,1531,200):\n",
    "    print(i)\n",
    "    for payload in payloads_caching[i:i + 200]:\n",
    "        url = url_dns_measurements_create + '?key=' + atlas_api_key\n",
    "        chaching_ids2.append(requests.post(url, data = json.dumps(payload), headers = headers))\n",
    "    \n",
    "    time.sleep(300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# tmp_res = []\n",
    "\n",
    "for item in chaching_ids2:\n",
    "    if isinstance(item.json(), list):\n",
    "        tmp_res.append((item.json()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tmp_res2 = []\n",
    "\n",
    "for i in tmp_res:\n",
    "    for j in i:\n",
    "        tmp_res2.append(j)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "rs = []\n",
    "\n",
    "for id in tmp_res2:\n",
    "    rs.append(requests.get(url_dns_measurements_get + 'id__in=' + str(id)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "chaching_ids2[0].json()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tlds2 = tlds.copy()\n",
    "\n",
    "for tld in deltlds:\n",
    "    tlds2.remove(tld)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# measuring\n",
    "for item in data:\n",
    "    item['definitions']['query_type'] = \"SOA\"\n",
    "    item['definitions']['query_argument'] = item['domain']\n",
    "    item['definitions']['description'] = item['tld'] + ' measurement'\n",
    "    \n",
    "url = url_dns_measurements_create + '?key=' + atlas_api_key\n",
    "measurement_ids = requests.post(url, data = json.dumps(payload), headers = headers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for id in measurement ids.json():\n",
    "    r = requests.get(url_dns_measurements_get + 'id__in=' + id)\n",
    "    # s = requests.get(r.json()['result'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "id 44710\n",
      "opcode QUERY\n",
      "rcode NXDOMAIN\n",
      "flags QR RD RA\n",
      "edns 0\n",
      "payload 4000\n",
      ";QUESTION\n",
      "747032ef-ce8e-4f4c-99f9-e433073738dd.de. IN A\n",
      ";ANSWER\n",
      ";AUTHORITY\n",
      "de. 5400 IN SOA f.nic.de. its.denic.de. 2017050961 7200 7200 3600000 7200\n",
      ";ADDITIONAL\n"
     ]
    }
   ],
   "source": [
    "import base64\n",
    "import dns.message\n",
    "\n",
    "x = dns.message.from_wire(base64.b64decode('rqaBgwABAAAAAQABJDc0NzAzMmVmLWNlOGUtNGY0Yy05OWY5LWU0MzMwNzM3MzhkZAJkZQAAAQABwDEABgABAAAVGAAoAWYDbmljwDEDaXRzBWRlbmljwDF4OcFRAAAcIAAAHCAANu6AAAAcIAAAKQ+gAAAAAAAA'))\n",
    "print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# help(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for i in range(0, len(data), 50):\n",
    "    print(data[i]['tld'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "definition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "probes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def find(lst, key, value):\n",
    "    for i, dic in enumerate(lst):\n",
    "        if dic[key] == value:\n",
    "            return i\n",
    "    return -1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('data/tld_type', 'r') as f:\n",
    "    data_type = json.loads(f.read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('data/tld_age', 'r') as f:\n",
    "    data_age = json.loads(f.read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_rt = [{'tld': tld, 'rt': np.mean(results[tld])} for tld in results]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for datum in data_rt:\n",
    "    ix = find(data_type, 'tld', datum['tld'])\n",
    "    datum['type'] = data_type[ix]['type']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "find(data_type, 'tld', 'aaa')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dft = pd.DataFrame(data_rt, columns = ['rt', 'tld', 'type'])\n",
    "dft.index = dft['tld']\n",
    "del dft['tld']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dft.loc[dft.type == 'country-code'].mean().rt, dft.loc[dft.type == 'generic'].mean().rt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfa.loc[dfa.age == 'new'].mean().rt, dfa.loc[dfa.age == 'old'].mean().rt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.mean().rt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "nbins = 160"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ax = dft.loc[dft.type == 'country-code'].hist('rt', \n",
    "                                              bins=nbins, \n",
    "                                              range=(0,1400),\n",
    "                                              cumulative=False,\n",
    "                                              align='mid',\n",
    "                                              figsize=(6,4))\n",
    "\n",
    "for a in ax:\n",
    "    for b in a:\n",
    "        b.set_xlim(0,1400)\n",
    "        b.set_xlabel(\"Response time (ms)\")\n",
    "        b.set_ylabel(\"Number of TLDs\")\n",
    "        b.set_title('')\n",
    "        fig = b.get_figure()\n",
    "        fig.savefig(\"imgs/per_cctlds.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ax = dft.loc[dft.type == 'generic'].hist('rt', bins=nbins, range=(0,1400),cumulative=False,figsize=(6,4))\n",
    "\n",
    "for a in ax:\n",
    "    for b in a:\n",
    "        b.set_xlim(0,1400)\n",
    "        b.set_xlabel(\"Response time (ms)\")\n",
    "        b.set_ylabel(\"Number of TLDs\")\n",
    "        b.set_title('')\n",
    "        fig = b.get_figure()\n",
    "        fig.savefig(\"imgs/per_gtlds.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_age[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for datum in data_age:\n",
    "    print(datum['tld'])\n",
    "    ix = find(data_rt, 'tld', datum['tld'].lower())\n",
    "    datum['rt'] = data_rt[ix]['rt']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dfa = pd.DataFrame(data_age, columns = ['age', 'rt', 'tld'])\n",
    "dfa.index = dfa['tld']\n",
    "del dfa['tld']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ax = dfa.loc[dfa.age == 'new'].hist('rt', \n",
    "                                    bins=nbins, \n",
    "                                    range=(0,1400),\n",
    "                                    cumulative=False)\n",
    "\n",
    "for a in ax:\n",
    "    for b in a:\n",
    "        b.set_xlim(0,1400)\n",
    "        b.set_xlabel(\"Response time (ms)\")\n",
    "        b.set_ylabel(\"Number of TLDs\")\n",
    "        b.set_title('')\n",
    "        fig = b.get_figure()\n",
    "        fig.savefig(\"imgs/per_new.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ax = dfa.loc[dfa.age == 'old'].hist('rt', \n",
    "                                    bins=nbins, \n",
    "                                    range=(0,1400),\n",
    "                                    cumulative=False)\n",
    "\n",
    "for a in ax:\n",
    "    for b in a:\n",
    "        b.set_xlim(0,1400)\n",
    "        b.set_xlabel(\"Response time (ms)\")\n",
    "        b.set_ylabel(\"Number of TLDs\")\n",
    "        b.set_title('')\n",
    "        fig = b.get_figure()\n",
    "        fig.savefig(\"imgs/per_old.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "indir = 'data/ripe/soa/'\n",
    "prb_rt = {}\n",
    "\n",
    "for root, dirs, filenames in os.walk(indir):\n",
    "    for f in filenames:\n",
    "        tld, _ = f.split('.')\n",
    "        \n",
    "        with open(indir + f, 'r') as f:\n",
    "            tld_results = json.loads(f.read())\n",
    "            \n",
    "            for probe in tld_results['result']:\n",
    "                prb_id = probe['prb_id']\n",
    "                \n",
    "                for result in probe['resultset']:\n",
    "                    if 'result' in result:\n",
    "                        if prb_id in prb_rt:\n",
    "                            prb_rt[prb_id].append(result['result']['rt'])\n",
    "                        else:\n",
    "                            prb_rt[prb_id] = [result['result']['rt']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "prb_data = [{'probe': probe, 'rt': np.mean(prb_rt[probe])} for probe in prb_rt]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = pd.DataFrame(prb_data, columns = ['probe', 'rt'])\n",
    "df.index = df['probe']\n",
    "del df['probe']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.plot.bar(figsize=(18,9))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "r = requests.get('https://atlas.ripe.net/api/v2/probes/?id__in=1' + str([probe for probe in prb_rt])[1:-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "probe_cc = {}\n",
    "\n",
    "for v in r.json()['results']:\n",
    "    probe_cc[v['id']] = v['country_code']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "xyz = []\n",
    "\n",
    "for datum in prb_data:\n",
    "    if datum['probe'] in probe_cc:\n",
    "        xyz.append({'cc': probe_cc[datum['probe']], 'rt': datum['rt']})\n",
    "#         datum['probe'] = probe_cc[datum['probe']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('data/cc_rt', 'r') as f:\n",
    "    xyz = json.loads(f.read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "xyz2 = {}\n",
    "\n",
    "for x in xyz:\n",
    "    if x['cc'] in xyz2:\n",
    "        xyz2[x['cc']][0] = x['rt'] + xyz2[x['cc']][0] / 2\n",
    "        xyz2[x['cc']][1] += 1\n",
    "    else: \n",
    "        xyz2[x['cc']] = [x['rt'], 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "xyz = [{'cc': x + ' (' + str(xyz2[x][1]) + ')', 'rt': xyz2[x][0]} for x in xyz2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dfcc = pd.DataFrame(xyz, columns = ['cc', 'rt'])\n",
    "dfcc.index = dfcc['cc']\n",
    "del dfcc['cc']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# dfcc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfcc.sort_values('rt', ascending=False).plot.bar(figsize=(12,7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "f = read_list('data/dig/tld_nss')\n",
    "data = {}\n",
    "\n",
    "for line in f:\n",
    "    tld, _, _, _, ns = line.split()\n",
    "    tld = tld.strip('.')\n",
    "\n",
    "    if tld != '.':\n",
    "        if tld in data:\n",
    "            data[tld] += 1\n",
    "        else:\n",
    "            data[tld] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ns_data = {}\n",
    "\n",
    "for i in range(0,16):\n",
    "    ns_data[i] = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for tld in results:\n",
    "    try:\n",
    "        ns_data[data[tld]].append({'tld': tld, 'rt': np.mean(results[tld])})\n",
    "    except:\n",
    "        print(tld)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(15,16):\n",
    "    if ns_data[i] == []:\n",
    "        del ns_data[i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# bins = []\n",
    "\n",
    "# for i in ns_data:\n",
    "#     dfxyz = pd.DataFrame(ns_data[i])\n",
    "#     bins.append(dfxyz.rt.nunique())\n",
    "    \n",
    "# bins = min(bins)\n",
    "\n",
    "for i in ns_data:\n",
    "    dfxyz = pd.DataFrame(ns_data[i])\n",
    "    ax = dfxyz.hist('rt', bins=32, range=(0,1400))\n",
    "\n",
    "    for a in ax:\n",
    "        for b in a:\n",
    "            b.set_xlim(0,1400)\n",
    "            b.set_xlabel(\"Response time (ms)\")\n",
    "            b.set_ylabel(\"Number of TLDs\")\n",
    "            b.set_title('Average response times of TLDs with ' + str(i) + ' name servers')\n",
    "            fig = b.get_figure()\n",
    "            fig.savefig(\"imgs/per_ns_\" + str(i) + \".png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tld_orgs = []\n",
    "import os\n",
    "indir = 'data/whois/'\n",
    "\n",
    "for root, dirs, filenames in os.walk(indir):\n",
    "    tld_orgs = [{'tld': tld, 'organisations': []} for tld in filenames]\n",
    "    \n",
    "    for fn in filenames:\n",
    "        with open(indir + fn, 'r') as f:\n",
    "            for line in f:\n",
    "                if line.startswith('organisation'):\n",
    "                    _, org = line.split('rganisation: ')\n",
    "                    i = find(tld_orgs, 'tld', fn)\n",
    "                    tld_orgs[i]['organisations'].append(org.strip('\\n'))\n",
    "    \n",
    "# tld_creation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "org_tlds = {}\n",
    "\n",
    "for item in tld_orgs:\n",
    "    for org in item['organisations']:\n",
    "        if org in org_tlds:\n",
    "            org_tlds[org].append(item['tld'].lower())\n",
    "        else:\n",
    "            org_tlds[org] = [item['tld'].lower()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "org_tlds_50 = [i for i in org_tlds if len(org_tlds[i]) >= 50]\n",
    "org_tlds_100 = [i for i in org_tlds if len(org_tlds[i]) >= 100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "donut_tlds = [i['tld'].lower() for i in tld_orgs if ' Donuts Inc.' in i['organisations']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "org_results = {}\n",
    "\n",
    "for org in org_tlds_50:\n",
    "    for tld in results:\n",
    "        if tld in org_tlds[org]:\n",
    "            v = {'tld': tld, 'rt': np.mean(results[tld])}\n",
    "\n",
    "            if org in org_results:\n",
    "                org_results[org].append(v)\n",
    "            else:\n",
    "                org_results[org] = [v]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(donut_tlds), len(results_donut)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bins = []\n",
    "\n",
    "for i in org_results:\n",
    "    dfxyz = pd.DataFrame(org_results[i])\n",
    "    bins.append(dfxyz.rt.nunique())\n",
    "    \n",
    "bins = min(bins)\n",
    "\n",
    "for i in org_results:\n",
    "    dfxyz = pd.DataFrame(org_results[i])\n",
    "    ax = dfxyz.hist(bins=160, range=(0,1400),cumulative=False)\n",
    "\n",
    "    for a in ax:\n",
    "        for b in a:\n",
    "            b.set_xlim(0,1400)\n",
    "            b.set_xlabel(\"Response time (ms)\")\n",
    "            b.set_ylabel(\"Number of TLDs\")\n",
    "            b.set_title('Average response times of TLDs organised by ' + str(i))\n",
    "            fig = b.get_figure()\n",
    "            fig.savefig(\"imgs/per_org_\" + str(i) + \".png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len([i['rt'] for i in org_results['Afilias']]), len(xdata['rt'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dftest = pd.DataFrame(xdata)\n",
    "dftest.plot.hist(stacked=True, bins=153, range=(0,1400),cumulative=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# [i for i in tld_orgs if 'Neustar, Inc.' in i['organisations']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}