diff options
Diffstat (limited to 'Performance.ipynb')
| -rw-r--r-- | Performance.ipynb | 1848 |
1 files changed, 1848 insertions, 0 deletions
diff --git a/Performance.ipynb b/Performance.ipynb new file mode 100644 index 0000000..6d08189 --- /dev/null +++ b/Performance.ipynb @@ -0,0 +1,1848 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "from dns.resolver import Resolver, NXDOMAIN, NoNameservers, Timeout, NoAnswer, query\n", + "from uuid import uuid4\n", + "from tqdm import tqdm\n", + "\n", + "import matplotlib as plt\n", + "import pandas as pd\n", + "import numpy as np\n", + "\n", + "import requests\n", + "import time\n", + "import json\n", + "import copy\n", + "import os\n", + "\n", + "%matplotlib inline" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "atlas_api_key = 'e057e19f-53ae-4b66-9b5e-c8bc00d7b4fe'\n", + "url_dns_measurements_create = 'https://atlas.ripe.net:443/api/v2/measurements/dns/'\n", + "url_dns_measurements_get = 'https://atlas.ripe.net:443/api/v2/measurements/dns/'\n", + "\n", + "newline = '\\n'\n", + "\n", + "figsize = (6, 4) #default\n", + "figsize = (15, 10)\n", + "\n", + "min_meas_id = 8759930\n", + "max_meas_id = 8770979\n", + "\n", + "headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def write_list(fn, data):\n", + " '''Writes a list to a file with each value on a new line'''\n", + " with open(fn, 'w') as f:\n", + " for datum in data:\n", + " f.write(datum + newline)\n", + " \n", + "def append_list(fn, data):\n", + " '''Appends a list to a file with each value on a new line'''\n", + " with open(fn, 'a') as f:\n", + " for datum in data:\n", + " f.write(datum + newline)\n", + " \n", + "def read_list(fn):\n", + " '''Reads a file and '''\n", + " with open(fn, 'r') as f:\n", + " return [line.strip(newline) for line in f]\n", + " \n", + "def write_json(fn, data):\n", + " with open(fn, 'w') as f:\n", + " f.write(json.dumps(data))\n", + " \n", + "def read_json(fn):\n", + " '''Read a json file (fn) and returns it as a dictionary'''\n", + " with open(fn, 'r') as f:\n", + " return json.loads(f.read())" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def write_data(fn, data):\n", + " \"\"\"Backs up the previous version of the data if it exists and writes the new data to a file.\"\"\"\n", + " # Backs up the previous data if it exists.\n", + " try:\n", + " write_json(\"data/backup/{}.json \".format(fn) + time.ctime().replace(' ', '-'), \n", + " read_json(\"data/{}.json\".format(fn)))\n", + " except:\n", + " pass\n", + "\n", + " write_json(\"data/{}.json\".format(fn), data)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "data = []\n", + "\n", + "with open('data/tlds', 'r') as f:\n", + " next(f)\n", + " \n", + " for line in f:\n", + " data.append({'tld': line[:-1].lower()})" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# Test tld set\n", + "# data = [{'tld': 'nl'}, {'tld': 'audi'}]" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def find_nxdomain(tld, max_tries = 3):\n", + " for _ in range(max_tries):\n", + " domain = '{}.{}'.format(str(uuid4()), tld)\n", + " \n", + " try:\n", + " query(domain)\n", + " except:\n", + " return domain\n", + " \n", + " return None\n", + "\n", + "def find_nxdomain_wildcard(tld, max_tries = 3):\n", + " for _ in range(max_tries):\n", + " domain = '{}.{}'.format(str(uuid4()), tld)\n", + "\n", + " response = !dig soa +noall +authority +noidn {domain}\n", + "\n", + " if response[0].startswith(tld):\n", + " return domain\n", + " \n", + " return None" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true, + "scrolled": true + }, + "outputs": [], + "source": [ + "for datum in data:\n", + " domain = str(uuid4()) + '.' + datum['tld']\n", + " \n", + " try:\n", + " query(domain)\n", + " print(datum['tld'], 'DOMAIN EXISTS')\n", + " except NXDOMAIN:\n", + " datum['domain'] = domain\n", + " except NoNameservers:\n", + " print(datum['tld'], 'NO NAMESERVERS')\n", + " except Timeout:\n", + " print(datum['tld'], 'TIME OUT')\n", + " except NoAnswer:\n", + " print(datum['tld'], 'NO ANSWER')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true, + "scrolled": true + }, + "outputs": [], + "source": [ + "for datum in data:\n", + " if not 'domain' in datum:\n", + " domain = str(uuid4()) + '.' + datum['tld']\n", + "\n", + " try:\n", + " query(domain)\n", + " print(datum['tld'], 'DOMAIN EXISTS')\n", + " except NXDOMAIN:\n", + " datum['domain'] = domain\n", + " except NoNameservers:\n", + " print(datum['tld'], 'NO NAMESERVERS')\n", + " except Timeout:\n", + " print(datum['tld'], 'TIME OUT')\n", + " except NoAnswer:\n", + " print(datum['tld'], 'NO ANSWER')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true, + "scrolled": true + }, + "outputs": [], + "source": [ + "# wildcard check\n", + "for item in data:\n", + " if len(item) == 1:\n", + " domain = str(uuid4()) + '.' + item['tld']\n", + " print(domain)\n", + " \n", + " bashCommand = \"dig soa +noall +authority \" + domain\n", + " process = subprocess.Popen(bashCommand, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)\n", + " out = str(process.stdout.read())\n", + " \n", + " print(out)\n", + " \n", + " if out.startswith(\"b'\" + item['tld']):\n", + " item['domain'] = domain" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "data_2 = data.copy()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# data = [{'tld': 'nl'}]\n", + "for datum in data:\n", + " if not 'domain' in datum:\n", + " print(datum)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "df = pd.DataFrame(data)\n", + "df.loc[df.domain.isnull()].tld" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "payload = {\n", + " \"bill_to\": \"bickerkards@gmail.com\",\n", + " \"is_oneoff\": True,\n", + " \"definitions\": [],\n", + " \"probes\": []\n", + "}\n", + " \n", + "definition = {\n", + " \"af\":4,\n", + " \"query_class\":\"IN\",\n", + " \"query_type\":\"A\",\n", + " \"query_argument\": \"nlnetlabs.nl\",\n", + " \"description\":\"Test getting probes\",\n", + " \"use_probe_resolver\":True,\n", + " \"resolve_on_probe\":False,\n", + " \"set_nsid_bit\":True,\n", + " \"protocol\":\"UDP\",\n", + " \"udp_payload_size\":512,\n", + " \"retry\":0,\n", + " \"skip_dns_check\":False,\n", + " \"include_qbuf\":False,\n", + " \"include_abuf\":True,\n", + " \"prepend_probe_id\":False,\n", + " \"set_rd_bit\":False,\n", + " \"set_do_bit\":False,\n", + " \"set_cd_bit\":False,\n", + " \"type\":\"dns\",\n", + " \"is_public\":True\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "probe_ids = [10262, 10287, 11040, 11429, 12515, 12873, 12956, 13623, 13728, 13769, 13788, 13799, 13804, 13805, 13810, 14237, 26057, 14564, 15156, 14691, 15594, 15799, 4205, 18131, 18195, 18691, 19326, 19740, 20111, 20353, 20493, 20531, 20621, 21003, 21035, 21122, 21251, 21345, 21703, 22286, 22695, 23031, 23085, 28240, 27972, 23697, 24807, 25011, 25148, 25323, 26936, 26378, 26627, 4155, 26823, 28355, 30676, 4829, 29006, 29183, 29405, 30225, 30324, 31201, 19306, 19634, 6025, 11660, 22388, 25182, 4123, 3812, 20923, 14384, 12389]\n", + "\n", + "probes = [\n", + " {\n", + " \"value\": str(probe_ids)[1:-1],\n", + " \"type\": \"probes\",\n", + " \"requested\": len(probe_ids)\n", + " }\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "payloads = []\n", + "step_size = 50\n", + " \n", + "for i in range(0, len(data), step_size):\n", + " defintions = []\n", + " \n", + " for datum in data[i:i + step_size]:\n", + " definition_caching = definition.copy()\n", + " definition_caching['query_type'] = \"NS\"\n", + " definition_caching['query_argument'] = datum['tld']\n", + " definition_caching['description'] = \"caching \" + datum['tld']\n", + " defintions.append(definition_caching)\n", + "\n", + " definition_measuring = definition.copy()\n", + " definition_measuring['query_type'] = \"SOA\"\n", + " definition_measuring['query_argument'] = datum['domain']\n", + " definition_measuring['description'] = \"measuring \" + datum['tld']\n", + " defintions.append(definition_measuring) \n", + "\n", + " new_payload = payload.copy()\n", + " new_payload['probes'] = probes\n", + " new_payload['definitions'] = defintions\n", + "\n", + " payloads.append(new_payload)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "measurement_ids = []\n", + "measurement_responses = []\n", + "url = url_dns_measurements_create + '?key=' + atlas_api_key" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "with open('data/payloads', 'r') as f:\n", + " payloads = json.loads(f.read())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# measurement_ids.append(requests.post(url, data = json.dumps(payloads[0]), headers = headers))\n", + "# [x['description'] for x in payloads[1]['definitions']]\n", + "# request.status_code\n", + "# measurement_ids\n", + "# len(payloads)\n", + "# request.json()\n", + "payloads[23]['definitions'][-1]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true, + "scrolled": true, + "slideshow": { + "slide_type": "-" + } + }, + "outputs": [], + "source": [ + "# split into 3\n", + "for payload in payloads[23:]:\n", + " \n", + " request = requests.post(url, data = json.dumps(payload), headers = headers)\n", + " print(request.status_code)\n", + " \n", + " while request.status_code == 400:\n", + " print(request.json())\n", + " request = requests.post(url, data = json.dumps(payload), headers = headers)\n", + " time.sleep(300)\n", + " print(request.status_code)\n", + " \n", + " measurement_ids += measurement_ids + request.json()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "for id in measurement_ids:\n", + " print(requests.get(url_dns_measurements_get + '?id__in=' + str(id)).json())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# with open('data/measurement_ids', 'w') as f:\n", + "# f.write(json.dumps(measurement_ids))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "temp_url = url_dns_measurements_get + '?id__lte=' + str(max(measurement_ids)) + '&id__gte=' + str(min(measurement_ids)) + '&description__startswith=measuring&mine=true'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "temp_url = url_dns_measurements_get + '?id__lte=' + str(max_meas_id) + '&id__gte=' + str(min_meas_id) + '&description__startswith=measuring&mine=true'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# # measurements = [requests.get(url_dns_measurements_get + '?id__in=' + str(id)).json() for id in measurement_ids]\n", + "# measurements = []\n", + "# pbar = tqdm(total=len(measurement_ids))\n", + "\n", + "# for id in measurement_ids:\n", + "# measurements.append(requests.get(url_dns_measurements_get + '?id__in=' + str(id)).json()) \n", + " \n", + "# pbar.update(1)\n", + "# pbar.close()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# xyz = probe_ids.copy()\n", + "\n", + "# for probe_idsd in [r['prb_id'] for r in request]:\n", + "# xyz.remove(probe_id)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "tlds = []\n", + "\n", + "with open('data/tlds') as f:\n", + "# next(f)\n", + " \n", + " for line in f:\n", + " tlds.append(line.strip('\\n').lower())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "tld_timeouts" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true, + "scrolled": true + }, + "outputs": [], + "source": [ + "results = {}\n", + "xprobes = {}\n", + "\n", + "next = True\n", + "measurements = requests.get(temp_url).json()\n", + "\n", + "while next:\n", + " for result in measurements['results']:\n", + " measurement_type, tld = result['description'].split()\n", + "\n", + " if measurement_type == 'measuring':\n", + " request = requests.get(result['result']).json()\n", + "\n", + " for probe in request:\n", + " for result2 in probe['resultset']:\n", + " if 'result' in result2:\n", + " if tld in results:\n", + " results[tld].append(result2['result']['rt'])\n", + " else:\n", + " results[tld] = [result2['result']['rt']]\n", + " else:\n", + " print(tld, probe['prb_id'], result2['error'])\n", + " \n", + " if tld in tld_timeouts:\n", + " tld_timeouts[tld] += 1\n", + " else:\n", + " tld_timeouts[tld] = 1\n", + "\n", + " if probe['prb_id'] in xprobes:\n", + " xprobes[probe['prb_id']] += 1\n", + " else:\n", + " xprobes[probe['prb_id']] = 1\n", + " \n", + " if measurements['next']:\n", + " print('\\n' + measurements['next'].split('=')[-1] + '\\n')\n", + " measurements = requests.get(measurements['next']).json() \n", + " else:\n", + " next = False\n", + " print('Done')" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "!mkdir data/atlas\n", + "!mkdir data/atlas/ns\n", + "!mkdir data/atlas/soa" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "temp_url = url_dns_measurements_get + '?id__lte=' + str(max_meas_id) + '&id__gte=' + str(min_meas_id) + '&mine=true'" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n", + " 0%| | 0/1531 [00:00<?, ?it/s]\u001b[A\n", + " 0%| | 1/1531 [00:00<05:10, 4.92it/s]\u001b[A\n", + " 0%| | 2/1531 [00:00<05:05, 5.00it/s]\u001b[A\n", + " 0%| | 3/1531 [00:00<04:56, 5.15it/s]\u001b[A\n", + " 0%| | 4/1531 [00:00<04:41, 5.42it/s]\u001b[A\n", + " 0%| | 5/1531 [00:03<27:53, 1.10s/it]\u001b[A\n", + " 0%| | 6/1531 [00:04<21:40, 1.17it/s]\u001b[A\n", + " 0%| | 7/1531 [00:04<19:31, 1.30it/s]\u001b[A\n", + " 1%| | 8/1531 [00:05<16:34, 1.53it/s]\u001b[A\n", + " 1%| | 9/1531 [00:05<13:47, 1.84it/s]\u001b[A\n", + " 1%| | 10/1531 [00:05<11:49, 2.14it/s]\u001b[A\n", + " 1%| | 11/1531 [00:05<09:56, 2.55it/s]\u001b[A\n", + " 1%| | 12/1531 [00:06<08:49, 2.87it/s]\u001b[A\n", + " 1%| | 13/1531 [00:06<07:36, 3.33it/s]\u001b[A\n", + " 1%| | 14/1531 [00:06<07:00, 3.60it/s]\u001b[A\n", + " 1%| | 15/1531 [00:06<06:32, 3.86it/s]\u001b[A\n", + " 1%| | 16/1531 [00:07<05:58, 4.23it/s]\u001b[A\n", + " 1%| | 17/1531 [00:07<05:58, 4.23it/s]\u001b[A\n", + " 1%| | 18/1531 [00:07<05:45, 4.38it/s]\u001b[A\n", + " 1%| | 19/1531 [00:07<05:42, 4.42it/s]\u001b[A\n", + " 7%|▋ | 100/1531 [00:24<04:47, 4.98it/s]\n" + ] + }, + { + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m/usr/lib/python3.6/site-packages/urllib3/connectionpool.py\u001b[0m in \u001b[0;36m_make_request\u001b[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[1;32m 378\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;31m# Python 2.7, use buffering of HTTP responses\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 379\u001b[0;31m \u001b[0mhttplib_response\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetresponse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbuffering\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 380\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;31m# Python 2.6 and older, Python 3\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mTypeError\u001b[0m: getresponse() got an unexpected keyword argument 'buffering'", + "\nDuring handling of the above exception, another exception occurred:\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m<ipython-input-35-f1320228216d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 8\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'description'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 9\u001b[0m \u001b[0mmeasurement_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtld\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'description'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m \u001b[0mrequest\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrequests\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'result'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjson\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 11\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 12\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mmeasurement_type\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'measuring'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/lib/python3.6/site-packages/requests/api.py\u001b[0m in \u001b[0;36mget\u001b[0;34m(url, params, **kwargs)\u001b[0m\n\u001b[1;32m 68\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 69\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msetdefault\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'allow_redirects'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 70\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mrequest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'get'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 71\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 72\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/lib/python3.6/site-packages/requests/api.py\u001b[0m in \u001b[0;36mrequest\u001b[0;34m(method, url, **kwargs)\u001b[0m\n\u001b[1;32m 54\u001b[0m \u001b[0;31m# cases, and look like a memory leak in others.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 55\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0msessions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSession\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 56\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmethod\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmethod\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 57\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 58\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/lib/python3.6/site-packages/requests/sessions.py\u001b[0m in \u001b[0;36mrequest\u001b[0;34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[0m\n\u001b[1;32m 486\u001b[0m }\n\u001b[1;32m 487\u001b[0m \u001b[0msend_kwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msettings\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 488\u001b[0;31m \u001b[0mresp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprep\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0msend_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 489\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 490\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mresp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/lib/python3.6/site-packages/requests/sessions.py\u001b[0m in \u001b[0;36msend\u001b[0;34m(self, request, **kwargs)\u001b[0m\n\u001b[1;32m 607\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 608\u001b[0m \u001b[0;31m# Send the request\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 609\u001b[0;31m \u001b[0mr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0madapter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 610\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 611\u001b[0m \u001b[0;31m# Total elapsed time of the request (approximately)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/lib/python3.6/site-packages/requests/adapters.py\u001b[0m in \u001b[0;36msend\u001b[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[1;32m 421\u001b[0m \u001b[0mdecode_content\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 422\u001b[0m \u001b[0mretries\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_retries\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 423\u001b[0;31m \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 424\u001b[0m )\n\u001b[1;32m 425\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/lib/python3.6/site-packages/urllib3/connectionpool.py\u001b[0m in \u001b[0;36murlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001b[0m\n\u001b[1;32m 598\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout_obj\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 599\u001b[0m \u001b[0mbody\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbody\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mheaders\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mheaders\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 600\u001b[0;31m chunked=chunked)\n\u001b[0m\u001b[1;32m 601\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 602\u001b[0m \u001b[0;31m# If we're going to release the connection in ``finally:``, then\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/lib/python3.6/site-packages/urllib3/connectionpool.py\u001b[0m in \u001b[0;36m_make_request\u001b[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[1;32m 380\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;31m# Python 2.6 and older, Python 3\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 381\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 382\u001b[0;31m \u001b[0mhttplib_response\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetresponse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 383\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 384\u001b[0m \u001b[0;31m# Remove the TypeError from the exception chain in Python 3;\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/lib/python3.6/http/client.py\u001b[0m in \u001b[0;36mgetresponse\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1329\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1330\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1331\u001b[0;31m \u001b[0mresponse\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbegin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1332\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mConnectionError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1333\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/lib/python3.6/http/client.py\u001b[0m in \u001b[0;36mbegin\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 295\u001b[0m \u001b[0;31m# read until we get a non-100 response\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 296\u001b[0m \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 297\u001b[0;31m \u001b[0mversion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstatus\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreason\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_read_status\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 298\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mstatus\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mCONTINUE\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 299\u001b[0m \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/lib/python3.6/http/client.py\u001b[0m in \u001b[0;36m_read_status\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 256\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 257\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_read_status\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 258\u001b[0;31m \u001b[0mline\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadline\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_MAXLINE\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"iso-8859-1\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 259\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mline\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0m_MAXLINE\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 260\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mLineTooLong\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"status line\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/lib/python3.6/socket.py\u001b[0m in \u001b[0;36mreadinto\u001b[0;34m(self, b)\u001b[0m\n\u001b[1;32m 584\u001b[0m \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 585\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 586\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecv_into\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 587\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 588\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_timeout_occurred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/lib/python3.6/ssl.py\u001b[0m in \u001b[0;36mrecv_into\u001b[0;34m(self, buffer, nbytes, flags)\u001b[0m\n\u001b[1;32m 1000\u001b[0m \u001b[0;34m\"non-zero flags not allowed in calls to recv_into() on %s\"\u001b[0m \u001b[0;34m%\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1001\u001b[0m self.__class__)\n\u001b[0;32m-> 1002\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnbytes\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbuffer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1003\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1004\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0msocket\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecv_into\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbuffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnbytes\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mflags\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/lib/python3.6/ssl.py\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, len, buffer)\u001b[0m\n\u001b[1;32m 863\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Read on closed or unwrapped SSL socket.\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 864\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 865\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sslobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbuffer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 866\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mSSLError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 867\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mSSL_ERROR_EOF\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msuppress_ragged_eofs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/lib/python3.6/ssl.py\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, len, buffer)\u001b[0m\n\u001b[1;32m 623\u001b[0m \"\"\"\n\u001b[1;32m 624\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mbuffer\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 625\u001b[0;31m \u001b[0mv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sslobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbuffer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 626\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 627\u001b[0m \u001b[0mv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sslobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ] + } + ], + "source": [ + "next_result = True\n", + "measurements = requests.get(temp_url).json()\n", + "pbar = tqdm(total=1531)\n", + "# print('1')\n", + "\n", + "while next_result:\n", + " for result in measurements['results']:\n", + " if len(result['description'].split()) == 2:\n", + " measurement_type, tld = result['description'].split()\n", + " request = requests.get(result['result']).json()\n", + "\n", + " if measurement_type == 'measuring':\n", + " with open('data/atlas/soa/{}.json'.format(tld.upper()), 'w') as f:\n", + " temp = copy.deepcopy(result)\n", + " temp['result'] = request\n", + "\n", + " f.write(json.dumps(temp))\n", + "\n", + " elif measurement_type == 'caching':\n", + " with open('data/atlas/ns/{}.json'.format(tld.upper()), 'w') as f:\n", + " temp = copy.deepcopy(result)\n", + " temp['result'] = request\n", + "\n", + " f.write(json.dumps(temp))\n", + "\n", + " pbar.update(1)\n", + " pbar.close()\n", + " \n", + " if measurements['next']:\n", + "# print(measurements['next'].split('=')[-1])\n", + " measurements = requests.get(measurements['next']).json() \n", + " else:\n", + " next_result = False\n", + " print('Done')" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "indir = 'data/atlas/soa/'\n", + "\n", + "data_perf = []\n", + "\n", + "for root, dirs, filenames in os.walk(indir):\n", + " for f in filenames:\n", + "# tld, _ = f.split('.')\n", + " tld = f\n", + " \n", + " datum = {'tld': tld, 'rt': [], 'timeouts': 0}\n", + " \n", + " with open(indir + f, 'r') as f:\n", + " tld_results = json.loads(f.read())\n", + " \n", + " for probe in tld_results['result']:\n", + " for result in probe['resultset']:\n", + " if 'result' in result:\n", + " datum['rt'].append(result['result']['rt']) \n", + " elif 'error' in result and 'timeout' in result['error']:\n", + " datum['timeouts'] += 1\n", + " \n", + " datum['rt'] = np.mean(datum['rt'])\n", + " data_perf.append(datum)" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [], + "source": [ + "write_data('data_perf', data_perf)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# df2.sort_values('ntimeouts', ascending=False).head()\n", + "# df2.ntimeouts.hist(bins=8, align='right')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df2 = pd.DataFrame(list(tld_timeouts.items()), columns=['tld', 'ntimeouts'])\n", + "ax = df2.hist(bins=8, align='left', color='grey')\n", + "\n", + "for a in ax:\n", + " for b in a:\n", + " b.set_xlabel(\"Number of timeouts\")\n", + " b.set_ylabel(\"Number of TLDs\")\n", + " b.set_title('')\n", + " b.set_yscale('log')\n", + " b.set_facecolor('lightgrey')\n", + " fig = b.get_figure()\n", + " fig.savefig(\"imgs/per_timeouts.pdf\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "[x for x in tld_timeouts if tld_timeouts[x] > 7]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "tld_timeouts['xn--ygbi2ammx']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "c = 0\n", + "for tld in tld_timeouts:\n", + " if tld_timeouts[tld] > 0:\n", + " c+=1\n", + "c / len(tld_timeouts)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "df2.loc[df2.ntimeouts > 5].sort_values('ntimeouts', ascending=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "tld_timeouts['fk']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "xdata = {'tld': [tld for tld in results], 'rt': [np.mean(results[tld]) for tld in results]}\n", + "df = pd.DataFrame(xdata, columns = ['tld', 'rt'])\n", + "df.index = df['tld']\n", + "del df['tld']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "edata = {'tld': [tld for tld in results], 'rt': [np.std(results[tld]) for tld in results]}\n", + "dfe = pd.DataFrame(edata, columns = ['tld', 'rt'])\n", + "dfe.index = dfe['tld']\n", + "del dfe['tld']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ax = df.hist(bins=160, range=(0,1400))\n", + "\n", + "for a in ax:\n", + " for b in a:\n", + " b.set_xlim(0,1400)\n", + " b.set_xlabel(\"Response time (ms)\")\n", + " b.set_ylabel(\"Number of TLDs\")\n", + " b.set_title('')\n", + " fig = b.get_figure()\n", + " fig.savefig(\"imgs/per.pdf\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "df.loc[df.rt > 600]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "caching_ids = [response.json() for response in chaching_ids]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "chaching_ids2 = []\n", + "\n", + "for i in range(202,1531,200):\n", + " print(i)\n", + " for payload in payloads_caching[i:i + 200]:\n", + " url = url_dns_measurements_create + '?key=' + atlas_api_key\n", + " chaching_ids2.append(requests.post(url, data = json.dumps(payload), headers = headers))\n", + " \n", + " time.sleep(300)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# tmp_res = []\n", + "\n", + "for item in chaching_ids2:\n", + " if isinstance(item.json(), list):\n", + " tmp_res.append((item.json()))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "tmp_res2 = []\n", + "\n", + "for i in tmp_res:\n", + " for j in i:\n", + " tmp_res2.append(j)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "rs = []\n", + "\n", + "for id in tmp_res2:\n", + " rs.append(requests.get(url_dns_measurements_get + 'id__in=' + str(id)))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "chaching_ids2[0].json()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "tlds2 = tlds.copy()\n", + "\n", + "for tld in deltlds:\n", + " tlds2.remove(tld)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# measuring\n", + "for item in data:\n", + " item['definitions']['query_type'] = \"SOA\"\n", + " item['definitions']['query_argument'] = item['domain']\n", + " item['definitions']['description'] = item['tld'] + ' measurement'\n", + " \n", + "url = url_dns_measurements_create + '?key=' + atlas_api_key\n", + "measurement_ids = requests.post(url, data = json.dumps(payload), headers = headers)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "for id in measurement ids.json():\n", + " r = requests.get(url_dns_measurements_get + 'id__in=' + id)\n", + " # s = requests.get(r.json()['result'])" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "id 44710\n", + "opcode QUERY\n", + "rcode NXDOMAIN\n", + "flags QR RD RA\n", + "edns 0\n", + "payload 4000\n", + ";QUESTION\n", + "747032ef-ce8e-4f4c-99f9-e433073738dd.de. IN A\n", + ";ANSWER\n", + ";AUTHORITY\n", + "de. 5400 IN SOA f.nic.de. its.denic.de. 2017050961 7200 7200 3600000 7200\n", + ";ADDITIONAL\n" + ] + } + ], + "source": [ + "import base64\n", + "import dns.message\n", + "\n", + "x = dns.message.from_wire(base64.b64decode('rqaBgwABAAAAAQABJDc0NzAzMmVmLWNlOGUtNGY0Yy05OWY5LWU0MzMwNzM3MzhkZAJkZQAAAQABwDEABgABAAAVGAAoAWYDbmljwDEDaXRzBWRlbmljwDF4OcFRAAAcIAAAHCAANu6AAAAcIAAAKQ+gAAAAAAAA'))\n", + "print(x)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# help(x)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "for i in range(0, len(data), 50):\n", + " print(data[i]['tld'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "definition" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "probes" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def find(lst, key, value):\n", + " for i, dic in enumerate(lst):\n", + " if dic[key] == value:\n", + " return i\n", + " return -1" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "with open('data/tld_type', 'r') as f:\n", + " data_type = json.loads(f.read())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "with open('data/tld_age', 'r') as f:\n", + " data_age = json.loads(f.read())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "data_rt = [{'tld': tld, 'rt': np.mean(results[tld])} for tld in results]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "for datum in data_rt:\n", + " ix = find(data_type, 'tld', datum['tld'])\n", + " datum['type'] = data_type[ix]['type']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "find(data_type, 'tld', 'aaa')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "dft = pd.DataFrame(data_rt, columns = ['rt', 'tld', 'type'])\n", + "dft.index = dft['tld']\n", + "del dft['tld']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "dft.loc[dft.type == 'country-code'].mean().rt, dft.loc[dft.type == 'generic'].mean().rt" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "dfa.loc[dfa.age == 'new'].mean().rt, dfa.loc[dfa.age == 'old'].mean().rt" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df.mean().rt" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "nbins = 160" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ax = dft.loc[dft.type == 'country-code'].hist('rt', \n", + " bins=nbins, \n", + " range=(0,1400),\n", + " cumulative=False,\n", + " align='mid',\n", + " figsize=(6,4))\n", + "\n", + "for a in ax:\n", + " for b in a:\n", + " b.set_xlim(0,1400)\n", + " b.set_xlabel(\"Response time (ms)\")\n", + " b.set_ylabel(\"Number of TLDs\")\n", + " b.set_title('')\n", + " fig = b.get_figure()\n", + " fig.savefig(\"imgs/per_cctlds.pdf\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ax = dft.loc[dft.type == 'generic'].hist('rt', bins=nbins, range=(0,1400),cumulative=False,figsize=(6,4))\n", + "\n", + "for a in ax:\n", + " for b in a:\n", + " b.set_xlim(0,1400)\n", + " b.set_xlabel(\"Response time (ms)\")\n", + " b.set_ylabel(\"Number of TLDs\")\n", + " b.set_title('')\n", + " fig = b.get_figure()\n", + " fig.savefig(\"imgs/per_gtlds.pdf\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "data_age[0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for datum in data_age:\n", + " print(datum['tld'])\n", + " ix = find(data_rt, 'tld', datum['tld'].lower())\n", + " datum['rt'] = data_rt[ix]['rt']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "dfa = pd.DataFrame(data_age, columns = ['age', 'rt', 'tld'])\n", + "dfa.index = dfa['tld']\n", + "del dfa['tld']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ax = dfa.loc[dfa.age == 'new'].hist('rt', \n", + " bins=nbins, \n", + " range=(0,1400),\n", + " cumulative=False)\n", + "\n", + "for a in ax:\n", + " for b in a:\n", + " b.set_xlim(0,1400)\n", + " b.set_xlabel(\"Response time (ms)\")\n", + " b.set_ylabel(\"Number of TLDs\")\n", + " b.set_title('')\n", + " fig = b.get_figure()\n", + " fig.savefig(\"imgs/per_new.pdf\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ax = dfa.loc[dfa.age == 'old'].hist('rt', \n", + " bins=nbins, \n", + " range=(0,1400),\n", + " cumulative=False)\n", + "\n", + "for a in ax:\n", + " for b in a:\n", + " b.set_xlim(0,1400)\n", + " b.set_xlabel(\"Response time (ms)\")\n", + " b.set_ylabel(\"Number of TLDs\")\n", + " b.set_title('')\n", + " fig = b.get_figure()\n", + " fig.savefig(\"imgs/per_old.pdf\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "indir = 'data/ripe/soa/'\n", + "prb_rt = {}\n", + "\n", + "for root, dirs, filenames in os.walk(indir):\n", + " for f in filenames:\n", + " tld, _ = f.split('.')\n", + " \n", + " with open(indir + f, 'r') as f:\n", + " tld_results = json.loads(f.read())\n", + " \n", + " for probe in tld_results['result']:\n", + " prb_id = probe['prb_id']\n", + " \n", + " for result in probe['resultset']:\n", + " if 'result' in result:\n", + " if prb_id in prb_rt:\n", + " prb_rt[prb_id].append(result['result']['rt'])\n", + " else:\n", + " prb_rt[prb_id] = [result['result']['rt']]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "prb_data = [{'probe': probe, 'rt': np.mean(prb_rt[probe])} for probe in prb_rt]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "df = pd.DataFrame(prb_data, columns = ['probe', 'rt'])\n", + "df.index = df['probe']\n", + "del df['probe']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "df.plot.bar(figsize=(18,9))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "r = requests.get('https://atlas.ripe.net/api/v2/probes/?id__in=1' + str([probe for probe in prb_rt])[1:-1])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "probe_cc = {}\n", + "\n", + "for v in r.json()['results']:\n", + " probe_cc[v['id']] = v['country_code']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "xyz = []\n", + "\n", + "for datum in prb_data:\n", + " if datum['probe'] in probe_cc:\n", + " xyz.append({'cc': probe_cc[datum['probe']], 'rt': datum['rt']})\n", + "# datum['probe'] = probe_cc[datum['probe']]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "with open('data/cc_rt', 'r') as f:\n", + " xyz = json.loads(f.read())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "xyz2 = {}\n", + "\n", + "for x in xyz:\n", + " if x['cc'] in xyz2:\n", + " xyz2[x['cc']][0] = x['rt'] + xyz2[x['cc']][0] / 2\n", + " xyz2[x['cc']][1] += 1\n", + " else: \n", + " xyz2[x['cc']] = [x['rt'], 1]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "xyz = [{'cc': x + ' (' + str(xyz2[x][1]) + ')', 'rt': xyz2[x][0]} for x in xyz2]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "dfcc = pd.DataFrame(xyz, columns = ['cc', 'rt'])\n", + "dfcc.index = dfcc['cc']\n", + "del dfcc['cc']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# dfcc" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "dfcc.sort_values('rt', ascending=False).plot.bar(figsize=(12,7))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "f = read_list('data/dig/tld_nss')\n", + "data = {}\n", + "\n", + "for line in f:\n", + " tld, _, _, _, ns = line.split()\n", + " tld = tld.strip('.')\n", + "\n", + " if tld != '.':\n", + " if tld in data:\n", + " data[tld] += 1\n", + " else:\n", + " data[tld] = 1" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "ns_data = {}\n", + "\n", + "for i in range(0,16):\n", + " ns_data[i] = []" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for tld in results:\n", + " try:\n", + " ns_data[data[tld]].append({'tld': tld, 'rt': np.mean(results[tld])})\n", + " except:\n", + " print(tld)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for i in range(15,16):\n", + " if ns_data[i] == []:\n", + " del ns_data[i]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# bins = []\n", + "\n", + "# for i in ns_data:\n", + "# dfxyz = pd.DataFrame(ns_data[i])\n", + "# bins.append(dfxyz.rt.nunique())\n", + " \n", + "# bins = min(bins)\n", + "\n", + "for i in ns_data:\n", + " dfxyz = pd.DataFrame(ns_data[i])\n", + " ax = dfxyz.hist('rt', bins=32, range=(0,1400))\n", + "\n", + " for a in ax:\n", + " for b in a:\n", + " b.set_xlim(0,1400)\n", + " b.set_xlabel(\"Response time (ms)\")\n", + " b.set_ylabel(\"Number of TLDs\")\n", + " b.set_title('Average response times of TLDs with ' + str(i) + ' name servers')\n", + " fig = b.get_figure()\n", + " fig.savefig(\"imgs/per_ns_\" + str(i) + \".png\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "tld_orgs = []\n", + "import os\n", + "indir = 'data/whois/'\n", + "\n", + "for root, dirs, filenames in os.walk(indir):\n", + " tld_orgs = [{'tld': tld, 'organisations': []} for tld in filenames]\n", + " \n", + " for fn in filenames:\n", + " with open(indir + fn, 'r') as f:\n", + " for line in f:\n", + " if line.startswith('organisation'):\n", + " _, org = line.split('rganisation: ')\n", + " i = find(tld_orgs, 'tld', fn)\n", + " tld_orgs[i]['organisations'].append(org.strip('\\n'))\n", + " \n", + "# tld_creation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "org_tlds = {}\n", + "\n", + "for item in tld_orgs:\n", + " for org in item['organisations']:\n", + " if org in org_tlds:\n", + " org_tlds[org].append(item['tld'].lower())\n", + " else:\n", + " org_tlds[org] = [item['tld'].lower()]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "org_tlds_50 = [i for i in org_tlds if len(org_tlds[i]) >= 50]\n", + "org_tlds_100 = [i for i in org_tlds if len(org_tlds[i]) >= 100]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "donut_tlds = [i['tld'].lower() for i in tld_orgs if ' Donuts Inc.' in i['organisations']]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "org_results = {}\n", + "\n", + "for org in org_tlds_50:\n", + " for tld in results:\n", + " if tld in org_tlds[org]:\n", + " v = {'tld': tld, 'rt': np.mean(results[tld])}\n", + "\n", + " if org in org_results:\n", + " org_results[org].append(v)\n", + " else:\n", + " org_results[org] = [v]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "len(donut_tlds), len(results_donut)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "bins = []\n", + "\n", + "for i in org_results:\n", + " dfxyz = pd.DataFrame(org_results[i])\n", + " bins.append(dfxyz.rt.nunique())\n", + " \n", + "bins = min(bins)\n", + "\n", + "for i in org_results:\n", + " dfxyz = pd.DataFrame(org_results[i])\n", + " ax = dfxyz.hist(bins=160, range=(0,1400),cumulative=False)\n", + "\n", + " for a in ax:\n", + " for b in a:\n", + " b.set_xlim(0,1400)\n", + " b.set_xlabel(\"Response time (ms)\")\n", + " b.set_ylabel(\"Number of TLDs\")\n", + " b.set_title('Average response times of TLDs organised by ' + str(i))\n", + " fig = b.get_figure()\n", + " fig.savefig(\"imgs/per_org_\" + str(i) + \".png\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "len([i['rt'] for i in org_results['Afilias']]), len(xdata['rt'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "dftest = pd.DataFrame(xdata)\n", + "dftest.plot.hist(stacked=True, bins=153, range=(0,1400),cumulative=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# [i for i in tld_orgs if 'Neustar, Inc.' in i['organisations']]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.1" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} |