blob: d6b3a98dd71895dfdd8062307e8b6b615b7f9e9d (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
|
# IP mapping
pbar = tqdm(total=len(data))
for item in data.copy():
try:
ip = socket.getaddrinfo(item['host'], None, socket.AF_INET)[-1][-1][0]
except Exception as e:
ip = None
if ip and 'ip' in item:
new_item = item.copy()
new_item['ip'] = ip
data.append(new_item)
elif ip:
item['ip'] = ip
try:
ip = socket.getaddrinfo(item['host'], None, socket.AF_INET6)[-1][-1][0]
except Exception as e:
ip = None
if ip and'ip' in item:
new_item = item.copy()
new_item['ip'] = ip
data.append(new_item)
elif ip:
item['ip'] = ip
if not 'ip' in item:
item['ip'] = None
pbar.update(1)
pbar.close()
# html parser
url = "https://www.iana.org/domains/root/db/"
html = urlopen(url)
soup = BeautifulSoup(html, 'html5lib')
data_tld_orgs = []
for item in soup.find_all(attrs={'class': 'iana-table'}):
for table in soup.find_all(attrs={'class': 'iana-table'}):
values = [td.get_text(strip=True) for td in table.find_all('td')]
for i in range(0, len(values), 3):
data_tld_orgs.append({'tld': values[i].strip('.'), 'type': values[i + 1], 'organisation': values[i + 2]})
# Root zone parser
data = []
with open('data/root.zone') as f:
# Skip header and root servers
for i in range(21):
next(f)
for line in f:
values = line.split()
if values[3] == 'NS':
tld = values[0][:-1]
ns = values[-1][:-1]
data.append({'host': ns, 'tld': tld})
|
