之前博文：
【说明】
数据采集使用Nornir进行批量并发执行，将采集到的数据写入 网络拓扑可视化 之二 建立的数据库中。
注意，本文介绍的采集都是通过ssh连接设备执行命令，用正则表达式取匹配信息，在生产环境中建议使用netconf取采集数据比较好，获取的是结构化信息，好处理。
一、ARP信息采集代码：
# arp_to_db.py
import os
import django
os.environ['DJANGO_SETTINGS_MODULE'] = 'topology.settings'
django.setup()
from nornir import InitNornir
from nornir_napalm.plugins.tasks import napalm_get
from arp.models import Device, ArpTable
def get_host_data(task):
"""获取设备信息的函数"""
task.run(
task=napalm_get, # 使用napalm来获取数据
getters=['facts', 'get_arp_table'] # 实际上就是执行napalm的get_facts()和get_arp_table()这2个方法
)
def update_arp_db(nornir_arpAndfacts_result, nornir):
"""将arp信息写入数据库的函数"""
for host in nornir_arpAndfacts_result:
result = nornir_arpAndfacts_result[host]
arp_table = result[1].result['get_arp_table']
ip = nornir.inventory.hosts[host].get('hostname', 'n/a')
device, create = Device.objects.update_or_create(
defaults={
'hostname': result[1].result['facts']['hostname'],
'serial_number': result[1].result['facts']['serial_number'],
'vendor': result[1].result['facts']['vendor'],
'model': result[1].result['facts']['model'],
},
ip=ip)
for arp in arp_table:
ip = arp['ip']
interface = arp['interface']
mac = arp['mac']
ArpTable.objects.update_or_create(
defaults={
'macaddress': mac,
'interface': interface,
'device': device
},
ipaddress=ip) # 如果是交换机接口ip对应的arp，那么需要ipaddress+interface作为不变的条件，这次我们只采集与服务器直连的交换机
return True
if __name__ == '__main__':
NORNIR_CONFIG_FILE = "nornir_config.yml"
nr = InitNornir(config_file=NORNIR_CONFIG_FILE)
get_host_data_result = nr.run(get_host_data)
update_status = update_arp_db(get_host_data_result, nr)
# nornir_config.yml
---
inventory:
plugin: SimpleInventory
options:
host_file: "inventory/hosts.yml"
group_file: "inventory/groups.yml"
runner:
plugin: threaded
options:
num_workers: 100# groups.yml
---
eve-vios:
username: admin
password: admin
connection_options:
napalm:
extras:
optional_args:
secret: admin# hosts.yml
---
router5:
hostname: 192.168.31.15
platform: ios
groups:
- eve-vios
router6:
hostname: 192.168.31.16
platform: ios
groups:
- eve-vios
二、lldp信息采集
nornir配置都差不多，只是hosts.yml里面增加了数据数量，下面只展示python的代码。
import os
import django
os.environ['DJANGO_SETTINGS_MODULE'] = 'topology.settings'
django.setup()
from nornir import InitNornir
from nornir_napalm.plugins.tasks import napalm_get
from arp.models import Device
from lldp.models import Link
def get_host_data(task):
"""Nornir Task for data collection on target hosts."""
task.run(
task=napalm_get,
getters=['facts', 'lldp_neighbors_detail']
)
# 接口名字适配
interface_full_name_map = {
'Eth': 'Ethernet',
'Fa': 'FastEthernet',
'Gi': 'GigabitEthernet',
'Te': 'TenGigabitEthernet',
}
# 处理为长名字
def if_fullname(ifname):
for k, v in interface_full_name_map.items():
if ifname.startswith(v):
return ifname
if ifname.startswith(k):
return ifname.replace(k, v)
return ifname
# 处理为短名字
def if_shortname(ifname):
for k, v in interface_full_name_map.items():
if ifname.startswith(v):
return ifname.replace(v, k)
return ifname
def update_lldp_db(nornir_lldpAndfacts_result, nornir):
for host in nornir_lldpAndfacts_result:
result = nornir_lldpAndfacts_result[host]
lldp_info = result[1].result['lldp_neighbors_detail']
facts_info = result[1].result['facts']
ip = nornir.inventory.hosts[host].get('hostname', 'n/a')
device, create = Device.objects.update_or_create(
defaults={
'hostname': facts_info['hostname'],
'serial_number': facts_info['serial_number'],
'vendor': facts_info['vendor'],
'model': facts_info['model'],
},
ip=ip)
for lldp in lldp_info: # Gi0/2 接口需要适配
lldp_dict = lldp_info[lldp]
local_device = device
local_port = str(lldp)
remote_device_name = lldp_dict[0]['remote_system_name']
remote_port = lldp_dict[0]['remote_port']
Link.objects.update_or_create(
defaults={
'remote_device_name': remote_device_name.split('.')[0],
'remote_port': if_fullname(remote_port),
},
local_device=local_device,
local_port=if_fullname(local_port),
)
return True
if __name__ == '__main__':
NORNIR_CONFIG_FILE = "nornir_config.yml"
nr = InitNornir(config_file=NORNIR_CONFIG_FILE)
get_host_data_result = nr.run(get_host_data)
update_status = update_lldp_db(get_host_data_result, nr)
三、route信息采集
import copy
import os
import django
# os.environ.setdefault("DJANGO_SETTINGS_MODULE", "topology.settings")
os.environ['DJANGO_SETTINGS_MODULE'] = 'topology.settings'
django.setup()
from nornir import InitNornir
from nornir_napalm.plugins.tasks import napalm_cli
from arp.models import Device
from route.models import Route, NextHop
import re
import pytricia
def get_host_data(task):
"""Nornir Task for data collection on target hosts."""
task.run(
task=napalm_cli,
commands=["show ip route"]
)
# Path to directory with routing table files.
# Each routing table MUST be in a separate .txt file.
RT_DIRECTORY = "./routing_tables"
# RegEx template string for IPv4 address matching.
REGEXP_IPv4_STR = (
r'((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.'
+ r'(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.'
+ r'(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.'
+ r'(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?))'
)
# IPv4 CIDR notation matching in user input.
REGEXP_INPUT_IPv4 = re.compile(r"^" + REGEXP_IPv4_STR + r"(\/\d\d?)?$")
# Local and Connected route strings matching.
REGEXP_ROUTE_LOCAL_CONNECTED = re.compile(
r'^(?P<routeType>[L|C])\s+'
+ r'((?P<ipaddress>\d\d?\d?\.\d\d?\d?\.\d\d?\d?\.\d\d?\d?)'
+ r'\s?'
+ r'(?P<maskOrPrefixLength>(\/\d\d?)?'
+ r'|(\d\d?\d?\.\d\d?\d?\.\d\d?\d?\.\d\d?\d?)?))'
+ r'\ is\ directly\ connected\,\ '
+ r'(?P<interface>\S+)',
re.MULTILINE
)
# Static and dynamic route strings matching.
REGEXP_ROUTE = re.compile(
r'^(\s{6})'
+ r'\d\d?\d?\.\d\d?\d?\.\d\d?\d?\.\d\d?\d?'
+ r'(?P<maskOrPrefixLength>\/\d\d?)' # mask
+ r'(\s{1})is\ subnetted,.*subnets\n'
+ r'^([^L|C])(\s+)'
+ r'(?P<subnet>\d\d?\d?\.\d\d?\d?\.\d\d?\d?\.\d\d?\d?)' #subnet
+ r'?(?P<viaPortion>\s.*via\s.*?\d\d?\d?\.\d\d?\d?\.\d\d?\d?\.\d\d?\d?,.*?\n)+', #nexthop ip
re.MULTILINE
)
REGEXP_VIA_PORTION = re.compile(
r'.*via\s+(\d\d?\d?\.\d\d?\d?\.\d\d?\d?\.\d\d?\d?).*'
)
ROUTERS = {}
GLOBAL_INTERFACE_TREE = pytricia.PyTricia()
def get_next_hop_iface_by_next_hop_ip(next_hop_ips, iface_mapping_ifaceIP):
print(next_hop_ips, iface_mapping_ifaceIP)
next_hop_ifaces = []
for hop in next_hop_ips:
iface = iface_mapping_ifaceIP.get(hop, None)
if iface:
next_hop_ifaces.append(iface)
return next_hop_ifaces
def parse_show_ip_route_ios_like(raw_routing_table):
"""
Parser for routing table text output.
Compatible with both Cisco IOS(IOS-XE) 'show ip route'
and Cisco ASA 'show route' output format.
Processes input text file and write into Python data structures.
Builds internal PyTricia search tree in 'route_tree'.
Generates local interface list for a router in 'interface_list'
Returns 'router' dictionary object with parsed data.
"""
router = {}
route_tree = pytricia.PyTricia()
interface_list = []
# Parse Local and Connected route strings in text.
for raw_route_string in REGEXP_ROUTE_LOCAL_CONNECTED.finditer(raw_routing_table):
subnet = (
raw_route_string.group('ipaddress')
+ convert_netmask_to_prefix_length(
raw_route_string.group('maskOrPrefixLength')
)
)
interface = raw_route_string.group('interface')
route_tree[subnet] = ((interface,), raw_route_string.group(0))
if raw_route_string.group('routeType') == 'L':
interface_list.append((interface, subnet,))
if not interface_list:
print('Failed to find routing table entries in given output')
return None
for raw_route_string in REGEXP_ROUTE.finditer(raw_routing_table):
# print(raw_route_string[0])
subnet = (
raw_route_string.group('subnet')
+ convert_netmask_to_prefix_length(
raw_route_string.group('maskOrPrefixLength')
)
)
via_portion = raw_route_string.group('viaPortion')
next_hops = []
if via_portion.count('via') > 1:
for line in via_portion.splitlines():
if line:
next_hops.append(REGEXP_VIA_PORTION.match(line).group(1))
else:
next_hops.append(REGEXP_VIA_PORTION.match(via_portion).group(1))
# 动态路由下一跳转为设备本地接口
next_hop_ifaces = transfer_nexthop_ip_to_local_iface(next_hops, route_tree)
# print(next_hop_ifaces)
route_tree[subnet] = (next_hops, next_hop_ifaces, raw_route_string.group(0))
router = {
'routing_table': route_tree,
'interface_list': interface_list,
}
return router
def convert_netmask_to_prefix_length(mask_or_pref):
"""
Gets subnet_mask (XXX.XXX.XXX.XXX) of /prefix_length (/XX).
For subnet_mask, converts it to /prefix_length and returns the result.
For /prefix_length, returns as is.
For empty input, returns "" string.
"""
if not mask_or_pref:
return ""
if re.match(r"^\/\d\d?$", mask_or_pref):
return mask_or_pref
if re.match(r"^\d\d?\d?\.\d\d?\d?\.\d\d?\d?\.\d\d?\d?$",
mask_or_pref):
return (
"/"
+ str(sum([bin(int(x)).count("1") for x in mask_or_pref.split(".")]))
)
return ""
def transfer_nexthop_ip_to_local_iface(next_hops, router_tree):
'''
router_tree为L C 路由
'''
next_hops_local_ifaces = []
for next_hop_ip in next_hops:
iface = get_local_iface_by_nexthop_ip(next_hop_ip, router_tree)
if iface:
next_hops_local_ifaces.append(iface)
else:
next_hops_local_ifaces.append(f'{next_hop_ip} can not get interface')
return next_hops_local_ifaces
def get_local_iface_by_nexthop_ip(destination, router_tree):
if destination in router_tree:
return router_tree[destination][0][0]
else:
return
def update_route_db(get_host_data_result, nornir):
for host in get_host_data_result:
raw_route_info = get_host_data_result[host][1].result['show ip route']
ip = nornir.inventory.hosts[host].get('hostname', 'n/a')
router = parse_show_ip_route_ios_like(raw_route_info)
devices = Device.objects.filter(ip=ip)
if len(devices) < 1:
raise ValueError(f'设备IP有误，{ip}不在Device数据表中，请录入后再操作！')
device = devices[0]
for prefix in router['routing_table']:
subnet = prefix
search_route = Route.objects.filter(subnet=subnet, device=device)
if len(search_route) < 1:
route, create = Route.objects.update_or_create(
defaults={
'subnet': subnet,
'device': device
},
subnet=subnet,
device=device)
else:
route = search_route[0]
nexhop_ips = router['routing_table'][prefix][0] if len(router['routing_table'][prefix]) > 2 else [
'0.0.0.0', ]
nexthop_local_ifaces = router['routing_table'][prefix][1] if len(router['routing_table'][prefix]) > 2 else \
router['routing_table'][prefix][0]
for i in range(len(nexhop_ips)):
search_nexthop = NextHop.objects.filter(
nexthop_ipaddress=nexhop_ips[i],
interface=nexthop_local_ifaces[i],
route=route)
if len(search_nexthop) < 1:
nexthop, create = NextHop.objects.update_or_create(
defaults={
'nexthop_ipaddress': nexhop_ips[i],
'interface': nexthop_local_ifaces[i],
'route': route
},
nexthop_ipaddress=nexhop_ips[i],
interface=nexthop_local_ifaces[i],
route=route
)
return True
if __name__ == '__main__':
NORNIR_CONFIG_FILE = "nornir_config.yml"
nornir = InitNornir(config_file=NORNIR_CONFIG_FILE)
get_host_data_result = nornir.run(get_host_data)
status = update_route_db(get_host_data_result, nornir)
print(status)
ps:采集路由这部分代码是参考了国外cisco一位工程师的博客 https://idebugall.github.io/traceroute-by-rt/
