KevinMidboe/mktxp-no-cli
1
0
Fork 0
mirror of https://github.com/KevinMidboe/mktxp-no-cli.git synced 2025-10-29 17:50:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

firewall/mktxp colllectors, fixes/optimizations

Browse Source
This commit is contained in:
Arseniy Kuznetsov
2021-01-24 09:19:31 +01:00
parent 31d0464eb2
commit 158b424e09
24 changed files with 538 additions and 404 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
mktxp/cli/config/config.py
View File

@@ -40,6 +40,7 @@ class MKTXPConfigKeys:
FE_DHCP_LEASE_KEY = 'dhcp_lease'
FE_DHCP_POOL_KEY = 'pool'
FE_INTERFACE_KEY = 'interface'
FE_FIREWALL_KEY = 'firewall'
FE_MONITOR_KEY = 'monitor'
FE_ROUTE_KEY = 'route'
FE_WIRELESS_KEY = 'wireless'
@@ -75,7 +76,7 @@ class MKTXPConfigKeys:
DEFAULT_MKTXP_BANDWIDTH_TEST_INTERVAL = 420
BOOLEAN_KEYS = (ENABLED_KEY, SSL_KEY, NO_SSL_CERTIFICATE, SSL_CERTIFICATE_VERIFY,
FE_DHCP_KEY, FE_DHCP_LEASE_KEY, FE_DHCP_POOL_KEY, FE_INTERFACE_KEY,
FE_DHCP_KEY, FE_DHCP_LEASE_KEY, FE_DHCP_POOL_KEY, FE_INTERFACE_KEY, FE_FIREWALL_KEY,
FE_MONITOR_KEY, FE_ROUTE_KEY, MKTXP_USE_COMMENTS_OVER_NAMES,
FE_WIRELESS_KEY, FE_WIRELESS_CLIENTS_KEY, FE_CAPSMAN_KEY, FE_CAPSMAN_CLIENTS_KEY)
@@ -91,7 +92,7 @@ class ConfigEntry:
MKTXPConfigKeys.USER_KEY, MKTXPConfigKeys.PASSWD_KEY,
MKTXPConfigKeys.SSL_KEY, MKTXPConfigKeys.NO_SSL_CERTIFICATE, MKTXPConfigKeys.SSL_CERTIFICATE_VERIFY,
MKTXPConfigKeys.FE_DHCP_KEY, MKTXPConfigKeys.FE_DHCP_LEASE_KEY, MKTXPConfigKeys.FE_DHCP_POOL_KEY, MKTXPConfigKeys.FE_INTERFACE_KEY,
MKTXPConfigKeys.FE_DHCP_KEY, MKTXPConfigKeys.FE_DHCP_LEASE_KEY, MKTXPConfigKeys.FE_DHCP_POOL_KEY, MKTXPConfigKeys.FE_INTERFACE_KEY, MKTXPConfigKeys.FE_FIREWALL_KEY,
MKTXPConfigKeys.FE_MONITOR_KEY, MKTXPConfigKeys.FE_ROUTE_KEY, MKTXPConfigKeys.FE_WIRELESS_KEY, MKTXPConfigKeys.FE_WIRELESS_CLIENTS_KEY,
MKTXPConfigKeys.FE_CAPSMAN_KEY, MKTXPConfigKeys.FE_CAPSMAN_CLIENTS_KEY, MKTXPConfigKeys.MKTXP_USE_COMMENTS_OVER_NAMES
])
@@ -246,9 +247,9 @@ class MKTXPConfigHandler:
entry_reader[MKTXPConfigKeys.PORT_KEY] = self._default_value_for_key(MKTXPConfigKeys.SSL_KEY, entry_reader[MKTXPConfigKeys.SSL_KEY])
write_needed = True # read from disk next time
if write_needed:
self.config[entry_name] = entry_reader
self.config.write()
if write_needed:
self.config[entry_name] = entry_reader
self.config.write()
return entry_reader
@@ -263,10 +264,10 @@ class MKTXPConfigHandler:
_entry_reader[key] = self._default_value_for_key(key)
write_needed = True # read from disk next time
if write_needed:
self._config[entry_name] = _entry_reader
self._config.write()
if write_needed:
self._config[entry_name] = _entry_reader
self._config.write()
return _entry_reader
def _default_value_for_key(self, key, value = None):

3
mktxp/cli/config/mktxp.conf
View File

@@ -28,6 +28,7 @@
dhcp_lease = True
pool = True
interface = True
firewall = True
monitor = True
route = True
wireless = True
@@ -35,4 +36,4 @@
capsman = True
capsman_clients = True
use_comments_over_names = False # when available, use comments instead of interface names
use_comments_over_names = False # when available, use comments instead of interfaces names

21
mktxp/cli/dispatch.py
View File

@@ -36,21 +36,15 @@ class MKTXPDispatcher:
elif args['sub_cmd'] == MKTXPCommands.SHOW:
self.show_entries(args)
elif args['sub_cmd'] == MKTXPCommands.ADD:
self.add_entry(args)
elif args['sub_cmd'] == MKTXPCommands.EDIT:
self.edit_entry(args)
elif args['sub_cmd'] == MKTXPCommands.DELETE:
self.delete_entry(args)
elif args['sub_cmd'] == MKTXPCommands.EXPORT:
self.start_export(args)
elif args['sub_cmd'] == MKTXPCommands.PRINT:
self.print(args)
elif args['sub_cmd'] == MKTXPCommands.EDIT:
self.edit_entry(args)
else:
# nothing to dispatch
return False
@@ -84,10 +78,6 @@ class MKTXPDispatcher:
print(f' {field}: {getattr(entry, field)}')
print('\n')
def add_entry(self, args):
entry_args = {key: value for key, value in args.items() if key not in set(['sub_cmd', 'entry_name'])}
config_handler.register_entry(entry_name = args['entry_name'], entry_args = entry_args)
def edit_entry(self, args):
editor = args['editor']
if not editor:
@@ -96,10 +86,7 @@ class MKTXPDispatcher:
subprocess.check_call([editor, config_handler.mktxp_conf_path])
else:
subprocess.check_call([editor, config_handler.usr_conf_data_path])
def delete_entry(self, args):
config_handler.unregister_entry(entry_name = args['entry_name'])
def start_export(self, args):
MKTXPProcessor.start()

87
mktxp/cli/options.py
View File

@@ -24,8 +24,6 @@ class MKTXPCommands:
EXPORT = 'export'
PRINT = 'print'
SHOW = 'show'
ADD = 'add'
DELETE = 'delete'
@classmethod
def commands_meta(cls):
@@ -35,8 +33,6 @@ class MKTXPCommands:
f'{cls.EXPORT}, ',
f'{cls.PRINT}, ',
f'{cls.SHOW}, ',
f'{cls.ADD}, ',
f'{cls.DELETE}',
'}'))
class MKTXPOptionsParser:
@@ -102,67 +98,6 @@ Selected metrics info can be printed on the command line. For more information,
help = "Shows MKTXP config files paths",
action = 'store_true')
# Add command
add_parser = subparsers.add_parser(MKTXPCommands.ADD,
description = 'Adds a new MKTXP router entry',
formatter_class=MKTXPHelpFormatter)
required_args_group = add_parser.add_argument_group('Required Arguments')
self._add_entry_name(required_args_group, registered_only = False, help = "Config entry name")
required_args_group.add_argument('-host', '--hostname', dest='hostname',
help = "IP address of RouterOS device to export metrics from",
type = str,
required=True)
required_args_group.add_argument('-usr', '--username', dest='username',
help = "username",
type = str,
required=True)
required_args_group.add_argument('-pwd', '--password', dest='password',
help = "password",
type = str,
required=True)
optional_args_group = add_parser.add_argument_group('Optional Arguments')
optional_args_group.add_argument('-e', dest='enabled',
help = "Enables entry for metrics processing",
action = 'store_false')
optional_args_group.add_argument('-port', dest='port',
help = "port",
default = MKTXPConfigKeys.DEFAULT_API_PORT,
type = int)
optional_args_group.add_argument('-ssl', '--use-ssl', dest='use_ssl',
help = "Connect via RouterOS api-ssl service",
action = 'store_true')
optional_args_group.add_argument('-no-ssl-cert', '--no-ssl-certificate', dest='no_ssl_certificate',
help = "Connect with configured RouterOS SSL ceritficate",
action = 'store_true')
optional_args_group.add_argument('-dhcp', '--export_dhcp', dest='dhcp',
help = "Export DHCP metrics",
action = 'store_true')
optional_args_group.add_argument('-dhcp_lease', '--export_dhcp_lease', dest='dhcp_lease',
help = "Export DHCP Lease metrics",
action = 'store_true')
optional_args_group.add_argument('-pool', '--export_pool', dest='pool',
help = "Export IP Pool metrics",
action = 'store_true')
optional_args_group.add_argument('-interface', '--export_interface', dest='interface',
help = "Export Interface metrics",
action = 'store_true')
optional_args_group.add_argument('-monitor', '--export_monitor', dest='monitor',
help = "Export Interface Monitor metrics",
action = 'store_true')
optional_args_group.add_argument('-route', '--export_route', dest='route',
help = "Export IP Route metrics",
action = 'store_true')
optional_args_group.add_argument('-wireless', '--export_wireless', dest='wireless',
help = "Export Wireless metrics",
action = 'store_true')
optional_args_group.add_argument('-capsman', '--export_capsman', dest='capsman',
help = "Export CAPsMAN metrics",
action = 'store_true')
# Edit command
edit_parser = subparsers.add_parser(MKTXPCommands.EDIT,
description = 'Edits an existing MKTXP router entry',
@@ -176,21 +111,14 @@ Selected metrics info can be printed on the command line. For more information,
help = f"Edit MKTXP internal configuration (advanced)",
action = 'store_true')
# Delete command
delete_parser = subparsers.add_parser(MKTXPCommands.DELETE,
description = 'Deletes an existing MKTXP router entry',
formatter_class=MKTXPHelpFormatter)
required_args_group = delete_parser.add_argument_group('Required Arguments')
self._add_entry_name(required_args_group, registered_only = True, help = "Name of entry to delete")
# Start command
start_parser = subparsers.add_parser(MKTXPCommands.EXPORT,
# Export command
export_parser = subparsers.add_parser(MKTXPCommands.EXPORT,
description = 'Starts exporting Miktorik Router Metrics to Prometheus',
formatter_class=MKTXPHelpFormatter)
# Print command
print_parser = subparsers.add_parser(MKTXPCommands.PRINT,
description = 'Displays seleted metrics on the command line',
description = 'Displays selected metrics on the command line',
formatter_class=MKTXPHelpFormatter)
required_args_group = print_parser.add_argument_group('Required Arguments')
self._add_entry_name(required_args_group, registered_only = True, help = "Name of config RouterOS entry")
@@ -212,17 +140,12 @@ Selected metrics info can be printed on the command line. For more information,
# check if there is a cmd to execute
self._check_cmd_args(args, parser)
if args['sub_cmd'] in (MKTXPCommands.DELETE, MKTXPCommands.SHOW, MKTXPCommands.PRINT):
if args['sub_cmd'] in (MKTXPCommands.SHOW, MKTXPCommands.PRINT):
# Registered Entry name could be a partial match, need to expand
if args['entry_name']:
args['entry_name'] = UniquePartialMatchList(config_handler.registered_entries()).find(args['entry_name'])
if args['sub_cmd'] == MKTXPCommands.ADD:
if args['entry_name'] in (config_handler.registered_entries()):
print(f"{args['entry_name']}: entry name already exists")
parser.exit()
elif args['sub_cmd'] == MKTXPCommands.PRINT:
if args['sub_cmd'] == MKTXPCommands.PRINT:
if not config_handler.entry(args['entry_name']).enabled:
print(f"Can not print metrics for disabled RouterOS entry: {args['entry_name']}\nRun 'mktxp edit' to review and enable it in the configuration file first")
parser.exit()

2
mktxp/cli/output/base_out.py
View File

@@ -97,5 +97,3 @@ class BaseOutputProcessor:
config_handler.re_compiled['interface_rate_rgx'] = interface_rate_rgx
rate = lambda interface_rate: 1000 if interface_rate.find('Mbps') < 0 else 1
return(int(float(interface_rate_rgx.sub('', interface_rate)) * rate(interface_rate)))

60
mktxp/collectors/bandwidth_collector.py Normal file
View File

@@ -0,0 +1,60 @@
# coding=utf8
## Copyright (c) 2020 Arseniy Kuznetsov
##
## This program is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License
## as published by the Free Software Foundation; either version 2
## of the License, or (at your option) any later version.
##
## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
## GNU General Public License for more details.
import speedtest
from datetime import datetime
from multiprocessing import Pool
from mktxp.cli.config.config import config_handler
from mktxp.collectors.base_collector import BaseCollector
result_list = [{'download': 0, 'upload': 0, 'ping': 0}]
def get_result(bandwidth_dict):
result_list[0] = bandwidth_dict
class BandwidthCollector(BaseCollector):
''' MKTXP collector
'''
def __init__(self):
self.pool = Pool()
self.last_call_timestamp = 0
def collect(self):
if result_list:
result_dict = result_list[0]
bandwidth_records = [{'direction': key, 'bandwidth': str(result_dict[key])} for key in ('download', 'upload')]
bandwidth_metrics = BaseCollector.gauge_collector('internet_bandwidth', 'Internet bandwidth in bits per second',
bandwidth_records, 'bandwidth', ['direction'], add_id_labels = False)
yield bandwidth_metrics
latency_records = [{'latency': str(result_dict['ping'])}]
latency_metrics = BaseCollector.gauge_collector('internet_latency', 'Internet latency in milliseconds',
latency_records, 'latency', [], add_id_labels = False)
yield latency_metrics
ts = datetime.now().timestamp()
if (ts - self.last_call_timestamp) > config_handler._entry().bandwidth_test_interval:
self.pool.apply_async(BandwidthCollector.bandwidth_worker, callback=get_result)
self.last_call_timestamp = ts
def __del__(self):
self.pool.close()
self.pool.join()
@staticmethod
def bandwidth_worker():
bandwidth_test = speedtest.Speedtest()
bandwidth_test.get_best_server()
bandwidth_test.download()
bandwidth_test.upload()
return bandwidth_test.results.dict()

5
mktxp/collectors/base_collector.py
View File

@@ -40,8 +40,9 @@ class BaseCollector:
return collector
@staticmethod
def gauge_collector(name, decription, router_records, metric_key, metric_labels=[]):
BaseCollector._add_id_labels(metric_labels)
def gauge_collector(name, decription, router_records, metric_key, metric_labels=[], add_id_labels = True):
if add_id_labels:
BaseCollector._add_id_labels(metric_labels)
collector = GaugeMetricFamily(f'mktxp_{name}', decription, labels=metric_labels)
for router_record in router_records:

68
mktxp/collectors/capsman_collector.py
View File

@@ -22,48 +22,44 @@ class CapsmanCollector(BaseCollector):
def collect(router_metric):
remote_caps_labels = ['identity', 'version', 'base_mac', 'board', 'base_mac']
remote_caps_records = router_metric.capsman_remote_caps_records(remote_caps_labels)
if not remote_caps_records:
return range(0)
remote_caps_metrics = BaseCollector.info_collector('capsman_remote_caps', 'CAPsMAN remote caps', remote_caps_records, remote_caps_labels)
yield remote_caps_metrics
if remote_caps_records:
remote_caps_metrics = BaseCollector.info_collector('capsman_remote_caps', 'CAPsMAN remote caps', remote_caps_records, remote_caps_labels)
yield remote_caps_metrics
registration_labels = ['interface', 'ssid', 'mac_address', 'tx_rate', 'rx_rate', 'rx_signal', 'uptime', 'bytes']
registration_records = router_metric.capsman_registration_table_records(registration_labels)
if not registration_records:
return range(0)
# calculate number of registrations per interface
registration_per_interface = {}
for registration_record in registration_records:
registration_per_interface[registration_record['interface']] = registration_per_interface.get(registration_record['interface'], 0) + 1
# compile registrations-per-interface records
registration_per_interface_records = [{ MKTXPConfigKeys.ROUTERBOARD_NAME: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
'interface': key, 'count': value} for key, value in registration_per_interface.items()]
# yield registrations-per-interface metrics
registration_per_interface_metrics = BaseCollector.gauge_collector('capsman_registrations_count', 'Number of active registration per CAPsMAN interface', registration_per_interface_records, 'count', ['interface'])
yield registration_per_interface_metrics
# the client info metrics
if router_metric.router_entry.capsman_clients:
# translate / trim / augment registration records
dhcp_lease_labels = ['mac_address', 'address', 'host_name', 'comment']
dhcp_lease_records = router_metric.dhcp_lease_records(dhcp_lease_labels)
if registration_records:
# calculate number of registrations per interface
registration_per_interface = {}
for registration_record in registration_records:
BaseOutputProcessor.augment_record(router_metric, registration_record, dhcp_lease_records)
tx_byte_metrics = BaseCollector.counter_collector('capsman_clients_tx_bytes', 'Number of sent packet bytes', registration_records, 'tx_bytes', ['dhcp_name'])
yield tx_byte_metrics
registration_per_interface[registration_record['interface']] = registration_per_interface.get(registration_record['interface'], 0) + 1
# compile registrations-per-interface records
registration_per_interface_records = [{ MKTXPConfigKeys.ROUTERBOARD_NAME: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
'interface': key, 'count': value} for key, value in registration_per_interface.items()]
# yield registrations-per-interface metrics
registration_per_interface_metrics = BaseCollector.gauge_collector('capsman_registrations_count', 'Number of active registration per CAPsMAN interface', registration_per_interface_records, 'count', ['interface'])
yield registration_per_interface_metrics
rx_byte_metrics = BaseCollector.counter_collector('capsman_clients_rx_bytes', 'Number of received packet bytes', registration_records, 'rx_bytes', ['dhcp_name'])
yield rx_byte_metrics
# the client info metrics
if router_metric.router_entry.capsman_clients:
# translate / trim / augment registration records
dhcp_lease_labels = ['mac_address', 'address', 'host_name', 'comment']
dhcp_lease_records = router_metric.dhcp_lease_records(dhcp_lease_labels)
for registration_record in registration_records:
BaseOutputProcessor.augment_record(router_metric, registration_record, dhcp_lease_records)
tx_byte_metrics = BaseCollector.counter_collector('capsman_clients_tx_bytes', 'Number of sent packet bytes', registration_records, 'tx_bytes', ['dhcp_name'])
yield tx_byte_metrics
signal_strength_metrics = BaseCollector.gauge_collector('capsman_clients_signal_strength', 'Client devices signal strength', registration_records, 'rx_signal', ['dhcp_name'])
yield signal_strength_metrics
rx_byte_metrics = BaseCollector.counter_collector('capsman_clients_rx_bytes', 'Number of received packet bytes', registration_records, 'rx_bytes', ['dhcp_name'])
yield rx_byte_metrics
registration_metrics = BaseCollector.info_collector('capsman_clients_devices', 'Registered client devices info',
registration_records, ['dhcp_name', 'dhcp_address', 'rx_signal', 'ssid', 'tx_rate', 'rx_rate', 'interface', 'mac_address', 'uptime'])
yield registration_metrics
signal_strength_metrics = BaseCollector.gauge_collector('capsman_clients_signal_strength', 'Client devices signal strength', registration_records, 'rx_signal', ['dhcp_name'])
yield signal_strength_metrics
registration_metrics = BaseCollector.info_collector('capsman_clients_devices', 'Registered client devices info',
registration_records, ['dhcp_name', 'dhcp_address', 'rx_signal', 'ssid', 'tx_rate', 'rx_rate', 'interface', 'mac_address', 'uptime'])
yield registration_metrics

38
mktxp/collectors/dhcp_collector.py
View File

@@ -14,6 +14,7 @@
from mktxp.cli.config.config import MKTXPConfigKeys
from mktxp.collectors.base_collector import BaseCollector
class DHCPCollector(BaseCollector):
''' DHCP Metrics collector
'''
@@ -21,24 +22,23 @@ class DHCPCollector(BaseCollector):
def collect(router_metric):
dhcp_lease_labels = ['active_address', 'mac_address', 'host_name', 'comment', 'server', 'expires_after']
dhcp_lease_records = router_metric.dhcp_lease_records(dhcp_lease_labels)
if not dhcp_lease_records:
return range(0)
if dhcp_lease_records:
# calculate number of leases per DHCP server
dhcp_lease_servers = {}
for dhcp_lease_record in dhcp_lease_records:
dhcp_lease_servers[dhcp_lease_record['server']] = dhcp_lease_servers.get(dhcp_lease_record['server'], 0) + 1
# calculate number of leases per DHCP server
dhcp_lease_servers = {}
for dhcp_lease_record in dhcp_lease_records:
dhcp_lease_servers[dhcp_lease_record['server']] = dhcp_lease_servers.get(dhcp_lease_record['server'], 0) + 1
# compile leases-per-server records
dhcp_lease_servers_records = [{ MKTXPConfigKeys.ROUTERBOARD_NAME: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
'server': key, 'count': value} for key, value in dhcp_lease_servers.items()]
# yield lease-per-server metrics
dhcp_lease_server_metrics = BaseCollector.gauge_collector('dhcp_lease_active_count', 'Number of active leases per DHCP server', dhcp_lease_servers_records, 'count', ['server'])
yield dhcp_lease_server_metrics
# compile leases-per-server records
dhcp_lease_servers_records = [{ MKTXPConfigKeys.ROUTERBOARD_NAME: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
'server': key, 'count': value} for key, value in dhcp_lease_servers.items()]
# yield lease-per-server metrics
dhcp_lease_server_metrics = BaseCollector.gauge_collector('dhcp_lease_active_count', 'Number of active leases per DHCP server', dhcp_lease_servers_records, 'count', ['server'])
yield dhcp_lease_server_metrics
# active lease metrics
if router_metric.router_entry.dhcp_lease:
dhcp_lease_metrics = BaseCollector.info_collector('dhcp_lease', 'DHCP Active Leases', dhcp_lease_records, dhcp_lease_labels)
yield dhcp_lease_metrics
# active lease metrics
if router_metric.router_entry.dhcp_lease:
dhcp_lease_metrics = BaseCollector.info_collector('dhcp_lease', 'DHCP Active Leases', dhcp_lease_records, dhcp_lease_labels)
yield dhcp_lease_metrics

44
mktxp/collectors/firewall_collector.py Normal file
View File

@@ -0,0 +1,44 @@
# coding=utf8
## Copyright (c) 2020 Arseniy Kuznetsov
##
## This program is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License
## as published by the Free Software Foundation; either version 2
## of the License, or (at your option) any later version.
##
## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
## GNU General Public License for more details.
from mktxp.collectors.base_collector import BaseCollector
from mktxp.cli.config.config import MKTXPConfigKeys
class FirewallCollector(BaseCollector):
''' Firewall rules traffic metrics collector
'''
@staticmethod
def collect(router_metric):
# initialize all pool counts, including those currently not used
firewall_labels = ['chain', 'action', 'bytes', 'comment']
firewall_filter_records = router_metric.firewall_records(firewall_labels)
if firewall_filter_records:
metris_records = [FirewallCollector.metric_record(router_metric, record) for record in firewall_filter_records]
firewall_filter_metrics = BaseCollector.counter_collector('firewall_filter', 'Total amount of bytes matched by firewall rules', metris_records, 'bytes', ['name'])
yield firewall_filter_metrics
firewall_raw_records = router_metric.firewall_records(firewall_labels, raw = True)
if firewall_raw_records:
metris_records = [FirewallCollector.metric_record(router_metric, record) for record in firewall_raw_records]
firewall_raw_metrics = BaseCollector.counter_collector('firewall_raw', 'Total amount of bytes matched by raw firewall rules', metris_records, 'bytes', ['name'])
yield firewall_raw_metrics
# Helpers
@staticmethod
def metric_record(router_metric, firewall_record):
name = f"| {firewall_record['chain']} | {firewall_record['action']} | {firewall_record['comment']}"
bytes = firewall_record['bytes']
return {MKTXPConfigKeys.ROUTERBOARD_NAME: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
'name': name, 'bytes': bytes}

16
mktxp/collectors/health_collector.py
View File

@@ -13,19 +13,17 @@
from mktxp.collectors.base_collector import BaseCollector
class HealthCollector(BaseCollector):
''' System Health Metrics collector
'''
@staticmethod
def collect(router_metric):
health_labels = ['voltage', 'temperature']
health_records = router_metric.health_records(health_labels)
if not health_records:
return range(0)
voltage_metrics = BaseCollector.gauge_collector('system_routerboard_voltage', 'Supplied routerboard voltage', health_records, 'voltage')
yield voltage_metrics
temperature_metrics = BaseCollector.gauge_collector('system_routerboard_temperature', ' Routerboard current temperature', health_records, 'temperature')
yield temperature_metrics
health_records = router_metric.health_records(health_labels)
if health_records:
voltage_metrics = BaseCollector.gauge_collector('system_routerboard_voltage', 'Supplied routerboard voltage', health_records, 'voltage')
yield voltage_metrics
temperature_metrics = BaseCollector.gauge_collector('system_routerboard_temperature', ' Routerboard current temperature', health_records, 'temperature')
yield temperature_metrics

10
mktxp/collectors/identity_collector.py
View File

@@ -19,10 +19,8 @@ class IdentityCollector(BaseCollector):
@staticmethod
def collect(router_metric):
identity_labels = ['name']
identity_records = router_metric.identity_records(identity_labels)
if not identity_records:
return range(0)
identity_metrics = BaseCollector.info_collector('system_identity', 'System identity', identity_records, identity_labels)
yield identity_metrics
identity_records = router_metric.identity_records(identity_labels)
if identity_records:
identity_metrics = BaseCollector.info_collector('system_identity', 'System identity', identity_records, identity_labels)
yield identity_metrics

47
mktxp/collectors/interface_collector.py
View File

@@ -20,36 +20,33 @@ class InterfaceCollector(BaseCollector):
def collect(router_metric):
interface_traffic_labels = ['name', 'comment', 'rx_byte', 'tx_byte', 'rx_packet', 'tx_packet', 'rx_error', 'tx_error', 'rx_drop', 'tx_drop']
interface_traffic_records = router_metric.interface_traffic_records(interface_traffic_labels)
if not interface_traffic_records:
return range(0)
if interface_traffic_records:
for interface_traffic_record in interface_traffic_records:
if interface_traffic_record.get('comment'):
interface_traffic_record['name'] = interface_traffic_record['comment'] if router_metric.router_entry.use_comments_over_names \
else f"{interface_traffic_record['name']} ({interface_traffic_record['comment']})"
for interface_traffic_record in interface_traffic_records:
if interface_traffic_record.get('comment'):
interface_traffic_record['name'] = interface_traffic_record['comment'] if router_metric.router_entry.use_comments_over_names \
else f"{interface_traffic_record['name']} ({interface_traffic_record['comment']})"
rx_byte_metric = BaseCollector.counter_collector('interface_rx_byte', 'Number of received bytes', interface_traffic_records, 'rx_byte', ['name'])
yield rx_byte_metric
rx_byte_metric = BaseCollector.counter_collector('interface_rx_byte', 'Number of received bytes', interface_traffic_records, 'rx_byte', ['name'])
yield rx_byte_metric
tx_byte_metric = BaseCollector.counter_collector('interface_tx_byte', 'Number of transmitted bytes', interface_traffic_records, 'tx_byte', ['name'])
yield tx_byte_metric
tx_byte_metric = BaseCollector.counter_collector('interface_tx_byte', 'Number of transmitted bytes', interface_traffic_records, 'tx_byte', ['name'])
yield tx_byte_metric
rx_packet_metric = BaseCollector.counter_collector('interface_rx_packet', 'Number of packets received', interface_traffic_records, 'rx_packet', ['name'])
yield rx_packet_metric
rx_packet_metric = BaseCollector.counter_collector('interface_rx_packet', 'Number of packets received', interface_traffic_records, 'rx_packet', ['name'])
yield rx_packet_metric
tx_packet_metric = BaseCollector.counter_collector('interface_tx_packet', 'Number of transmitted packets', interface_traffic_records, 'tx_packet', ['name'])
yield tx_packet_metric
tx_packet_metric = BaseCollector.counter_collector('interface_tx_packet', 'Number of transmitted packets', interface_traffic_records, 'tx_packet', ['name'])
yield tx_packet_metric
rx_error_metric = BaseCollector.counter_collector('interface_rx_error', 'Number of packets received with an error', interface_traffic_records, 'rx_error', ['name'])
yield rx_error_metric
rx_error_metric = BaseCollector.counter_collector('interface_rx_error', 'Number of packets received with an error', interface_traffic_records, 'rx_error', ['name'])
yield rx_error_metric
tx_error_metric = BaseCollector.counter_collector('interface_tx_error', 'Number of packets transmitted with an error', interface_traffic_records, 'tx_error', ['name'])
yield tx_error_metric
rx_drop_metric = BaseCollector.counter_collector('interface_rx_drop', 'Number of received packets being dropped', interface_traffic_records, 'rx_drop', ['name'])
yield rx_drop_metric
tx_drop_metric = BaseCollector.counter_collector('interface_tx_drop', 'Number of transmitted packets being dropped', interface_traffic_records, 'tx_drop', ['name'])
yield tx_drop_metric
tx_error_metric = BaseCollector.counter_collector('interface_tx_error', 'Number of packets transmitted with an error', interface_traffic_records, 'tx_error', ['name'])
yield tx_error_metric
rx_drop_metric = BaseCollector.counter_collector('interface_rx_drop', 'Number of received packets being dropped', interface_traffic_records, 'rx_drop', ['name'])
yield rx_drop_metric
tx_drop_metric = BaseCollector.counter_collector('interface_tx_drop', 'Number of transmitted packets being dropped', interface_traffic_records, 'tx_drop', ['name'])
yield tx_drop_metric

51
mktxp/collectors/mktxp_collector.py
View File

@@ -11,47 +11,16 @@
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
## GNU General Public License for more details.
from mktxp.collectors.base_collector import BaseCollector
import speedtest
from datetime import datetime
from multiprocessing import Pool
from prometheus_client import Gauge
from mktxp.cli.config.config import config_handler
result_list = [{'download': 0, 'upload': 0, 'ping': 0}]
def get_result(bandwidth_dict):
result_list.append(bandwidth_dict)
class MKTXPCollector:
''' MKTXP collector
'''
def __init__(self):
self.pool = Pool()
self.last_call_timestamp = 0
self.gauge_bandwidth = Gauge('mktxp_internet_bandwidth', 'Internet bandwidth in bits per second', ['direction'])
self.gauge_latency = Gauge('mktxp_internet_latency', 'Internet bandwidth latency in milliseconds')
def collect(self):
if result_list:
bandwidth_dict = result_list.pop(0)
self.gauge_bandwidth.labels('download').set(bandwidth_dict["download"])
self.gauge_bandwidth.labels('upload').set(bandwidth_dict["upload"])
self.gauge_latency.set(bandwidth_dict["ping"])
ts = datetime.now().timestamp()
if (ts - self.last_call_timestamp) > config_handler._entry().bandwidth_test_interval:
self.pool.apply_async(MKTXPCollector.bandwidth_worker, callback=get_result)
self.last_call_timestamp = ts
def __del__(self):
self.pool.close()
self.pool.join()
class MKTXPCollector(BaseCollector):
''' System Identity Metrics collector
'''
@staticmethod
def bandwidth_worker():
bandwidth_test = speedtest.Speedtest()
bandwidth_test.get_best_server()
bandwidth_test.download()
bandwidth_test.upload()
return bandwidth_test.results.dict()
def collect(router_metric):
mktxp_records = router_metric.mktxp_records()
if mktxp_records:
mktxp_duration_metric = BaseCollector.counter_collector('collection_time', 'Total time spent collecting metrics in milliseconds', mktxp_records, 'duration', ['name'])
yield mktxp_duration_metric

35
mktxp/collectors/monitor_collector.py
View File

@@ -21,28 +21,25 @@ class MonitorCollector(BaseCollector):
def collect(router_metric):
monitor_labels = ('status', 'rate', 'full_duplex', 'name')
monitor_records = router_metric.interface_monitor_records(monitor_labels, include_comments = True)
if not monitor_records:
return range(0)
if monitor_records:
# translate records to appropriate values
for monitor_record in monitor_records:
for monitor_label in monitor_labels:
value = monitor_record.get(monitor_label, None)
if value:
monitor_record[monitor_label] = MonitorCollector._translated_values(monitor_label, value)
# translate records to appropriate values
for monitor_record in monitor_records:
for monitor_label in monitor_labels:
value = monitor_record.get(monitor_label, None)
if value:
monitor_record[monitor_label] = MonitorCollector._translated_values(monitor_label, value)
monitor_status_metrics = BaseCollector.gauge_collector('interface_status', 'Current interface link status', monitor_records, 'status', ['name'])
yield monitor_status_metrics
monitor_status_metrics = BaseCollector.gauge_collector('interface_status', 'Current interface link status', monitor_records, 'status', ['name'])
yield monitor_status_metrics
# limit records according to the relevant metrics
rate_records = [monitor_record for monitor_record in monitor_records if monitor_record.get('rate', None)]
monitor_rates_metrics = BaseCollector.gauge_collector('interface_rate', 'Actual interface connection data rate', rate_records, 'rate', ['name'])
yield monitor_rates_metrics
full_duplex_records = [monitor_record for monitor_record in monitor_records if monitor_record.get('full_duplex', None)]
monitor_rates_metrics = BaseCollector.gauge_collector('interface_full_duplex', 'Full duplex data transmission', full_duplex_records, 'full_duplex', ['name'])
yield monitor_rates_metrics
# limit records according to the relevant metrics
rate_records = [monitor_record for monitor_record in monitor_records if monitor_record.get('rate', None)]
monitor_rates_metrics = BaseCollector.gauge_collector('interface_rate', 'Actual interface connection data rate', rate_records, 'rate', ['name'])
yield monitor_rates_metrics
full_duplex_records = [monitor_record for monitor_record in monitor_records if monitor_record.get('full_duplex', None)]
monitor_rates_metrics = BaseCollector.gauge_collector('interface_full_duplex', 'Full duplex data transmission', full_duplex_records, 'full_duplex', ['name'])
yield monitor_rates_metrics
# Helpers
@staticmethod

33
mktxp/collectors/pool_collector.py
View File

@@ -19,25 +19,22 @@ class PoolCollector(BaseCollector):
'''
@staticmethod
def collect(router_metric):
# initialize all pool counts, including those currently not used
pool_records = router_metric.pool_records(['name'])
if not pool_records:
return range(0)
if pool_records:
pool_used_labels = ['pool']
pool_used_counts = {pool_record['name']: 0 for pool_record in pool_records}
pool_used_labels = ['pool']
pool_used_counts = {pool_record['name']: 0 for pool_record in pool_records}
# for pools in usage, calculate the current numbers
pool_used_records = router_metric.pool_used_records(pool_used_labels)
for pool_used_record in pool_used_records:
pool_used_counts[pool_used_record['pool']] = pool_used_counts.get(pool_used_record['pool'], 0) + 1
# for pools in usage, calculate the current numbers
pool_used_records = router_metric.pool_used_records(pool_used_labels)
for pool_used_record in pool_used_records:
pool_used_counts[pool_used_record['pool']] = pool_used_counts.get(pool_used_record['pool'], 0) + 1
# compile used-per-pool records
used_per_pool_records = [{ MKTXPConfigKeys.ROUTERBOARD_NAME: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
'pool': key, 'count': value} for key, value in pool_used_counts.items()]
# yield used-per-pool metrics
used_per_pool_metrics = BaseCollector.gauge_collector('ip_pool_used', 'Number of used addresses per IP pool', used_per_pool_records, 'count', ['pool'])
yield used_per_pool_metrics
# compile used-per-pool records
used_per_pool_records = [{ MKTXPConfigKeys.ROUTERBOARD_NAME: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
'pool': key, 'count': value} for key, value in pool_used_counts.items()]
# yield used-per-pool metrics
used_per_pool_metrics = BaseCollector.gauge_collector('ip_pool_used', 'Number of used addresses per IP pool', used_per_pool_records, 'count', ['pool'])
yield used_per_pool_metrics

51
mktxp/collectors/resource_collector.py
View File

@@ -23,41 +23,40 @@ class SystemResourceCollector(BaseCollector):
'cpu', 'cpu_count', 'cpu_frequency', 'cpu_load',
'free_hdd_space', 'total_hdd_space',
'architecture_name', 'board_name']
resource_records = router_metric.system_resource_records(resource_labels)
if not resource_records:
return range(0)
if resource_records:
# translate records to appropriate values
translated_fields = ['uptime']
for resource_record in resource_records:
for translated_field in translated_fields:
value = resource_record.get(translated_field, None)
if value:
resource_record[translated_field] = SystemResourceCollector._translated_values(translated_field, value)
# translate records to appropriate values
translated_fields = ['uptime']
for resource_record in resource_records:
for translated_field in translated_fields:
value = resource_record.get(translated_field, None)
if value:
resource_record[translated_field] = SystemResourceCollector._translated_values(translated_field, value)
uptime_metrics = BaseCollector.gauge_collector('system_uptime', 'Time interval since boot-up', resource_records, 'uptime', ['version', 'board_name', 'cpu', 'architecture_name'])
yield uptime_metrics
uptime_metrics = BaseCollector.gauge_collector('system_uptime', 'Time interval since boot-up', resource_records, 'uptime', ['version', 'board_name', 'cpu', 'architecture_name'])
yield uptime_metrics
free_memory_metrics = BaseCollector.gauge_collector('system_free_memory', 'Unused amount of RAM', resource_records, 'free_memory', ['version', 'board_name', 'cpu', 'architecture_name'])
yield free_memory_metrics
free_memory_metrics = BaseCollector.gauge_collector('system_free_memory', 'Unused amount of RAM', resource_records, 'free_memory', ['version', 'board_name', 'cpu', 'architecture_name'])
yield free_memory_metrics
total_memory_metrics = BaseCollector.gauge_collector('system_total_memory', 'Amount of installed RAM', resource_records, 'total_memory', ['version', 'board_name', 'cpu', 'architecture_name'])
yield total_memory_metrics
total_memory_metrics = BaseCollector.gauge_collector('system_total_memory', 'Amount of installed RAM', resource_records, 'total_memory', ['version', 'board_name', 'cpu', 'architecture_name'])
yield total_memory_metrics
free_hdd_metrics = BaseCollector.gauge_collector('system_free_hdd_space', 'Free space on hard drive or NAND', resource_records, 'free_hdd_space', ['version', 'board_name', 'cpu', 'architecture_name'])
yield free_hdd_metrics
free_hdd_metrics = BaseCollector.gauge_collector('system_free_hdd_space', 'Free space on hard drive or NAND', resource_records, 'free_hdd_space', ['version', 'board_name', 'cpu', 'architecture_name'])
yield free_hdd_metrics
total_hdd_metrics = BaseCollector.gauge_collector('system_total_hdd_space', 'Size of the hard drive or NAND', resource_records, 'total_hdd_space', ['version', 'board_name', 'cpu', 'architecture_name'])
yield total_hdd_metrics
total_hdd_metrics = BaseCollector.gauge_collector('system_total_hdd_space', 'Size of the hard drive or NAND', resource_records, 'total_hdd_space', ['version', 'board_name', 'cpu', 'architecture_name'])
yield total_hdd_metrics
cpu_load_metrics = BaseCollector.gauge_collector('system_cpu_load', 'Percentage of used CPU resources', resource_records, 'cpu_load', ['version', 'board_name', 'cpu', 'architecture_name'])
yield cpu_load_metrics
cpu_load_metrics = BaseCollector.gauge_collector('system_cpu_load', 'Percentage of used CPU resources', resource_records, 'cpu_load', ['version', 'board_name', 'cpu', 'architecture_name'])
yield cpu_load_metrics
cpu_count_metrics = BaseCollector.gauge_collector('system_cpu_count', 'Number of CPUs present on the system', resource_records, 'cpu_count', ['version', 'board_name', 'cpu', 'architecture_name'])
yield cpu_count_metrics
cpu_count_metrics = BaseCollector.gauge_collector('system_cpu_count', 'Number of CPUs present on the system', resource_records, 'cpu_count', ['version', 'board_name', 'cpu', 'architecture_name'])
yield cpu_count_metrics
cpu_frequency_metrics = BaseCollector.gauge_collector('system_cpu_frequency', 'Current CPU frequency', resource_records, 'cpu_frequency', ['version', 'board_name', 'cpu', 'architecture_name'])
yield cpu_frequency_metrics
cpu_frequency_metrics = BaseCollector.gauge_collector('system_cpu_frequency', 'Current CPU frequency', resource_records, 'cpu_frequency', ['version', 'board_name', 'cpu', 'architecture_name'])
yield cpu_frequency_metrics
# Helpers

48
mktxp/collectors/route_collector.py
View File

@@ -21,32 +21,30 @@ class RouteCollector(BaseCollector):
def collect(router_metric):
route_labels = ['connect', 'dynamic', 'static', 'bgp', 'ospf']
route_records = router_metric.route_records(route_labels)
if not route_records:
return range(0)
# compile total routes records
total_routes = len(route_records)
total_routes_records = [{ MKTXPConfigKeys.ROUTERBOARD_NAME: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
'count': total_routes
}]
total_routes_metrics = BaseCollector.gauge_collector('routes_total_routes', 'Overall number of routes in RIB', total_routes_records, 'count')
yield total_routes_metrics
if route_records:
# compile total routes records
total_routes = len(route_records)
total_routes_records = [{ MKTXPConfigKeys.ROUTERBOARD_NAME: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
'count': total_routes
}]
total_routes_metrics = BaseCollector.gauge_collector('routes_total_routes', 'Overall number of routes in RIB', total_routes_records, 'count')
yield total_routes_metrics
# init routes per protocol (with 0)
routes_per_protocol = {route_label: 0 for route_label in route_labels}
for route_record in route_records:
for route_label in route_labels:
if route_record.get(route_label):
routes_per_protocol[route_label] += 1
# init routes per protocol (with 0)
routes_per_protocol = {route_label: 0 for route_label in route_labels}
for route_record in route_records:
for route_label in route_labels:
if route_record.get(route_label):
routes_per_protocol[route_label] += 1
# compile route-per-protocol records
route_per_protocol_records = [{ MKTXPConfigKeys.ROUTERBOARD_NAME: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
'protocol': key, 'count': value} for key, value in routes_per_protocol.items()]
# yield route-per-protocol metrics
route_per_protocol_metrics = BaseCollector.gauge_collector('routes_protocol_count', 'Number of routes per protocol in RIB', route_per_protocol_records, 'count', ['protocol'])
yield route_per_protocol_metrics
# compile route-per-protocol records
route_per_protocol_records = [{ MKTXPConfigKeys.ROUTERBOARD_NAME: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
'protocol': key, 'count': value} for key, value in routes_per_protocol.items()]
# yield route-per-protocol metrics
route_per_protocol_metrics = BaseCollector.gauge_collector('routes_protocol_count', 'Number of routes per protocol in RIB', route_per_protocol_records, 'count', ['protocol'])
yield route_per_protocol_metrics

74
mktxp/collectors/wlan_collector.py
View File

@@ -21,61 +21,53 @@ class WLANCollector(BaseCollector):
def collect(router_metric):
monitor_labels = ['channel', 'noise_floor', 'overall_tx_ccq', 'registered_clients']
monitor_records = router_metric.interface_monitor_records(monitor_labels, 'wireless')
if not monitor_records:
return range(0)
if monitor_records:
# sanitize records for relevant labels
noise_floor_records = [monitor_record for monitor_record in monitor_records if monitor_record.get('noise_floor')]
tx_ccq_records = [monitor_record for monitor_record in monitor_records if monitor_record.get('overall_tx_ccq')]
registered_clients_records = [monitor_record for monitor_record in monitor_records if monitor_record.get('registered_clients')]
# sanitize records for relevant labels
noise_floor_records = [monitor_record for monitor_record in monitor_records if monitor_record.get('noise_floor')]
tx_ccq_records = [monitor_record for monitor_record in monitor_records if monitor_record.get('overall_tx_ccq')]
registered_clients_records = [monitor_record for monitor_record in monitor_records if monitor_record.get('registered_clients')]
if noise_floor_records:
noise_floor_metrics = BaseCollector.gauge_collector('wlan_noise_floor', 'Noise floor threshold', noise_floor_records, 'noise_floor', ['channel'])
yield noise_floor_metrics
if noise_floor_records:
noise_floor_metrics = BaseCollector.gauge_collector('wlan_noise_floor', 'Noise floor threshold', noise_floor_records, 'noise_floor', ['channel'])
yield noise_floor_metrics
if tx_ccq_records:
overall_tx_ccq_metrics = BaseCollector.gauge_collector('wlan_overall_tx_ccq', 'Client Connection Quality for transmitting', tx_ccq_records, 'overall_tx_ccq', ['channel'])
yield overall_tx_ccq_metrics
if registered_clients_records:
registered_clients_metrics = BaseCollector.gauge_collector('wlan_registered_clients', 'Number of registered clients', registered_clients_records, 'registered_clients', ['channel'])
yield registered_clients_metrics
if tx_ccq_records:
overall_tx_ccq_metrics = BaseCollector.gauge_collector('wlan_overall_tx_ccq', 'Client Connection Quality for transmitting', tx_ccq_records, 'overall_tx_ccq', ['channel'])
yield overall_tx_ccq_metrics
if registered_clients_records:
registered_clients_metrics = BaseCollector.gauge_collector('wlan_registered_clients', 'Number of registered clients', registered_clients_records, 'registered_clients', ['channel'])
yield registered_clients_metrics
# the client info metrics
if router_metric.router_entry.wireless_clients:
registration_labels = ['interface', 'ssid', 'mac_address', 'tx_rate', 'rx_rate', 'uptime', 'bytes', 'signal_to_noise', 'tx_ccq', 'signal_strength']
registration_records = router_metric.wireless_registration_table_records(registration_labels)
if not registration_records:
return range(0)
if registration_records:
dhcp_lease_labels = ['mac_address', 'address', 'host_name', 'comment']
dhcp_lease_records = router_metric.dhcp_lease_records(dhcp_lease_labels)
for registration_record in registration_records:
BaseOutputProcessor.augment_record(router_metric, registration_record, dhcp_lease_records)
dhcp_lease_labels = ['mac_address', 'address', 'host_name', 'comment']
dhcp_lease_records = router_metric.dhcp_lease_records(dhcp_lease_labels)
for registration_record in registration_records:
BaseOutputProcessor.augment_record(router_metric, registration_record, dhcp_lease_records)
tx_byte_metrics = BaseCollector.counter_collector('wlan_clients_tx_bytes', 'Number of sent packet bytes', registration_records, 'tx_bytes', ['dhcp_name'])
yield tx_byte_metrics
tx_byte_metrics = BaseCollector.counter_collector('wlan_clients_tx_bytes', 'Number of sent packet bytes', registration_records, 'tx_bytes', ['dhcp_name'])
yield tx_byte_metrics
rx_byte_metrics = BaseCollector.counter_collector('wlan_clients_rx_bytes', 'Number of received packet bytes', registration_records, 'rx_bytes', ['dhcp_name'])
yield rx_byte_metrics
rx_byte_metrics = BaseCollector.counter_collector('wlan_clients_rx_bytes', 'Number of received packet bytes', registration_records, 'rx_bytes', ['dhcp_name'])
yield rx_byte_metrics
signal_strength_metrics = BaseCollector.gauge_collector('wlan_clients_signal_strength', 'Average strength of the client signal recevied by AP', registration_records, 'signal_strength', ['dhcp_name'])
yield signal_strength_metrics
signal_strength_metrics = BaseCollector.gauge_collector('wlan_clients_signal_strength', 'Average strength of the client signal recevied by AP', registration_records, 'signal_strength', ['dhcp_name'])
yield signal_strength_metrics
signal_to_noise_metrics = BaseCollector.gauge_collector('wlan_clients_signal_to_noise', 'Client devices signal to noise ratio', registration_records, 'signal_to_noise', ['dhcp_name'])
yield signal_to_noise_metrics
signal_to_noise_metrics = BaseCollector.gauge_collector('wlan_clients_signal_to_noise', 'Client devices signal to noise ratio', registration_records, 'signal_to_noise', ['dhcp_name'])
yield signal_to_noise_metrics
tx_ccq_metrics = BaseCollector.gauge_collector('wlan_clients_tx_ccq', 'Client Connection Quality (CCQ) for transmit', registration_records, 'tx_ccq', ['dhcp_name'])
yield tx_ccq_metrics
tx_ccq_metrics = BaseCollector.gauge_collector('wlan_clients_tx_ccq', 'Client Connection Quality (CCQ) for transmit', registration_records, 'tx_ccq', ['dhcp_name'])
yield tx_ccq_metrics
registration_metrics = BaseCollector.info_collector('wlan_clients_devices', 'Client devices info',
registration_records, ['dhcp_name', 'dhcp_address', 'rx_signal', 'ssid', 'tx_rate', 'rx_rate', 'interface', 'mac_address', 'uptime'])
yield registration_metrics
return range(0)
registration_metrics = BaseCollector.info_collector('wlan_clients_devices', 'Client devices info',
registration_records, ['dhcp_name', 'dhcp_address', 'rx_signal', 'ssid', 'tx_rate', 'rx_rate', 'interface', 'mac_address', 'uptime'])
yield registration_metrics

45
mktxp/collectors_handler.py
View File

@@ -11,6 +11,7 @@
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
## GNU General Public License for more details.
from timeit import default_timer
from mktxp.collectors.dhcp_collector import DHCPCollector
from mktxp.collectors.interface_collector import InterfaceCollector
from mktxp.collectors.health_collector import HealthCollector
@@ -21,18 +22,21 @@ from mktxp.collectors.resource_collector import SystemResourceCollector
from mktxp.collectors.route_collector import RouteCollector
from mktxp.collectors.wlan_collector import WLANCollector
from mktxp.collectors.capsman_collector import CapsmanCollector
from mktxp.collectors.bandwidth_collector import BandwidthCollector
from mktxp.collectors.firewall_collector import FirewallCollector
from mktxp.collectors.mktxp_collector import MKTXPCollector
class CollectorsHandler:
''' MKTXP Collectors Handler
'''
def __init__(self, metrics_handler):
self.metrics_handler = metrics_handler
self.mktxpCollector = MKTXPCollector()
self.bandwidthCollector = BandwidthCollector()
def collect(self):
# process mktxp internal metrics
self.mktxpCollector.collect()
yield from self.bandwidthCollector.collect()
for router_metric in self.metrics_handler.router_metrics:
if not router_metric.api_connection.is_connected():
@@ -40,29 +44,60 @@ class CollectorsHandler:
router_metric.api_connection.connect()
continue
start = default_timer()
yield from IdentityCollector.collect(router_metric)
router_metric.time_spent['IdentityCollector'] += default_timer() - start
start = default_timer()
yield from SystemResourceCollector.collect(router_metric)
router_metric.time_spent['SystemResourceCollector'] += default_timer() - start
start = default_timer()
yield from HealthCollector.collect(router_metric)
router_metric.time_spent['HealthCollector'] += default_timer() - start
if router_metric.router_entry.dhcp:
yield from DHCPCollector.collect(router_metric)
start = default_timer()
yield from DHCPCollector.collect(router_metric)
router_metric.time_spent['DHCPCollector'] += default_timer() - start
if router_metric.router_entry.pool:
start = default_timer()
yield from PoolCollector.collect(router_metric)
router_metric.time_spent['PoolCollector'] += default_timer() - start
if router_metric.router_entry.interface:
start = default_timer()
yield from InterfaceCollector.collect(router_metric)
router_metric.time_spent['InterfaceCollector'] += default_timer() - start
if router_metric.router_entry.firewall:
start = default_timer()
yield from FirewallCollector.collect(router_metric)
router_metric.time_spent['FirewallCollector'] += default_timer() - start
if router_metric.router_entry.monitor:
start = default_timer()
yield from MonitorCollector.collect(router_metric)
router_metric.time_spent['MonitorCollector'] += default_timer() - start
if router_metric.router_entry.route:
start = default_timer()
yield from RouteCollector.collect(router_metric)
router_metric.time_spent['RouteCollector'] += default_timer() - start
if router_metric.router_entry.wireless:
start = default_timer()
yield from WLANCollector.collect(router_metric)
router_metric.time_spent['WLANCollector'] += default_timer() - start
if router_metric.router_entry.capsman:
start = default_timer()
yield from CapsmanCollector.collect(router_metric)
router_metric.time_spent['CapsmanCollector'] += default_timer() - start
yield from MKTXPCollector.collect(router_metric)
return range(0)

48
mktxp/router_metric.py
View File

@@ -26,6 +26,18 @@ class RouterMetric:
MKTXPConfigKeys.ROUTERBOARD_NAME: self.router_name,
MKTXPConfigKeys.ROUTERBOARD_ADDRESS: self.router_entry.hostname
}
self.time_spent = { 'IdentityCollector': 0,
'SystemResourceCollector': 0,
'HealthCollector': 0,
'DHCPCollector': 0,
'PoolCollector': 0,
'InterfaceCollector': 0,
'FirewallCollector': 0,
'MonitorCollector': 0,
'RouteCollector': 0,
'WLANCollector': 0,
'CapsmanCollector': 0
}
def identity_records(self, identity_labels = []):
try:
@@ -79,7 +91,7 @@ class RouterMetric:
def interface_monitor_records(self, interface_monitor_labels = [], kind = 'ethernet', include_comments = False):
try:
interfaces = self.api_connection.router_api().get_resource(f'/interface/{kind}').get()
interface_names = [(interface['name'], interface.get('comment')) for interface in interfaces]
interface_names = [(interface['name'], interface.get('comment'), interface.get('running')) for interface in interfaces]
interface_monitor = lambda int_num : self.api_connection.router_api().get_resource(f'/interface/{kind}').call('monitor', {'once':'', 'numbers':f'{int_num}'})
interface_monitor_records = [interface_monitor(int_num)[0] for int_num in range(len(interface_names))]
@@ -144,8 +156,33 @@ class RouterMetric:
print(f'Error getting caps-man registration table info from router{self.router_name}@{self.router_entry.hostname}: {exc}')
return None
def firewall_records(self, firewall_labels = [], raw = False, matching_only = True):
try:
filter_path = '/ip/firewall/filter' if not raw else '/ip/firewall/raw'
firewall_records = self.api_connection.router_api().get_resource(filter_path).call('print', {'stats':'', 'all':''})
if matching_only:
firewall_records = [record for record in firewall_records if int(record.get('bytes', '0')) > 0]
# translation rules
translation_table = {}
# if 'id' in firewall_labels:
# translation_table['id'] = lambda id: str(int(id[1:], 16) - 1)
if 'comment' in firewall_labels:
translation_table['comment'] = lambda c: c if c else ''
return self._trimmed_records(firewall_records, firewall_labels, translation_table = translation_table)
except Exception as exc:
print(f'Error getting firewall filters info from router{self.router_name}@{self.router_entry.hostname}: {exc}')
return None
def mktxp_records(self):
mktxp_records = []
for key in self.time_spent.keys():
mktxp_records.append({'name': key, 'duration': self.time_spent[key]})
# translation rules
translation_table = {'duration': lambda d: d*1000}
return self._trimmed_records(mktxp_records, translation_table = translation_table)
# Helpers
def _trimmed_records(self, router_records, metric_labels, add_router_id = True):
def _trimmed_records(self, router_records, metric_labels = [], add_router_id = True, translation_table = {}):
if len(metric_labels) == 0 and len(router_records) > 0:
metric_labels = router_records[0].keys()
metric_labels = set(metric_labels)
@@ -157,9 +194,8 @@ class RouterMetric:
if add_router_id:
for key, value in self.router_id.items():
translated_record[key] = value
# translate fields if needed
for key, func in translation_table.items():
translated_record[key] = func(translated_record.get(key))
labeled_records.append(translated_record)
return labeled_records

12
mktxp/utils/utils.py
View File

@@ -13,6 +13,7 @@
import os, sys, shlex, tempfile, shutil, re
import subprocess, hashlib
from timeit import default_timer
from collections.abc import Iterable
from contextlib import contextmanager
from multiprocessing import Process, Event
@@ -35,6 +36,15 @@ def temp_dir(quiet = True):
if not quiet:
print ('Error while removing a tmp dir: {}'.format(e.args[0]))
class Benchmark:
def __enter__(self):
self.start = default_timer()
return self
def __exit__(self, *args):
self.time = default_timer() - self.start
class CmdProcessingError(Exception):
pass
@@ -253,5 +263,3 @@ class RepeatableTimer:
break
self.finished.wait(self.interval)