KevinMidboe/mktxp-no-cli
1
0
Fork 0
mirror of https://github.com/KevinMidboe/mktxp-no-cli.git synced 2025-12-07 20:08:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

DS refactor, fixes/optimizations

Browse Source
This commit is contained in:
Arseniy Kuznetsov
2021-02-06 15:12:35 +01:00
parent 52bfe9c16d
commit f405d58410
44 changed files with 998 additions and 559 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
mktxp/collectors/bandwidth_collector.py
View File

@@ -12,16 +12,19 @@
## GNU General Public License for more details.
import socket
import speedtest
from datetime import datetime
from multiprocessing import Pool
from mktxp.cli.config.config import config_handler
from mktxp.collectors.base_collector import BaseCollector
result_list = [{'download': 0, 'upload': 0, 'ping': 0}]
def get_result(bandwidth_dict):
result_list[0] = bandwidth_dict
class BandwidthCollector(BaseCollector):
''' MKTXP collector
'''
@@ -53,8 +56,21 @@ class BandwidthCollector(BaseCollector):
@staticmethod
def bandwidth_worker():
bandwidth_test = speedtest.Speedtest()
bandwidth_test.get_best_server()
bandwidth_test.download()
bandwidth_test.upload()
return bandwidth_test.results.dict()
if BandwidthCollector.inet_connected():
bandwidth_test = speedtest.Speedtest()
bandwidth_test.get_best_server()
bandwidth_test.download()
bandwidth_test.upload()
return bandwidth_test.results.dict()
else:
return {'download': 0, 'upload': 0, 'ping': 0}
@staticmethod
def inet_connected(host="8.8.8.8", port=53, timeout=3):
try:
socket.setdefaulttimeout(timeout)
socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((host, port))
return True
except socket.error as exc:
return False

1
mktxp/collectors/base_collector.py
View File

@@ -15,6 +15,7 @@
from prometheus_client.core import GaugeMetricFamily, CounterMetricFamily, InfoMetricFamily
from mktxp.cli.config.config import MKTXPConfigKeys
class BaseCollector:
''' Base Collector methods
For use by custom collectors

20
mktxp/collectors/capsman_collector.py
View File

@@ -11,43 +11,47 @@
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
## GNU General Public License for more details.
from mktxp.cli.output.base_out import BaseOutputProcessor
from mktxp.cli.config.config import MKTXPConfigKeys
from mktxp.collectors.base_collector import BaseCollector
from mktxp.datasources.dhcp_ds import DHCPMetricsDataSource
from mktxp.datasources.capsman_ds import CapsmanCapsMetricsDataSource, CapsmanRegistrationsMetricsDataSource
class CapsmanCollector(BaseCollector):
''' CAPsMAN Metrics collector
'''
@staticmethod
def collect(router_metric):
def collect(router_entry):
remote_caps_labels = ['identity', 'version', 'base_mac', 'board', 'base_mac']
remote_caps_records = router_metric.capsman_remote_caps_records(remote_caps_labels)
remote_caps_records = CapsmanCapsMetricsDataSource.metric_records(router_entry, metric_labels = remote_caps_labels)
if remote_caps_records:
remote_caps_metrics = BaseCollector.info_collector('capsman_remote_caps', 'CAPsMAN remote caps', remote_caps_records, remote_caps_labels)
yield remote_caps_metrics
registration_labels = ['interface', 'ssid', 'mac_address', 'tx_rate', 'rx_rate', 'rx_signal', 'uptime', 'bytes']
registration_records = router_metric.capsman_registration_table_records(registration_labels)
registration_records = CapsmanRegistrationsMetricsDataSource.metric_records(router_entry, metric_labels = registration_labels)
if registration_records:
# calculate number of registrations per interface
registration_per_interface = {}
for registration_record in registration_records:
registration_per_interface[registration_record['interface']] = registration_per_interface.get(registration_record['interface'], 0) + 1
# compile registrations-per-interface records
registration_per_interface_records = [{ MKTXPConfigKeys.ROUTERBOARD_NAME: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
registration_per_interface_records = [{ MKTXPConfigKeys.ROUTERBOARD_NAME: router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
'interface': key, 'count': value} for key, value in registration_per_interface.items()]
# yield registrations-per-interface metrics
registration_per_interface_metrics = BaseCollector.gauge_collector('capsman_registrations_count', 'Number of active registration per CAPsMAN interface', registration_per_interface_records, 'count', ['interface'])
yield registration_per_interface_metrics
# the client info metrics
if router_metric.router_entry.capsman_clients:
if router_entry.config_entry.capsman_clients:
# translate / trim / augment registration records
dhcp_lease_labels = ['mac_address', 'address', 'host_name', 'comment']
dhcp_lease_records = router_metric.dhcp_lease_records(dhcp_lease_labels)
dhcp_lease_records = DHCPMetricsDataSource.metric_records(router_entry, metric_labels = dhcp_lease_labels)
for registration_record in registration_records:
BaseOutputProcessor.augment_record(router_metric, registration_record, dhcp_lease_records)
BaseOutputProcessor.augment_record(router_entry, registration_record, dhcp_lease_records)
tx_byte_metrics = BaseCollector.counter_collector('capsman_clients_tx_bytes', 'Number of sent packet bytes', registration_records, 'tx_bytes', ['dhcp_name'])
yield tx_byte_metrics

14
mktxp/collectors/dhcp_collector.py
View File

@@ -11,17 +11,19 @@
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
## GNU General Public License for more details.
from mktxp.cli.config.config import MKTXPConfigKeys
from mktxp.collectors.base_collector import BaseCollector
from mktxp.datasources.dhcp_ds import DHCPMetricsDataSource
class DHCPCollector(BaseCollector):
''' DHCP Metrics collector
'''
@staticmethod
def collect(router_metric):
dhcp_lease_labels = ['active_address', 'mac_address', 'host_name', 'comment', 'server', 'expires_after']
dhcp_lease_records = router_metric.dhcp_lease_records(dhcp_lease_labels)
def collect(router_entry):
dhcp_lease_labels = ['active_address', 'address', 'mac_address', 'host_name', 'comment', 'server', 'expires_after']
dhcp_lease_records = DHCPMetricsDataSource.metric_records(router_entry, metric_labels = dhcp_lease_labels)
if dhcp_lease_records:
# calculate number of leases per DHCP server
dhcp_lease_servers = {}
@@ -29,8 +31,8 @@ class DHCPCollector(BaseCollector):
dhcp_lease_servers[dhcp_lease_record['server']] = dhcp_lease_servers.get(dhcp_lease_record['server'], 0) + 1
# compile leases-per-server records
dhcp_lease_servers_records = [{ MKTXPConfigKeys.ROUTERBOARD_NAME: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
dhcp_lease_servers_records = [{ MKTXPConfigKeys.ROUTERBOARD_NAME: router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
'server': key, 'count': value} for key, value in dhcp_lease_servers.items()]
# yield lease-per-server metrics
@@ -38,7 +40,7 @@ class DHCPCollector(BaseCollector):
yield dhcp_lease_server_metrics
# active lease metrics
if router_metric.router_entry.dhcp_lease:
if router_entry.config_entry.dhcp_lease:
dhcp_lease_metrics = BaseCollector.info_collector('dhcp_lease', 'DHCP Active Leases', dhcp_lease_records, dhcp_lease_labels)
yield dhcp_lease_metrics

21
mktxp/collectors/firewall_collector.py
View File

@@ -11,34 +11,37 @@
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
## GNU General Public License for more details.
from mktxp.collectors.base_collector import BaseCollector
from mktxp.cli.config.config import MKTXPConfigKeys
from mktxp.collectors.base_collector import BaseCollector
from mktxp.datasources.firewall_ds import FirewallMetricsDataSource
class FirewallCollector(BaseCollector):
''' Firewall rules traffic metrics collector
'''
@staticmethod
def collect(router_metric):
def collect(router_entry):
# initialize all pool counts, including those currently not used
firewall_labels = ['chain', 'action', 'bytes', 'comment']
firewall_filter_records = router_metric.firewall_records(firewall_labels)
firewall_filter_records = FirewallMetricsDataSource.metric_records(router_entry, metric_labels = firewall_labels)
if firewall_filter_records:
metris_records = [FirewallCollector.metric_record(router_metric, record) for record in firewall_filter_records]
metris_records = [FirewallCollector.metric_record(router_entry, record) for record in firewall_filter_records]
firewall_filter_metrics = BaseCollector.counter_collector('firewall_filter', 'Total amount of bytes matched by firewall rules', metris_records, 'bytes', ['name'])
yield firewall_filter_metrics
firewall_raw_records = router_metric.firewall_records(firewall_labels, raw = True)
firewall_raw_records = FirewallMetricsDataSource.metric_records(router_entry, metric_labels = firewall_labels, raw = True)
if firewall_raw_records:
metris_records = [FirewallCollector.metric_record(router_metric, record) for record in firewall_raw_records]
metris_records = [FirewallCollector.metric_record(router_entry, record) for record in firewall_raw_records]
firewall_raw_metrics = BaseCollector.counter_collector('firewall_raw', 'Total amount of bytes matched by raw firewall rules', metris_records, 'bytes', ['name'])
yield firewall_raw_metrics
# Helpers
@staticmethod
def metric_record(router_metric, firewall_record):
def metric_record(router_entry, firewall_record):
name = f"| {firewall_record['chain']} | {firewall_record['action']} | {firewall_record['comment']}"
bytes = firewall_record['bytes']
return {MKTXPConfigKeys.ROUTERBOARD_NAME: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
return {MKTXPConfigKeys.ROUTERBOARD_NAME: router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
'name': name, 'bytes': bytes}

6
mktxp/collectors/health_collector.py
View File

@@ -11,16 +11,18 @@
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
## GNU General Public License for more details.
from mktxp.collectors.base_collector import BaseCollector
from mktxp.datasources.health_ds import HealthMetricsDataSource
class HealthCollector(BaseCollector):
''' System Health Metrics collector
'''
@staticmethod
def collect(router_metric):
def collect(router_entry):
health_labels = ['voltage', 'temperature']
health_records = router_metric.health_records(health_labels)
health_records = HealthMetricsDataSource.metric_records(router_entry, metric_labels = health_labels)
if health_records:
voltage_metrics = BaseCollector.gauge_collector('system_routerboard_voltage', 'Supplied routerboard voltage', health_records, 'voltage')
yield voltage_metrics

7
mktxp/collectors/identity_collector.py
View File

@@ -11,15 +11,18 @@
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
## GNU General Public License for more details.
from mktxp.collectors.base_collector import BaseCollector
from mktxp.datasources.identity_ds import IdentityMetricsDataSource
class IdentityCollector(BaseCollector):
''' System Identity Metrics collector
'''
@staticmethod
def collect(router_metric):
def collect(router_entry):
identity_labels = ['name']
identity_records = router_metric.identity_records(identity_labels)
identity_records = IdentityMetricsDataSource.metric_records(router_entry, metric_labels = identity_labels)
if identity_records:
identity_metrics = BaseCollector.info_collector('system_identity', 'System identity', identity_records, identity_labels)
yield identity_metrics

11
mktxp/collectors/interface_collector.py
View File

@@ -11,20 +11,23 @@
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
## GNU General Public License for more details.
from mktxp.collectors.base_collector import BaseCollector
from mktxp.datasources.interface_ds import InterfaceTrafficMetricsDataSource
class InterfaceCollector(BaseCollector):
''' Router Interface Metrics collector
'''
@staticmethod
def collect(router_metric):
def collect(router_entry):
interface_traffic_labels = ['name', 'comment', 'rx_byte', 'tx_byte', 'rx_packet', 'tx_packet', 'rx_error', 'tx_error', 'rx_drop', 'tx_drop']
interface_traffic_records = router_metric.interface_traffic_records(interface_traffic_labels)
interface_traffic_records = InterfaceTrafficMetricsDataSource.metric_records(router_entry, metric_labels = interface_traffic_labels)
if interface_traffic_records:
for interface_traffic_record in interface_traffic_records:
if interface_traffic_record.get('comment'):
interface_traffic_record['name'] = interface_traffic_record['comment'] if router_metric.router_entry.use_comments_over_names \
interface_traffic_record['name'] = interface_traffic_record['comment'] if router_entry.config_entry.use_comments_over_names \
else f"{interface_traffic_record['name']} ({interface_traffic_record['comment']})"
rx_byte_metric = BaseCollector.counter_collector('interface_rx_byte', 'Number of received bytes', interface_traffic_records, 'rx_byte', ['name'])

7
mktxp/collectors/mktxp_collector.py
View File

@@ -11,14 +11,17 @@
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
## GNU General Public License for more details.
from mktxp.collectors.base_collector import BaseCollector
from mktxp.datasources.mktxp_ds import MKTXPMetricsDataSource
class MKTXPCollector(BaseCollector):
''' System Identity Metrics collector
'''
@staticmethod
def collect(router_metric):
mktxp_records = router_metric.mktxp_records()
def collect(router_entry):
mktxp_records = MKTXPMetricsDataSource.metric_records(router_entry)
if mktxp_records:
mktxp_duration_metric = BaseCollector.counter_collector('collection_time', 'Total time spent collecting metrics in milliseconds', mktxp_records, 'duration', ['name'])
yield mktxp_duration_metric

7
mktxp/collectors/monitor_collector.py
View File

@@ -11,16 +11,19 @@
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
## GNU General Public License for more details.
from mktxp.collectors.base_collector import BaseCollector
from mktxp.cli.output.base_out import BaseOutputProcessor
from mktxp.datasources.interface_ds import InterfaceMonitorMetricsDataSource
class MonitorCollector(BaseCollector):
''' Ethernet Interface Monitor Metrics collector
'''
@staticmethod
def collect(router_metric):
def collect(router_entry):
monitor_labels = ('status', 'rate', 'full_duplex', 'name')
monitor_records = router_metric.interface_monitor_records(monitor_labels, include_comments = True)
monitor_records = InterfaceMonitorMetricsDataSource.metric_records(router_entry, metric_labels = monitor_labels, include_comments = True)
if monitor_records:
# translate records to appropriate values
for monitor_record in monitor_records:

13
mktxp/collectors/pool_collector.py
View File

@@ -11,28 +11,31 @@
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
## GNU General Public License for more details.
from mktxp.cli.config.config import MKTXPConfigKeys
from mktxp.collectors.base_collector import BaseCollector
from mktxp.datasources.pool_ds import PoolMetricsDataSource, PoolUsedMetricsDataSource
class PoolCollector(BaseCollector):
''' IP Pool Metrics collector
'''
@staticmethod
def collect(router_metric):
def collect(router_entry):
# initialize all pool counts, including those currently not used
pool_records = router_metric.pool_records(['name'])
pool_records = PoolMetricsDataSource.metric_records(router_entry, metric_labels = ['name'])
if pool_records:
pool_used_labels = ['pool']
pool_used_counts = {pool_record['name']: 0 for pool_record in pool_records}
# for pools in usage, calculate the current numbers
pool_used_records = router_metric.pool_used_records(pool_used_labels)
pool_used_records = PoolUsedMetricsDataSource.metric_records(router_entry, metric_labels = pool_used_labels)
for pool_used_record in pool_used_records:
pool_used_counts[pool_used_record['pool']] = pool_used_counts.get(pool_used_record['pool'], 0) + 1
# compile used-per-pool records
used_per_pool_records = [{ MKTXPConfigKeys.ROUTERBOARD_NAME: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
used_per_pool_records = [{ MKTXPConfigKeys.ROUTERBOARD_NAME: router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
'pool': key, 'count': value} for key, value in pool_used_counts.items()]
# yield used-per-pool metrics

9
mktxp/collectors/resource_collector.py
View File

@@ -11,20 +11,23 @@
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
## GNU General Public License for more details.
from mktxp.collectors.base_collector import BaseCollector
from mktxp.cli.output.base_out import BaseOutputProcessor
from mktxp.datasources.system_resource_ds import SystemResourceMetricsDataSource
class SystemResourceCollector(BaseCollector):
''' System Resource Metrics collector
'''
@staticmethod
def collect(router_metric):
def collect(router_entry):
resource_labels = ['uptime', 'version', 'free_memory', 'total_memory',
'cpu', 'cpu_count', 'cpu_frequency', 'cpu_load',
'free_hdd_space', 'total_hdd_space',
'architecture_name', 'board_name']
resource_records = router_metric.system_resource_records(resource_labels)
resource_records = SystemResourceMetricsDataSource.metric_records(router_entry, metric_labels = resource_labels)
if resource_records:
# translate records to appropriate values
translated_fields = ['uptime']

15
mktxp/collectors/route_collector.py
View File

@@ -11,21 +11,24 @@
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
## GNU General Public License for more details.
from mktxp.cli.config.config import MKTXPConfigKeys
from mktxp.collectors.base_collector import BaseCollector
from mktxp.datasources.route_ds import RouteMetricsDataSource
class RouteCollector(BaseCollector):
''' IP Route Metrics collector
'''
@staticmethod
def collect(router_metric):
def collect(router_entry):
route_labels = ['connect', 'dynamic', 'static', 'bgp', 'ospf']
route_records = router_metric.route_records(route_labels)
route_records = RouteMetricsDataSource.metric_records(router_entry, metric_labels = route_labels)
if route_records:
# compile total routes records
total_routes = len(route_records)
total_routes_records = [{ MKTXPConfigKeys.ROUTERBOARD_NAME: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
total_routes_records = [{ MKTXPConfigKeys.ROUTERBOARD_NAME: router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
'count': total_routes
}]
total_routes_metrics = BaseCollector.gauge_collector('routes_total_routes', 'Overall number of routes in RIB', total_routes_records, 'count')
@@ -40,8 +43,8 @@ class RouteCollector(BaseCollector):
routes_per_protocol[route_label] += 1
# compile route-per-protocol records
route_per_protocol_records = [{ MKTXPConfigKeys.ROUTERBOARD_NAME: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
route_per_protocol_records = [{ MKTXPConfigKeys.ROUTERBOARD_NAME: router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
'protocol': key, 'count': value} for key, value in routes_per_protocol.items()]
# yield route-per-protocol metrics

17
mktxp/collectors/wlan_collector.py
View File

@@ -11,16 +11,21 @@
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
## GNU General Public License for more details.
from mktxp.cli.output.base_out import BaseOutputProcessor
from mktxp.collectors.base_collector import BaseCollector
from mktxp.datasources.dhcp_ds import DHCPMetricsDataSource
from mktxp.datasources.wireless_ds import WirelessMetricsDataSource
from mktxp.datasources.interface_ds import InterfaceMonitorMetricsDataSource
class WLANCollector(BaseCollector):
''' Wireless Metrics collector
'''
@staticmethod
def collect(router_metric):
def collect(router_entry):
monitor_labels = ['channel', 'noise_floor', 'overall_tx_ccq', 'registered_clients']
monitor_records = router_metric.interface_monitor_records(monitor_labels, 'wireless')
monitor_records = InterfaceMonitorMetricsDataSource.metric_records(router_entry, metric_labels = monitor_labels, kind = 'wireless')
if monitor_records:
# sanitize records for relevant labels
noise_floor_records = [monitor_record for monitor_record in monitor_records if monitor_record.get('noise_floor')]
@@ -40,15 +45,15 @@ class WLANCollector(BaseCollector):
yield registered_clients_metrics
# the client info metrics
if router_metric.router_entry.wireless_clients:
if router_entry.config_entry.wireless_clients:
registration_labels = ['interface', 'ssid', 'mac_address', 'tx_rate', 'rx_rate', 'uptime', 'bytes', 'signal_to_noise', 'tx_ccq', 'signal_strength']
registration_records = router_metric.wireless_registration_table_records(registration_labels)
registration_records = WirelessMetricsDataSource.metric_records(router_entry, metric_labels = registration_labels)
if registration_records:
dhcp_lease_labels = ['mac_address', 'address', 'host_name', 'comment']
dhcp_lease_records = router_metric.dhcp_lease_records(dhcp_lease_labels)
dhcp_lease_records = DHCPMetricsDataSource.metric_records(router_entry, metric_labels = dhcp_lease_labels)
for registration_record in registration_records:
BaseOutputProcessor.augment_record(router_metric, registration_record, dhcp_lease_records)
BaseOutputProcessor.augment_record(router_entry, registration_record, dhcp_lease_records)
tx_byte_metrics = BaseCollector.counter_collector('wlan_clients_tx_bytes', 'Number of sent packet bytes', registration_records, 'tx_bytes', ['dhcp_name'])
yield tx_byte_metrics