mirror of
https://github.com/KevinMidboe/mktxp-no-cli.git
synced 2025-10-29 17:50:23 +00:00
Collector registry, fixes/optimizations
This commit is contained in:
Arseniy Kuznetsov
38
mktxp/flow/collector_handler.py
Normal file
38
mktxp/flow/collector_handler.py
Normal file
@@ -0,0 +1,38 @@
|
||||
# coding=utf8
|
||||
## Copyright (c) 2020 Arseniy Kuznetsov
|
||||
##
|
||||
## This program is free software; you can redistribute it and/or
|
||||
## modify it under the terms of the GNU General Public License
|
||||
## as published by the Free Software Foundation; either version 2
|
||||
## of the License, or (at your option) any later version.
|
||||
##
|
||||
## This program is distributed in the hope that it will be useful,
|
||||
## but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
## GNU General Public License for more details.
|
||||
|
||||
|
||||
from timeit import default_timer
|
||||
|
||||
|
||||
class CollectorHandler:
|
||||
''' MKTXP Collectors Handler
|
||||
'''
|
||||
def __init__(self, entries_handler, collector_registry):
|
||||
self.entries_handler = entries_handler
|
||||
self.collector_registry = collector_registry
|
||||
|
||||
def collect(self):
|
||||
yield from self.collector_registry.bandwidthCollector.collect()
|
||||
|
||||
for router_entry in self.entries_handler.router_entries:
|
||||
if not router_entry.api_connection.is_connected():
|
||||
# let's pick up on things in the next run
|
||||
router_entry.api_connection.connect()
|
||||
continue
|
||||
|
||||
for collector_ID, collect_func in self.collector_registry.registered_collectors.items():
|
||||
start = default_timer()
|
||||
yield from collect_func(router_entry)
|
||||
router_entry.time_spent[collector_ID] += default_timer() - start
|
||||
|
||||
60
mktxp/flow/collector_registry.py
Normal file
60
mktxp/flow/collector_registry.py
Normal file
@@ -0,0 +1,60 @@
|
||||
# coding=utf8
|
||||
## Copyright (c) 2020 Arseniy Kuznetsov
|
||||
##
|
||||
## This program is free software; you can redistribute it and/or
|
||||
## modify it under the terms of the GNU General Public License
|
||||
## as published by the Free Software Foundation; either version 2
|
||||
## of the License, or (at your option) any later version.
|
||||
##
|
||||
## This program is distributed in the hope that it will be useful,
|
||||
## but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
## GNU General Public License for more details.
|
||||
|
||||
|
||||
from collections import OrderedDict
|
||||
from mktxp.cli.config.config import MKTXPConfigKeys
|
||||
from mktxp.collector.dhcp_collector import DHCPCollector
|
||||
from mktxp.collector.interface_collector import InterfaceCollector
|
||||
from mktxp.collector.health_collector import HealthCollector
|
||||
from mktxp.collector.identity_collector import IdentityCollector
|
||||
from mktxp.collector.monitor_collector import MonitorCollector
|
||||
from mktxp.collector.pool_collector import PoolCollector
|
||||
from mktxp.collector.resource_collector import SystemResourceCollector
|
||||
from mktxp.collector.route_collector import RouteCollector
|
||||
from mktxp.collector.wlan_collector import WLANCollector
|
||||
from mktxp.collector.capsman_collector import CapsmanCollector
|
||||
from mktxp.collector.bandwidth_collector import BandwidthCollector
|
||||
from mktxp.collector.firewall_collector import FirewallCollector
|
||||
from mktxp.collector.mktxp_collector import MKTXPCollector
|
||||
|
||||
|
||||
class CollectorRegistry:
|
||||
''' MKTXP Collectors Registry
|
||||
'''
|
||||
def __init__(self):
|
||||
self.registered_collectors = OrderedDict()
|
||||
|
||||
# bandwidth collector is not router-entry related, so registering directly
|
||||
self.bandwidthCollector = BandwidthCollector()
|
||||
|
||||
self.register('IdentityCollector', IdentityCollector.collect)
|
||||
self.register('SystemResourceCollector', SystemResourceCollector.collect)
|
||||
self.register('HealthCollector', HealthCollector.collect)
|
||||
|
||||
self.register('DHCPCollector', DHCPCollector.collect)
|
||||
self.register('PoolCollector', PoolCollector.collect)
|
||||
self.register('InterfaceCollector', InterfaceCollector.collect)
|
||||
|
||||
self.register('FirewallCollector', FirewallCollector.collect)
|
||||
self.register('MonitorCollector', MonitorCollector.collect)
|
||||
self.register('RouteCollector', RouteCollector.collect)
|
||||
|
||||
self.register('WLANCollector', WLANCollector.collect)
|
||||
self.register('CapsmanCollector', CapsmanCollector.collect)
|
||||
|
||||
self.register('MKTXPCollector', MKTXPCollector.collect)
|
||||
|
||||
def register(self, collector_ID, collect_func):
|
||||
self.registered_collectors[collector_ID] = collect_func
|
||||
|
||||
@@ -1,104 +0,0 @@
|
||||
# coding=utf8
|
||||
## Copyright (c) 2020 Arseniy Kuznetsov
|
||||
##
|
||||
## This program is free software; you can redistribute it and/or
|
||||
## modify it under the terms of the GNU General Public License
|
||||
## as published by the Free Software Foundation; either version 2
|
||||
## of the License, or (at your option) any later version.
|
||||
##
|
||||
## This program is distributed in the hope that it will be useful,
|
||||
## but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
## GNU General Public License for more details.
|
||||
|
||||
|
||||
from timeit import default_timer
|
||||
from mktxp.collector.dhcp_collector import DHCPCollector
|
||||
from mktxp.collector.interface_collector import InterfaceCollector
|
||||
from mktxp.collector.health_collector import HealthCollector
|
||||
from mktxp.collector.identity_collector import IdentityCollector
|
||||
from mktxp.collector.monitor_collector import MonitorCollector
|
||||
from mktxp.collector.pool_collector import PoolCollector
|
||||
from mktxp.collector.resource_collector import SystemResourceCollector
|
||||
from mktxp.collector.route_collector import RouteCollector
|
||||
from mktxp.collector.wlan_collector import WLANCollector
|
||||
from mktxp.collector.capsman_collector import CapsmanCollector
|
||||
from mktxp.collector.bandwidth_collector import BandwidthCollector
|
||||
from mktxp.collector.firewall_collector import FirewallCollector
|
||||
from mktxp.collector.mktxp_collector import MKTXPCollector
|
||||
|
||||
|
||||
class CollectorsHandler:
|
||||
''' MKTXP Collectors Handler
|
||||
'''
|
||||
def __init__(self, entries_handler):
|
||||
self.entries_handler = entries_handler
|
||||
self.bandwidthCollector = BandwidthCollector()
|
||||
|
||||
def collect(self):
|
||||
# process mktxp internal metrics
|
||||
yield from self.bandwidthCollector.collect()
|
||||
|
||||
for router_entry in self.entries_handler.router_entries:
|
||||
if not router_entry.api_connection.is_connected():
|
||||
# let's pick up on things in the next run
|
||||
router_entry.api_connection.connect()
|
||||
continue
|
||||
|
||||
start = default_timer()
|
||||
yield from IdentityCollector.collect(router_entry)
|
||||
router_entry.time_spent['IdentityCollector'] += default_timer() - start
|
||||
|
||||
start = default_timer()
|
||||
yield from SystemResourceCollector.collect(router_entry)
|
||||
router_entry.time_spent['SystemResourceCollector'] += default_timer() - start
|
||||
|
||||
start = default_timer()
|
||||
yield from HealthCollector.collect(router_entry)
|
||||
router_entry.time_spent['HealthCollector'] += default_timer() - start
|
||||
|
||||
if router_entry.config_entry.dhcp:
|
||||
start = default_timer()
|
||||
yield from DHCPCollector.collect(router_entry)
|
||||
router_entry.time_spent['DHCPCollector'] += default_timer() - start
|
||||
|
||||
if router_entry.config_entry.pool:
|
||||
start = default_timer()
|
||||
yield from PoolCollector.collect(router_entry)
|
||||
router_entry.time_spent['PoolCollector'] += default_timer() - start
|
||||
|
||||
if router_entry.config_entry.interface:
|
||||
start = default_timer()
|
||||
yield from InterfaceCollector.collect(router_entry)
|
||||
router_entry.time_spent['InterfaceCollector'] += default_timer() - start
|
||||
|
||||
if router_entry.config_entry.firewall:
|
||||
start = default_timer()
|
||||
yield from FirewallCollector.collect(router_entry)
|
||||
router_entry.time_spent['FirewallCollector'] += default_timer() - start
|
||||
|
||||
if router_entry.config_entry.monitor:
|
||||
start = default_timer()
|
||||
yield from MonitorCollector.collect(router_entry)
|
||||
router_entry.time_spent['MonitorCollector'] += default_timer() - start
|
||||
|
||||
if router_entry.config_entry.route:
|
||||
start = default_timer()
|
||||
yield from RouteCollector.collect(router_entry)
|
||||
router_entry.time_spent['RouteCollector'] += default_timer() - start
|
||||
|
||||
if router_entry.config_entry.wireless:
|
||||
start = default_timer()
|
||||
yield from WLANCollector.collect(router_entry)
|
||||
router_entry.time_spent['WLANCollector'] += default_timer() - start
|
||||
|
||||
if router_entry.config_entry.capsman:
|
||||
start = default_timer()
|
||||
yield from CapsmanCollector.collect(router_entry)
|
||||
router_entry.time_spent['CapsmanCollector'] += default_timer() - start
|
||||
|
||||
yield from MKTXPCollector.collect(router_entry)
|
||||
|
||||
|
||||
|
||||
|
||||
0
mktxp/flow/processor/__init__.py
Normal file
0
mktxp/flow/processor/__init__.py
Normal file
66
mktxp/flow/processor/base_proc.py
Normal file
66
mktxp/flow/processor/base_proc.py
Normal file
@@ -0,0 +1,66 @@
|
||||
# coding=utf8
|
||||
## Copyright (c) 2020 Arseniy Kuznetsov
|
||||
##
|
||||
## This program is free software; you can redistribute it and/or
|
||||
## modify it under the terms of the GNU General Public License
|
||||
## as published by the Free Software Foundation; either version 2
|
||||
## of the License, or (at your option) any later version.
|
||||
##
|
||||
## This program is distributed in the hope that it will be useful,
|
||||
## but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
## GNU General Public License for more details.
|
||||
|
||||
|
||||
from http.server import HTTPServer
|
||||
from datetime import datetime
|
||||
from prometheus_client.core import REGISTRY
|
||||
from prometheus_client import MetricsHandler
|
||||
|
||||
from mktxp.cli.config.config import config_handler
|
||||
from mktxp.flow.collector_handler import CollectorHandler
|
||||
from mktxp.flow.collector_registry import CollectorRegistry
|
||||
from mktxp.flow.router_entries_handler import RouterEntriesHandler
|
||||
|
||||
from mktxp.cli.output.capsman_out import CapsmanOutput
|
||||
from mktxp.cli.output.wifi_out import WirelessOutput
|
||||
from mktxp.cli.output.dhcp_out import DHCPOutput
|
||||
|
||||
|
||||
class ExportProcessor:
|
||||
''' Base Export Processing
|
||||
'''
|
||||
@staticmethod
|
||||
def start():
|
||||
REGISTRY.register(CollectorHandler(RouterEntriesHandler(), CollectorRegistry()))
|
||||
ExportProcessor.run(port=config_handler.system_entry().port)
|
||||
|
||||
@staticmethod
|
||||
def run(server_class=HTTPServer, handler_class=MetricsHandler, port=None):
|
||||
server_address = ('', port)
|
||||
httpd = server_class(server_address, handler_class)
|
||||
current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
print(f'{current_time} Running HTTP metrics server on port {port}')
|
||||
httpd.serve_forever()
|
||||
|
||||
|
||||
class OutputProcessor:
|
||||
''' Base CLI Processing
|
||||
'''
|
||||
@staticmethod
|
||||
def capsman_clients(entry_name):
|
||||
router_entry = RouterEntriesHandler.router_entry(entry_name)
|
||||
if router_entry:
|
||||
CapsmanOutput.clients_summary(router_entry)
|
||||
|
||||
@staticmethod
|
||||
def wifi_clients(entry_name):
|
||||
router_entry = RouterEntriesHandler.router_entry(entry_name)
|
||||
if router_entry:
|
||||
WirelessOutput.clients_summary(router_entry)
|
||||
|
||||
@staticmethod
|
||||
def dhcp_clients(entry_name):
|
||||
router_entry = RouterEntriesHandler.router_entry(entry_name)
|
||||
if router_entry:
|
||||
DHCPOutput.clients_summary(router_entry)
|
||||
131
mktxp/flow/processor/output.py
Normal file
131
mktxp/flow/processor/output.py
Normal file
@@ -0,0 +1,131 @@
|
||||
# coding=utf8
|
||||
## Copyright (c) 2020 Arseniy Kuznetsov
|
||||
##
|
||||
## This program is free software; you can redistribute it and/or
|
||||
## modify it under the terms of the GNU General Public License
|
||||
## as published by the Free Software Foundation; either version 2
|
||||
## of the License, or (at your option) any later version.
|
||||
##
|
||||
## This program is distributed in the hope that it will be useful,
|
||||
## but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
## GNU General Public License for more details.
|
||||
|
||||
|
||||
import re, os
|
||||
from datetime import timedelta
|
||||
from collections import namedtuple
|
||||
from texttable import Texttable
|
||||
from humanize import naturaldelta
|
||||
from mktxp.cli.config.config import config_handler
|
||||
|
||||
|
||||
class BaseOutputProcessor:
|
||||
OutputCapsmanEntry = namedtuple('OutputCapsmanEntry', ['dhcp_name', 'dhcp_address', 'mac_address', 'rx_signal', 'interface', 'ssid', 'tx_rate', 'rx_rate', 'uptime'])
|
||||
OutputCapsmanEntry.__new__.__defaults__ = ('',) * len(OutputCapsmanEntry._fields)
|
||||
|
||||
OutputWiFiEntry = namedtuple('OutputWiFiEntry', ['dhcp_name', 'dhcp_address', 'mac_address', 'signal_strength', 'signal_to_noise', 'interface', 'tx_rate', 'rx_rate', 'uptime'])
|
||||
OutputWiFiEntry.__new__.__defaults__ = ('',) * len(OutputWiFiEntry._fields)
|
||||
|
||||
OutputDHCPEntry = namedtuple('OutputDHCPEntry', ['host_name', 'server', 'mac_address', 'address', 'active_address', 'expires_after'])
|
||||
OutputDHCPEntry.__new__.__defaults__ = ('',) * len(OutputDHCPEntry._fields)
|
||||
|
||||
@staticmethod
|
||||
def augment_record(router_entry, registration_record, dhcp_lease_records):
|
||||
try:
|
||||
dhcp_lease_record = next((dhcp_lease_record for dhcp_lease_record in dhcp_lease_records if dhcp_lease_record['mac_address']==registration_record['mac_address']))
|
||||
dhcp_name = BaseOutputProcessor.dhcp_name(router_entry, dhcp_lease_record)
|
||||
dhcp_address = dhcp_lease_record.get('address', '')
|
||||
except StopIteration:
|
||||
dhcp_name = registration_record['mac_address']
|
||||
dhcp_address = 'No DHCP Record'
|
||||
|
||||
registration_record['dhcp_name'] = dhcp_name
|
||||
registration_record['dhcp_address'] = dhcp_address
|
||||
|
||||
# split out tx/rx bytes
|
||||
if registration_record.get('bytes'):
|
||||
registration_record['tx_bytes'] = registration_record['bytes'].split(',')[0]
|
||||
registration_record['rx_bytes'] = registration_record['bytes'].split(',')[1]
|
||||
del registration_record['bytes']
|
||||
|
||||
if registration_record.get('tx_rate'):
|
||||
registration_record['tx_rate'] = BaseOutputProcessor.parse_rates(registration_record['tx_rate'])
|
||||
if registration_record.get('rx_rate'):
|
||||
registration_record['rx_rate'] = BaseOutputProcessor.parse_rates(registration_record['rx_rate'])
|
||||
if registration_record.get('uptime'):
|
||||
registration_record['uptime'] = naturaldelta(BaseOutputProcessor.parse_timedelta_seconds(registration_record['uptime']), months=True, minimum_unit='seconds', when=None)
|
||||
|
||||
if registration_record.get('signal_strength'):
|
||||
registration_record['signal_strength'] = BaseOutputProcessor.parse_signal_strength(registration_record['signal_strength'])
|
||||
if registration_record.get('rx_signal'):
|
||||
registration_record['rx_signal'] = BaseOutputProcessor.parse_signal_strength(registration_record['rx_signal'])
|
||||
|
||||
@staticmethod
|
||||
def dhcp_name(router_entry, dhcp_lease_record, drop_comment = False):
|
||||
dhcp_name = dhcp_lease_record.get('host_name')
|
||||
dhcp_comment = dhcp_lease_record.get('comment')
|
||||
|
||||
if dhcp_name and dhcp_comment:
|
||||
dhcp_name = f'{dhcp_name[0:20]} ({dhcp_comment[0:20]})' if not router_entry.config_entry.use_comments_over_names else dhcp_comment
|
||||
elif dhcp_comment:
|
||||
dhcp_name = dhcp_comment
|
||||
else:
|
||||
dhcp_name = dhcp_lease_record.get('mac_address') if not dhcp_name else dhcp_name
|
||||
|
||||
if drop_comment:
|
||||
del dhcp_lease_record['comment']
|
||||
|
||||
return dhcp_name
|
||||
|
||||
@staticmethod
|
||||
def parse_rates(rate):
|
||||
wifi_rates_rgx = config_handler.re_compiled.get('wifi_rates_rgx')
|
||||
if not wifi_rates_rgx:
|
||||
wifi_rates_rgx = re.compile(r'(\d*(?:\.\d*)?)([GgMmKk]bps?)')
|
||||
config_handler.re_compiled['wifi_rates_rgx'] = wifi_rates_rgx
|
||||
rc = wifi_rates_rgx.search(rate)
|
||||
return f'{int(float(rc[1]))} {rc[2]}'
|
||||
|
||||
@staticmethod
|
||||
def parse_timedelta(time):
|
||||
duration_interval_rgx = config_handler.re_compiled.get('duration_interval_rgx')
|
||||
if not duration_interval_rgx:
|
||||
duration_interval_rgx = re.compile(r'((?P<weeks>\d+)w)?((?P<days>\d+)d)?((?P<hours>\d+)h)?((?P<minutes>\d+)m)?((?P<seconds>\d+)s)?')
|
||||
config_handler.re_compiled['duration_interval_rgx'] = duration_interval_rgx
|
||||
time_dict = duration_interval_rgx.match(time).groupdict()
|
||||
return timedelta(**{key: int(value) for key, value in time_dict.items() if value})
|
||||
|
||||
@staticmethod
|
||||
def parse_timedelta_seconds(time):
|
||||
return BaseOutputProcessor.parse_timedelta(time).total_seconds()
|
||||
|
||||
@staticmethod
|
||||
def parse_signal_strength(signal_strength):
|
||||
wifi_signal_strength_rgx = config_handler.re_compiled.get('wifi_signal_strength_rgx')
|
||||
if not wifi_signal_strength_rgx:
|
||||
# wifi_signal_strength_rgx = re.compile(r'(-?\d+(?:\.\d+)?)(dBm)?')
|
||||
wifi_signal_strength_rgx = re.compile(r'(-?\d+(?:\.\d+)?)')
|
||||
config_handler.re_compiled['wifi_signal_strength_rgx'] = wifi_signal_strength_rgx
|
||||
return wifi_signal_strength_rgx.search(signal_strength).group()
|
||||
|
||||
@staticmethod
|
||||
def parse_interface_rate(interface_rate):
|
||||
interface_rate_rgx = config_handler.re_compiled.get('interface_rate_rgx')
|
||||
if not interface_rate_rgx:
|
||||
interface_rate_rgx = re.compile(r'[^.\-\d]')
|
||||
config_handler.re_compiled['interface_rate_rgx'] = interface_rate_rgx
|
||||
rate = lambda interface_rate: 1000 if interface_rate.find('Mbps') < 0 else 1
|
||||
return(int(float(interface_rate_rgx.sub('', interface_rate)) * rate(interface_rate)))
|
||||
|
||||
@staticmethod
|
||||
def output_table(outputEntry = None):
|
||||
table = Texttable(max_width = os.get_terminal_size().columns)
|
||||
table.set_deco(Texttable.HEADER | Texttable.BORDER | Texttable.VLINES )
|
||||
if outputEntry:
|
||||
table.header(outputEntry._fields)
|
||||
table.set_cols_align(['l']+ ['c']*(len(outputEntry._fields)-1))
|
||||
return table
|
||||
|
||||
|
||||
|
||||
@@ -46,7 +46,7 @@ class RouterAPIConnection:
|
||||
ssl_verify = self.config_entry.ssl_certificate_verify,
|
||||
ssl_context = ctx)
|
||||
|
||||
self.connection.socket_timeout = config_handler._entry().socket_timeout
|
||||
self.connection.socket_timeout = config_handler.system_entry().socket_timeout
|
||||
self.api = None
|
||||
|
||||
def is_connected(self):
|
||||
@@ -91,7 +91,7 @@ class RouterAPIConnection:
|
||||
return False
|
||||
|
||||
def _connect_delay(self):
|
||||
mktxp_entry = config_handler._entry()
|
||||
mktxp_entry = config_handler.system_entry()
|
||||
connect_delay = (1 + self.successive_failure_count / mktxp_entry.delay_inc_div) * mktxp_entry.initial_delay_on_failure
|
||||
return connect_delay if connect_delay < mktxp_entry.max_delay_on_failure else mktxp_entry.max_delay_on_failure
|
||||
|
||||
|
||||
@@ -22,7 +22,7 @@ class RouterEntriesHandler:
|
||||
def __init__(self):
|
||||
self.router_entries = []
|
||||
for router_name in config_handler.registered_entries():
|
||||
entry = config_handler.entry(router_name)
|
||||
entry = config_handler.config_entry(router_name)
|
||||
if entry.enabled:
|
||||
self.router_entries.append(RouterEntry(router_name))
|
||||
|
||||
@@ -32,7 +32,7 @@ class RouterEntriesHandler:
|
||||
for router_name in config_handler.registered_entries():
|
||||
if router_name == entry_name:
|
||||
if enabled_only:
|
||||
entry = config_handler.entry(router_name)
|
||||
entry = config_handler.config_entry(router_name)
|
||||
if not entry.enabled:
|
||||
break
|
||||
router_entry = RouterEntry(router_name)
|
||||
|
||||
@@ -21,7 +21,7 @@ class RouterEntry:
|
||||
'''
|
||||
def __init__(self, router_name):
|
||||
self.router_name = router_name
|
||||
self.config_entry = config_handler.entry(router_name)
|
||||
self.config_entry = config_handler.config_entry(router_name)
|
||||
self.api_connection = RouterAPIConnection(router_name, self.config_entry)
|
||||
self.router_id = {
|
||||
MKTXPConfigKeys.ROUTERBOARD_NAME: self.router_name,
|
||||
@@ -37,5 +37,6 @@ class RouterEntry:
|
||||
'MonitorCollector': 0,
|
||||
'RouteCollector': 0,
|
||||
'WLANCollector': 0,
|
||||
'CapsmanCollector': 0
|
||||
'CapsmanCollector': 0,
|
||||
'MKTXPCollector': 0
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user