KevinMidboe/mktxp-no-cli
1
0
Fork 0
mirror of https://github.com/KevinMidboe/mktxp-no-cli.git synced 2025-10-29 17:50:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

connections stats collector / cmd output, remote dhcp resolver, fixes / optimizations

Browse Source
This commit is contained in:
Arseniy Kuznetsov
2023-02-04 20:48:51 +01:00
parent 18ddbe3311
commit 9a381d028c
21 changed files with 351 additions and 161 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
mktxp/flow/collector_handler.py
View File

@@ -1,15 +1,15 @@
# coding=utf8
# Copyright (c) 2020 Arseniy Kuznenowov
## Copyright (c) 2020 Arseniy Kuznetsov
##
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
## This program is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License
## as published by the Free Software Foundation; either version 2
## of the License, or (at your option) any later version.
##
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
## GNU General Public License for more details.
from concurrent.futures import ThreadPoolExecutor, as_completed
from timeit import default_timer
@@ -43,7 +43,7 @@ class CollectorHandler:
start = default_timer()
yield from collect_func(router_entry)
router_entry.time_spent[collector_ID] += default_timer() - start
router_entry.is_done()
def collect_router_entry_async(self, router_entry, scrape_timeout_event, total_scrape_timeout_event):
results = []
@@ -60,6 +60,7 @@ class CollectorHandler:
result = list(collect_func(router_entry))
results += result
router_entry.time_spent[collector_ID] += default_timer() - start
router_entry.is_done()
return results

44
mktxp/flow/collector_registry.py
View File

@@ -13,6 +13,7 @@
from collections import OrderedDict
from mktxp.cli.config.config import CollectorKeys
from mktxp.collector.dhcp_collector import DHCPCollector
from mktxp.collector.package_collector import PackageCollector
from mktxp.collector.connection_collector import IPConnectionCollector
@@ -36,7 +37,6 @@ from mktxp.collector.user_collector import UserCollector
from mktxp.collector.queue_collector import QueueTreeCollector
from mktxp.collector.queue_collector import QueueSimpleCollector
class CollectorRegistry:
''' MKTXP Collectors Registry
'''
@@ -46,33 +46,33 @@ class CollectorRegistry:
# bandwidth collector is not router-entry related, so registering directly
self.bandwidthCollector = BandwidthCollector()
self.register('IdentityCollector', IdentityCollector.collect)
self.register('SystemResourceCollector', SystemResourceCollector.collect)
self.register('HealthCollector', HealthCollector.collect)
self.register('PublicIPAddressCollector', PublicIPAddressCollector.collect)
self.register(CollectorKeys.IDENTITY_COLLECTOR, IdentityCollector.collect)
self.register(CollectorKeys.SYSTEM_RESOURCE_COLLECTOR, SystemResourceCollector.collect)
self.register(CollectorKeys.HEALTH_COLLECTOR, HealthCollector.collect)
self.register(CollectorKeys.PUBLIC_IP_ADDRESS_COLLECTOR, PublicIPAddressCollector.collect)
self.register('IPv6NeighborCollector', IPv6NeighborCollector.collect)
self.register(CollectorKeys.IPV6_NEIGHBOR_COLLECTOR, IPv6NeighborCollector.collect)
self.register('PackageCollector', PackageCollector.collect)
self.register('DHCPCollector', DHCPCollector.collect)
self.register('IPConnectionCollector', IPConnectionCollector.collect)
self.register('PoolCollector', PoolCollector.collect)
self.register('InterfaceCollector', InterfaceCollector.collect)
self.register(CollectorKeys.PACKAGE_COLLECTOR, PackageCollector.collect)
self.register(CollectorKeys.DHCP_COLLECTOR, DHCPCollector.collect)
self.register(CollectorKeys.IP_CONNECTION_COLLECTOR, IPConnectionCollector.collect)
self.register(CollectorKeys.POOL_COLLECTOR, PoolCollector.collect)
self.register(CollectorKeys.INTERFACE_COLLECTOR, InterfaceCollector.collect)
self.register('FirewallCollector', FirewallCollector.collect)
self.register('MonitorCollector', MonitorCollector.collect)
self.register('POECollector', POECollector.collect)
self.register('NetwatchCollector', NetwatchCollector.collect)
self.register('RouteCollector', RouteCollector.collect)
self.register(CollectorKeys.FIREWALL_COLLECTOR, FirewallCollector.collect)
self.register(CollectorKeys.MONITOR_COLLECTOR, MonitorCollector.collect)
self.register(CollectorKeys.POE_COLLECTOR, POECollector.collect)
self.register(CollectorKeys.NETWATCH_COLLECTOR, NetwatchCollector.collect)
self.register(CollectorKeys.ROUTE_COLLECTOR, RouteCollector.collect)
self.register('WLANCollector', WLANCollector.collect)
self.register('CapsmanCollector', CapsmanCollector.collect)
self.register(CollectorKeys.WLAN_COLLECTOR, WLANCollector.collect)
self.register(CollectorKeys.CAPSMAN_COLLECTOR, CapsmanCollector.collect)
self.register('UserCollector', UserCollector.collect)
self.register('QueueTreeCollector', QueueTreeCollector.collect)
self.register('QueueSimpleCollector', QueueSimpleCollector.collect)
self.register(CollectorKeys.USER_COLLECTOR, UserCollector.collect)
self.register(CollectorKeys.QUEUE_TREE_COLLECTOR, QueueTreeCollector.collect)
self.register(CollectorKeys.QUEUE_SIMPLE_COLLECTOR, QueueSimpleCollector.collect)
self.register('MKTXPCollector', MKTXPCollector.collect)
self.register(CollectorKeys.MKTXP_COLLECTOR, MKTXPCollector.collect)
def register(self, collector_ID, collect_func):
self.registered_collectors[collector_ID] = collect_func

9
mktxp/flow/processor/base_proc.py
View File

@@ -25,6 +25,7 @@ from mktxp.flow.router_entries_handler import RouterEntriesHandler
from mktxp.cli.output.capsman_out import CapsmanOutput
from mktxp.cli.output.wifi_out import WirelessOutput
from mktxp.cli.output.dhcp_out import DHCPOutput
from mktxp.cli.output.conn_stats_out import ConnectionsStatsOutput
class ExportProcessor:
@@ -64,3 +65,11 @@ class OutputProcessor:
router_entry = RouterEntriesHandler.router_entry(entry_name)
if router_entry:
DHCPOutput.clients_summary(router_entry)
@staticmethod
def conn_stats(entry_name):
router_entry = RouterEntriesHandler.router_entry(entry_name)
if router_entry:
ConnectionsStatsOutput.clients_summary(router_entry)

38
mktxp/flow/processor/output.py
View File

@@ -19,6 +19,7 @@ from texttable import Texttable
from humanize import naturaldelta
from mktxp.cli.config.config import config_handler
from mktxp.datasource.wireless_ds import WirelessMetricsDataSource
from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource
from math import floor, log
@@ -35,20 +36,13 @@ class BaseOutputProcessor:
OutputDHCPEntry = namedtuple('OutputDHCPEntry', ['host_name', 'server', 'mac_address', 'address', 'active_address', 'expires_after'])
OutputDHCPEntry.__new__.__defaults__ = ('',) * len(OutputDHCPEntry._fields)
@staticmethod
def augment_record(router_entry, registration_record, dhcp_lease_records):
dhcp_name = registration_record.get('mac_address')
dhcp_address = 'No DHCP Record'
if dhcp_lease_records:
try:
dhcp_lease_record = next((dhcp_lease_record for dhcp_lease_record in dhcp_lease_records if dhcp_lease_record.get('mac_address')==registration_record.get('mac_address')))
dhcp_name = BaseOutputProcessor.dhcp_name(router_entry, dhcp_lease_record)
dhcp_address = dhcp_lease_record.get('address', '')
except StopIteration:
pass
OutputConnStatsEntry = namedtuple('OutputConnStatsEntry', ['dhcp_name', 'src_address', 'connection_count', 'dst_addresses'])
OutputConnStatsEntry.__new__.__defaults__ = ('',) * len(OutputConnStatsEntry._fields)
registration_record['dhcp_name'] = dhcp_name
registration_record['dhcp_address'] = dhcp_address
@staticmethod
def augment_record(router_entry, registration_record, id_key = 'mac_address'):
BaseOutputProcessor.resolve_dhcp(router_entry, registration_record, id_key)
# split out tx/rx bytes
if registration_record.get('bytes'):
@@ -85,9 +79,25 @@ class BaseOutputProcessor:
if drop_comment:
del dhcp_lease_record['comment']
return dhcp_name
@staticmethod
def resolve_dhcp(router_entry, registration_record, id_key = 'mac_address', resolve_address = True):
if not router_entry.dhcp_records:
DHCPMetricsDataSource.metric_records(router_entry)
dhcp_name = registration_record.get(id_key)
dhcp_address = 'No DHCP Record'
dhcp_lease_record = router_entry.dhcp_record(dhcp_name)
if dhcp_lease_record:
dhcp_name = BaseOutputProcessor.dhcp_name(router_entry, dhcp_lease_record)
dhcp_address = dhcp_lease_record.get('address', '')
registration_record['dhcp_name'] = dhcp_name
if resolve_address:
registration_record['dhcp_address'] = dhcp_address
@staticmethod
def parse_rates(rate):
wifi_rates_rgx = config_handler.re_compiled.get('wifi_rates_rgx')

44
mktxp/flow/router_entries_handler.py
View File

@@ -20,24 +20,34 @@ class RouterEntriesHandler:
''' Handles RouterOS entries defined in MKTXP config
'''
def __init__(self):
self.router_entries = []
for router_name in config_handler.registered_entries():
router_entry = RouterEntriesHandler.router_entry(router_name, enabled_only = True)
if router_entry:
self.router_entries.append(router_entry)
self._router_entries = {}
for router_name in config_handler.registered_entries():
router_entry = RouterEntry(router_name)
if router_entry.config_entry.remote_dhcp_entry and config_handler.registered_entry(router_entry.config_entry.remote_dhcp_entry):
router_entry.dhcp_entry = RouterEntry(router_entry.config_entry.remote_dhcp_entry)
self._router_entries[router_name] = router_entry
@property
def router_entries(self):
return (entry for key, entry in self._router_entries.items() if entry.config_entry.enabled) \
if self._router_entries else None
def router_entry(self, entry_name, enabled_only = False):
entry = self._router_entries.get(entry_name)
if entry and (entry.config_entry.enabled or not enabled_only):
return entry
return None
@staticmethod
def router_entry(entry_name, enabled_only = False):
router_entry = None
for router_name in config_handler.registered_entries():
if router_name == entry_name:
config_entry = config_handler.config_entry(router_name)
if enabled_only and not config_entry.enabled:
break
router_entry = RouterEntry(router_name)
router_entry.dhcp_entry = RouterEntriesHandler.router_entry(config_entry.remote_dhcp_entry)
break
''' A static router entry initialiser
'''
config_entry = config_handler.config_entry(entry_name)
if enabled_only and not config_entry.enabled:
return None
router_entry = RouterEntry(entry_name)
if config_entry.remote_dhcp_entry and config_handler.registered_entry(config_entry.remote_dhcp_entry):
router_entry.dhcp_entry = RouterEntry(config_entry.remote_dhcp_entry)
return router_entry

89
mktxp/flow/router_entry.py
View File

@@ -12,7 +12,8 @@
## GNU General Public License for more details.
from mktxp.cli.config.config import config_handler, MKTXPConfigKeys
from collections import namedtuple
from mktxp.cli.config.config import config_handler, MKTXPConfigKeys, CollectorKeys
from mktxp.flow.router_connection import RouterAPIConnection
@@ -29,39 +30,71 @@ class RouterEntry:
}
self.wifi_package = None
self.dhcp_entry = None
self.time_spent = { CollectorKeys.IDENTITY_COLLECTOR: 0,
CollectorKeys.SYSTEM_RESOURCE_COLLECTOR: 0,
CollectorKeys.HEALTH_COLLECTOR: 0,
CollectorKeys.PUBLIC_IP_ADDRESS_COLLECTOR: 0,
CollectorKeys.IPV6_NEIGHBOR_COLLECTOR: 0,
CollectorKeys.PACKAGE_COLLECTOR: 0,
CollectorKeys.DHCP_COLLECTOR: 0,
CollectorKeys.POOL_COLLECTOR: 0,
CollectorKeys.IP_CONNECTION_COLLECTOR: 0,
CollectorKeys.INTERFACE_COLLECTOR: 0,
CollectorKeys.FIREWALL_COLLECTOR: 0,
CollectorKeys.MONITOR_COLLECTOR: 0,
CollectorKeys.POE_COLLECTOR: 0,
CollectorKeys.NETWATCH_COLLECTOR: 0,
CollectorKeys.ROUTE_COLLECTOR: 0,
CollectorKeys.WLAN_COLLECTOR: 0,
CollectorKeys.CAPSMAN_COLLECTOR: 0,
CollectorKeys.QUEUE_TREE_COLLECTOR: 0,
CollectorKeys.QUEUE_SIMPLE_COLLECTOR: 0,
CollectorKeys.USER_COLLECTOR: 0,
CollectorKeys.MKTXP_COLLECTOR: 0
}
self._dhcp_entry = None
self._dhcp_records = {}
@property
def dhcp_entry(self):
if self._dhcp_entry:
return self._dhcp_entry
return self
@dhcp_entry.setter
def dhcp_entry(self, dhcp_entry):
self._dhcp_entry = dhcp_entry
self.time_spent = { 'IdentityCollector': 0,
'SystemResourceCollector': 0,
'HealthCollector': 0,
'PublicIPAddressCollector': 0,
'IPv6NeighborCollector': 0,
'PackageCollector': 0,
'DHCPCollector': 0,
'PoolCollector': 0,
'IPConnectionCollector': 0,
'InterfaceCollector': 0,
'FirewallCollector': 0,
'MonitorCollector': 0,
'POECollector': 0,
'NetwatchCollector': 0,
'RouteCollector': 0,
'WLANCollector': 0,
'CapsmanCollector': 0,
'QueueTreeCollector': 0,
'QueueSimpleCollector': 0,
'UserCollector': 0,
'MKTXPCollector': 0
}
@property
def dhcp_records(self):
return (entry.record for key, entry in self._dhcp_records.items() if entry.type == 'mac_address') \
if self._dhcp_records else None
@dhcp_records.setter
def dhcp_records(self, dhcp_records):
for dhcp_record in dhcp_records:
if dhcp_record.get('mac_address'):
self._dhcp_records[dhcp_record.get('mac_address')] = DHCPCacheEntry('mac_address', dhcp_record)
if dhcp_record.get('address'):
dhcp_record['type'] = 'address'
self._dhcp_records[dhcp_record.get('address')] = DHCPCacheEntry('address', dhcp_record)
def dhcp_record(self, key):
if self._dhcp_records and self._dhcp_records.get(key):
return self._dhcp_records[key].record
return None
def is_ready(self):
self.is_done() #flush caches, just in case
is_ready = True
self.wifi_package = None
if not self.api_connection.is_connected():
is_ready = False
# let's get connected now
self.api_connection.connect()
if self.dhcp_entry:
self.dhcp_entry.api_connection.connect()
if self._dhcp_entry:
self._dhcp_entry.api_connection.connect()
return is_ready
def is_done(self):
self.wifi_package = None
self._dhcp_records = {}
DHCPCacheEntry = namedtuple('DHCPCacheEntry', ['type', 'record'])