KevinMidboe/mktxp-no-cli
1
0
Fork 0
mirror of https://github.com/KevinMidboe/mktxp-no-cli.git synced 2025-10-29 09:40:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

connections stats collector / cmd output, remote dhcp resolver, fixes / optimizations

Browse Source
This commit is contained in:
Arseniy Kuznetsov
2023-02-04 20:48:51 +01:00
parent 18ddbe3311
commit 9a381d028c
21 changed files with 351 additions and 161 deletions
Download Patch File Download Diff File Expand all files Collapse all files

73
mktxp/cli/config/config.py
View File

@@ -24,6 +24,29 @@ from mktxp.utils.utils import FSHelper
''' MKTXP conf file handling
'''
class CollectorKeys:
IDENTITY_COLLECTOR = 'IdentityCollector'
SYSTEM_RESOURCE_COLLECTOR = 'SystemResourceCollector'
HEALTH_COLLECTOR = 'HealthCollector'
PUBLIC_IP_ADDRESS_COLLECTOR = 'PublicIPAddressCollector'
IPV6_NEIGHBOR_COLLECTOR = 'IPv6NeighborCollector'
PACKAGE_COLLECTOR = 'PackageCollector'
DHCP_COLLECTOR = 'DHCPCollector'
POOL_COLLECTOR = 'PoolCollector'
IP_CONNECTION_COLLECTOR = 'IPConnectionCollector'
INTERFACE_COLLECTOR = 'InterfaceCollector'
FIREWALL_COLLECTOR = 'FirewallCollector'
MONITOR_COLLECTOR = 'MonitorCollector'
POE_COLLECTOR = 'POECollector'
NETWATCH_COLLECTOR = 'NetwatchCollector'
ROUTE_COLLECTOR = 'RouteCollector'
WLAN_COLLECTOR = 'WLANCollector'
CAPSMAN_COLLECTOR = 'CapsmanCollector'
QUEUE_TREE_COLLECTOR = 'QueueTreeCollector'
QUEUE_SIMPLE_COLLECTOR = 'QueueSimpleCollector'
USER_COLLECTOR = 'UserCollector'
MKTXP_COLLECTOR = 'MKTXPCollector'
class MKTXPConfigKeys:
''' MKTXP config file keys
@@ -44,6 +67,7 @@ class MKTXPConfigKeys:
FE_DHCP_LEASE_KEY = 'dhcp_lease'
FE_DHCP_POOL_KEY = 'pool'
FE_IP_CONNECTIONS_KEY = 'connections'
FE_CONNECTION_STATS_KEY = 'connection_stats'
FE_INTERFACE_KEY = 'interface'
FE_FIREWALL_KEY = 'firewall'
@@ -104,7 +128,7 @@ class MKTXPConfigKeys:
BOOLEAN_KEYS_NO = {ENABLED_KEY, SSL_KEY, NO_SSL_CERTIFICATE,
SSL_CERTIFICATE_VERIFY, FE_IPV6_FIREWALL_KEY, FE_IPV6_NEIGHBOR_KEY}
SSL_CERTIFICATE_VERIFY, FE_IPV6_FIREWALL_KEY, FE_IPV6_NEIGHBOR_KEY, FE_CONNECTION_STATS_KEY}
# Feature keys enabled by default
BOOLEAN_KEYS_YES = {FE_DHCP_KEY, FE_PACKAGE_KEY, FE_DHCP_LEASE_KEY, FE_DHCP_POOL_KEY, FE_IP_CONNECTIONS_KEY, FE_INTERFACE_KEY, FE_FIREWALL_KEY,
@@ -116,6 +140,7 @@ class MKTXPConfigKeys:
SYSTEM_BOOLEAN_KEYS_NO = {MKTXP_VERBOSE_MODE, MKTXP_FETCH_IN_PARALLEL}
STR_KEYS = (HOST_KEY, USER_KEY, PASSWD_KEY, FE_REMOTE_DHCP_ENTRY)
INT_KEYS = ()
MKTXP_INT_KEYS = (PORT_KEY, MKTXP_SOCKET_TIMEOUT, MKTXP_INITIAL_DELAY, MKTXP_MAX_DELAY,
MKTXP_INC_DIV, MKTXP_BANDWIDTH_TEST_INTERVAL, MKTXP_MIN_COLLECT_INTERVAL,
MKTXP_MAX_WORKER_THREADS, MKTXP_MAX_SCRAPE_DURATION, MKTXP_TOTAL_MAX_SCRAPE_DURATION)
@@ -130,7 +155,7 @@ class ConfigEntry:
MKTXPConfigKeys.SSL_KEY, MKTXPConfigKeys.NO_SSL_CERTIFICATE, MKTXPConfigKeys.SSL_CERTIFICATE_VERIFY,
MKTXPConfigKeys.FE_DHCP_KEY, MKTXPConfigKeys.FE_PACKAGE_KEY, MKTXPConfigKeys.FE_DHCP_LEASE_KEY, MKTXPConfigKeys.FE_DHCP_POOL_KEY, MKTXPConfigKeys.FE_INTERFACE_KEY,
MKTXPConfigKeys.FE_FIREWALL_KEY, MKTXPConfigKeys.FE_MONITOR_KEY, MKTXPConfigKeys.FE_ROUTE_KEY, MKTXPConfigKeys.FE_WIRELESS_KEY, MKTXPConfigKeys.FE_WIRELESS_CLIENTS_KEY,
MKTXPConfigKeys.FE_IP_CONNECTIONS_KEY, MKTXPConfigKeys.FE_CAPSMAN_KEY, MKTXPConfigKeys.FE_CAPSMAN_CLIENTS_KEY, MKTXPConfigKeys.FE_POE_KEY, MKTXPConfigKeys.FE_NETWATCH_KEY,
MKTXPConfigKeys.FE_IP_CONNECTIONS_KEY, MKTXPConfigKeys.FE_CONNECTION_STATS_KEY, MKTXPConfigKeys.FE_CAPSMAN_KEY, MKTXPConfigKeys.FE_CAPSMAN_CLIENTS_KEY, MKTXPConfigKeys.FE_POE_KEY, MKTXPConfigKeys.FE_NETWATCH_KEY,
MKTXPConfigKeys.MKTXP_USE_COMMENTS_OVER_NAMES, MKTXPConfigKeys.FE_PUBLIC_IP_KEY, MKTXPConfigKeys.FE_IPV6_FIREWALL_KEY, MKTXPConfigKeys.FE_IPV6_NEIGHBOR_KEY,
MKTXPConfigKeys.FE_USER_KEY, MKTXPConfigKeys.FE_QUEUE_KEY, MKTXPConfigKeys.FE_REMOTE_DHCP_ENTRY
])
@@ -235,17 +260,18 @@ class MKTXPConfigHandler:
def registered_entries(self):
''' All MKTXP registered entries
'''
registered_entries = [entry_name for entry_name in self.config.keys()]
if not registered_entries:
registered_entries = [MKTXPConfigKeys.NO_ENTRIES_REGISTERED]
return (entry_name for entry_name in self.config.keys())
return registered_entries
def registered_entry(self, entry_name):
''' A specific MKTXP registered entry by name
'''
return self.config.get(entry_name)
def config_entry(self, entry_name):
''' Given an entry name, reads and returns the entry info
'''
entry_reader = self._config_entry_reader(entry_name)
return ConfigEntry.MKTXPConfigEntry(**entry_reader)
return ConfigEntry.MKTXPConfigEntry(**entry_reader) if entry_reader else None
def system_entry(self):
''' MKTXP internal config entry
@@ -258,7 +284,10 @@ class MKTXPConfigHandler:
''' (Force-)Read conf data from disk
'''
self.config = ConfigObj(self.usr_conf_data_path)
self.config.preserve_comments = True
self._config = ConfigObj(self.mktxp_conf_path)
self._config.preserve_comments = True
def _create_os_path(self, os_path, resource_path):
if not os.path.exists(os_path):
@@ -272,7 +301,7 @@ class MKTXPConfigHandler:
new_keys = []
for key in MKTXPConfigKeys.BOOLEAN_KEYS_NO.union(MKTXPConfigKeys.BOOLEAN_KEYS_YES):
if self.config[entry_name].get(key):
if self.config[entry_name].get(key) is not None:
config_entry_reader[key] = self.config[entry_name].as_bool(key)
else:
config_entry_reader[key] = True if key in MKTXPConfigKeys.BOOLEAN_KEYS_YES else False
@@ -288,6 +317,13 @@ class MKTXPConfigHandler:
if key is MKTXPConfigKeys.PASSWD_KEY and type(config_entry_reader[key]) is list:
config_entry_reader[key] = ','.join(config_entry_reader[key])
for key in MKTXPConfigKeys.INT_KEYS:
if self.config[entry_name].get(key):
config_entry_reader[key] = self.config[entry_name].as_int(key)
else:
config_entry_reader[key] = self._default_value_for_key(key)
new_keys.append(key) # read from disk next time
# port
if self.config[entry_name].get(MKTXPConfigKeys.PORT_KEY):
config_entry_reader[MKTXPConfigKeys.PORT_KEY] = self.config[entry_name].as_int(
@@ -322,9 +358,8 @@ class MKTXPConfigHandler:
new_keys.append(key) # read from disk next time
for key in MKTXPConfigKeys.SYSTEM_BOOLEAN_KEYS_NO.union(MKTXPConfigKeys.SYSTEM_BOOLEAN_KEYS_YES):
if self._config[entry_name].get(key):
system_entry_reader[key] = self._config[entry_name].as_bool(
key)
if self._config[entry_name].get(key) is not None:
system_entry_reader[key] = self._config[entry_name].as_bool(key)
else:
system_entry_reader[key] = True if key in MKTXPConfigKeys.SYSTEM_BOOLEAN_KEYS_YES else False
new_keys.append(key) # read from disk next time
@@ -344,14 +379,14 @@ class MKTXPConfigHandler:
def _default_value_for_key(self, key, value=None):
return {
MKTXPConfigKeys.SSL_KEY: lambda value: MKTXPConfigKeys.DEFAULT_API_SSL_PORT if value else MKTXPConfigKeys.DEFAULT_API_PORT,
MKTXPConfigKeys.PORT_KEY: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_PORT,
MKTXPConfigKeys.FE_REMOTE_DHCP_ENTRY: lambda value: MKTXPConfigKeys.DEFAULT_FE_REMOTE_DHCP_ENTRY,
MKTXPConfigKeys.MKTXP_SOCKET_TIMEOUT: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_SOCKET_TIMEOUT,
MKTXPConfigKeys.MKTXP_INITIAL_DELAY: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_INITIAL_DELAY,
MKTXPConfigKeys.MKTXP_MAX_DELAY: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_MAX_DELAY,
MKTXPConfigKeys.MKTXP_INC_DIV: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_INC_DIV,
MKTXPConfigKeys.MKTXP_BANDWIDTH_TEST_INTERVAL: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_BANDWIDTH_TEST_INTERVAL,
MKTXPConfigKeys.MKTXP_MIN_COLLECT_INTERVAL: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_MIN_COLLECT_INTERVAL,
MKTXPConfigKeys.PORT_KEY: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_PORT,
MKTXPConfigKeys.FE_REMOTE_DHCP_ENTRY: lambda _: MKTXPConfigKeys.DEFAULT_FE_REMOTE_DHCP_ENTRY,
MKTXPConfigKeys.MKTXP_SOCKET_TIMEOUT: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_SOCKET_TIMEOUT,
MKTXPConfigKeys.MKTXP_INITIAL_DELAY: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_INITIAL_DELAY,
MKTXPConfigKeys.MKTXP_MAX_DELAY: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_MAX_DELAY,
MKTXPConfigKeys.MKTXP_INC_DIV: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_INC_DIV,
MKTXPConfigKeys.MKTXP_BANDWIDTH_TEST_INTERVAL: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_BANDWIDTH_TEST_INTERVAL,
MKTXPConfigKeys.MKTXP_MIN_COLLECT_INTERVAL: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_MIN_COLLECT_INTERVAL,
MKTXPConfigKeys.MKTXP_FETCH_IN_PARALLEL: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_FETCH_IN_PARALLEL,
MKTXPConfigKeys.MKTXP_MAX_WORKER_THREADS: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_MAX_WORKER_THREADS,
MKTXPConfigKeys.MKTXP_MAX_SCRAPE_DURATION: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_MAX_SCRAPE_DURATION,

3
mktxp/cli/config/mktxp.conf
View File

@@ -27,7 +27,10 @@
installed_packages = True # Installed packages
dhcp = True # DHCP general metrics
dhcp_lease = True # DHCP lease metrics
connections = True # IP connections metrics
connection_stats = False # Open IP connections metrics
pool = True # Pool metrics
interface = True # Interfaces traffic metrics

3
mktxp/cli/dispatch.py
View File

@@ -98,6 +98,9 @@ class MKTXPDispatcher:
elif args['dhcp_clients']:
OutputProcessor.dhcp_clients(args['entry_name'])
elif args['conn_stats']:
OutputProcessor.conn_stats(args['entry_name'])
else:
print("Select metric option(s) to print out, or run 'mktxp print -h' to find out more")

5
mktxp/cli/options.py
View File

@@ -146,6 +146,11 @@ Selected metrics info can be printed on the command line. For more information,
help = "DHCP clients metrics",
action = 'store_true')
optional_args_group.add_argument('-cn', '--conn_stats', dest='conn_stats',
help = "IP connections stats",
action = 'store_true')
# Options checking
def _check_args(self, args, parser):
''' Validation of supplied CLI arguments

8
mktxp/cli/output/capsman_out.py
View File

@@ -13,8 +13,6 @@
from mktxp.flow.processor.output import BaseOutputProcessor
from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource
from mktxp.datasource.wireless_ds import WirelessMetricsDataSource
from mktxp.datasource.capsman_ds import CapsmanRegistrationsMetricsDataSource
class CapsmanOutput:
@@ -29,13 +27,9 @@ class CapsmanOutput:
return
# translate / trim / augment registration records
dhcp_lease_labels = ['host_name', 'comment', 'address', 'mac_address']
dhcp_entry = WirelessMetricsDataSource.dhcp_entry(router_entry)
dhcp_lease_records = DHCPMetricsDataSource.metric_records(dhcp_entry, metric_labels = dhcp_lease_labels, add_router_id = False)
dhcp_rt_by_interface = {}
for registration_record in sorted(registration_records, key = lambda rt_record: rt_record['rx_signal'], reverse=True):
BaseOutputProcessor.augment_record(router_entry, registration_record, dhcp_lease_records)
BaseOutputProcessor.augment_record(router_entry, registration_record)
interface = registration_record['interface']
if interface in dhcp_rt_by_interface.keys():

49
mktxp/cli/output/conn_stats_out.py Normal file
View File

@@ -0,0 +1,49 @@
# coding=utf8
## Copyright (c) 2020 Arseniy Kuznetsov
##
## This program is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License
## as published by the Free Software Foundation; either version 2
## of the License, or (at your option) any later version.
##
## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
## GNU General Public License for more details.
from mktxp.flow.processor.output import BaseOutputProcessor
from mktxp.datasource.connection_ds import IPConnectionStatsDatasource
class ConnectionsStatsOutput:
''' Connections Stats Output
'''
@staticmethod
def clients_summary(router_entry):
connection_records = IPConnectionStatsDatasource.metric_records(router_entry, add_router_id = False)
if not connection_records:
print('No connection stats records')
return
conn_cnt = 0
output_records = []
for registration_record in sorted(connection_records, key = lambda rt_record: rt_record['connection_count'], reverse=True):
BaseOutputProcessor.resolve_dhcp(router_entry, registration_record, id_key = 'src_address', resolve_address = False)
output_records.append(registration_record)
conn_cnt += registration_record['connection_count']
output_records_cnt = 0
output_entry = BaseOutputProcessor.OutputConnStatsEntry
output_table = BaseOutputProcessor.output_table(output_entry)
for record in output_records:
output_table.add_row(output_entry(**record))
output_table.add_row(output_entry())
output_records_cnt += 1
print (output_table.draw())
print(f'Distinct source addresses: {output_records_cnt}')
print(f'Total open connections: {conn_cnt}', '\n')

4
mktxp/cli/output/dhcp_out.py
View File

@@ -22,13 +22,13 @@ class DHCPOutput:
@staticmethod
def clients_summary(router_entry):
dhcp_lease_labels = ['host_name', 'comment', 'active_address', 'address', 'mac_address', 'server', 'expires_after']
dhcp_lease_records = DHCPMetricsDataSource.metric_records(router_entry, metric_labels = dhcp_lease_labels, add_router_id = False)
dhcp_lease_records = DHCPMetricsDataSource.metric_records(router_entry, metric_labels = dhcp_lease_labels, add_router_id = False, translate = False, dhcp_cache = False)
if not dhcp_lease_records:
print('No DHCP registration records')
return
dhcp_by_server = {}
for dhcp_lease_record in sorted(dhcp_lease_records, key = lambda dhcp_record: dhcp_record['active_address'], reverse=True):
for dhcp_lease_record in sorted(dhcp_lease_records, key = lambda dhcp_record: dhcp_record['address'], reverse=True):
server = dhcp_lease_record.get('server', 'all')
if server == 'all':
dhcp_lease_record['server'] = server

8
mktxp/cli/output/wifi_out.py
View File

@@ -13,9 +13,7 @@
from mktxp.flow.processor.output import BaseOutputProcessor
from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource
from mktxp.datasource.wireless_ds import WirelessMetricsDataSource
from mktxp.flow.router_entries_handler import RouterEntriesHandler
class WirelessOutput:
@@ -30,15 +28,11 @@ class WirelessOutput:
return
# translate / trim / augment registration records
dhcp_lease_labels = ['host_name', 'comment', 'address', 'mac_address']
dhcp_entry = WirelessMetricsDataSource.dhcp_entry(router_entry)
dhcp_lease_records = DHCPMetricsDataSource.metric_records(dhcp_entry, metric_labels = dhcp_lease_labels, add_router_id = False)
dhcp_rt_by_interface = {}
key = lambda rt_record: rt_record['signal_strength'] if rt_record.get('signal_strength') else rt_record['interface']
for registration_record in sorted(registration_records, key = key, reverse=True):
BaseOutputProcessor.augment_record(router_entry, registration_record, dhcp_lease_records)
BaseOutputProcessor.augment_record(router_entry, registration_record)
interface = registration_record['interface']
if interface in dhcp_rt_by_interface.keys():

6
mktxp/collector/capsman_collector.py
View File

@@ -15,7 +15,6 @@
from mktxp.cli.config.config import MKTXPConfigKeys
from mktxp.flow.processor.output import BaseOutputProcessor
from mktxp.collector.base_collector import BaseCollector
from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource
from mktxp.datasource.capsman_ds import CapsmanCapsMetricsDataSource, CapsmanRegistrationsMetricsDataSource, CapsmanInterfacesDatasource
from mktxp.datasource.wireless_ds import WirelessMetricsDataSource
@@ -53,11 +52,8 @@ class CapsmanCollector(BaseCollector):
if router_entry.config_entry.capsman_clients:
# translate / trim / augment registration records
dhcp_lease_labels = ['mac_address', 'address', 'host_name', 'comment']
dhcp_entry = WirelessMetricsDataSource.dhcp_entry(router_entry)
dhcp_lease_records = DHCPMetricsDataSource.metric_records(dhcp_entry, metric_labels = dhcp_lease_labels)
for registration_record in registration_records:
BaseOutputProcessor.augment_record(router_entry, registration_record, dhcp_lease_records)
BaseOutputProcessor.augment_record(router_entry, registration_record)
tx_byte_metrics = BaseCollector.counter_collector('capsman_clients_tx_bytes', 'Number of sent packet bytes', registration_records, 'tx_bytes', ['dhcp_name'])
yield tx_byte_metrics

26
mktxp/collector/connection_collector.py
View File

@@ -13,7 +13,8 @@
from mktxp.collector.base_collector import BaseCollector
from mktxp.datasource.connection_ds import IPConnectionDatasource
from mktxp.flow.processor.output import BaseOutputProcessor
from mktxp.datasource.connection_ds import IPConnectionDatasource, IPConnectionStatsDatasource
class IPConnectionCollector(BaseCollector):
@@ -21,11 +22,20 @@ class IPConnectionCollector(BaseCollector):
'''
@staticmethod
def collect(router_entry):
if not router_entry.config_entry.connections:
return
connection_records = IPConnectionDatasource.metric_records(router_entry)
if connection_records:
if router_entry.config_entry.connections:
connection_records = IPConnectionDatasource.metric_records(router_entry)
if connection_records:
connection_metrics = BaseCollector.gauge_collector('ip_connections_total', 'Number of IP connections', connection_records, 'count',)
yield connection_metrics
if router_entry.config_entry.connection_stats:
connection_stats_records = IPConnectionStatsDatasource.metric_records(router_entry)
for connection_stat_record in connection_stats_records:
BaseOutputProcessor.augment_record(router_entry, connection_stat_record, id_key = 'src_address')
connection_stats_labels = ['src_address', 'dst_addresses', 'dhcp_name']
connection_stats_metrics_gauge = BaseCollector.gauge_collector('connection_stats', 'Open connection stats',
connection_stats_records, 'connection_count', connection_stats_labels)
yield connection_stats_metrics_gauge
connection_metrics = BaseCollector.gauge_collector('ip_connections_total', 'Number of IP connections', connection_records, 'count',)
yield connection_metrics

7
mktxp/collector/wlan_collector.py
View File

@@ -14,7 +14,6 @@
from mktxp.flow.processor.output import BaseOutputProcessor
from mktxp.collector.base_collector import BaseCollector
from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource
from mktxp.datasource.wireless_ds import WirelessMetricsDataSource
from mktxp.datasource.interface_ds import InterfaceMonitorMetricsDataSource
@@ -52,12 +51,8 @@ class WLANCollector(BaseCollector):
registration_labels = ['interface', 'ssid', 'mac_address', 'tx_rate', 'rx_rate', 'uptime', 'bytes', 'signal_to_noise', 'tx_ccq', 'signal_strength', 'signal']
registration_records = WirelessMetricsDataSource.metric_records(router_entry, metric_labels = registration_labels)
if registration_records:
dhcp_lease_labels = ['mac_address', 'address', 'host_name', 'comment']
dhcp_entry = WirelessMetricsDataSource.dhcp_entry(router_entry)
dhcp_lease_records = DHCPMetricsDataSource.metric_records(dhcp_entry, metric_labels = dhcp_lease_labels)
for registration_record in registration_records:
BaseOutputProcessor.augment_record(router_entry, registration_record, dhcp_lease_records)
BaseOutputProcessor.augment_record(router_entry, registration_record)
tx_byte_metrics = BaseCollector.counter_collector('wlan_clients_tx_bytes', 'Number of sent packet bytes', registration_records, 'tx_bytes', ['dhcp_name'])
yield tx_byte_metrics

42
mktxp/datasource/connection_ds.py
View File

@@ -12,6 +12,7 @@
## GNU General Public License for more details.
from collections import namedtuple
from mktxp.datasource.base_ds import BaseDSProcessor
@@ -33,3 +34,44 @@ class IPConnectionDatasource:
except Exception as exc:
print(f'Error getting IP connection info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}')
return None
class IPConnectionStatsDatasource:
''' IP connections stats data provider
'''
@staticmethod
def metric_records(router_entry, *, metric_labels = None, add_router_id = True):
if metric_labels is None:
metric_labels = []
try:
connection_records = router_entry.api_connection.router_api().get_resource('/ip/firewall/connection/').call('print', \
{'proplist':'src-address,dst-address,protocol'})
# calculate number of connections per src-address
connections_per_src_address = {}
for connection_record in connection_records:
#address, port = (connection_record['src-address'].split(':') + [None])[:2]
address = connection_record['src-address'].split(':')[0]
destination = f"{connection_record.get('dst-address')}({connection_record.get('protocol')})"
count, destinations = 0, set()
if connections_per_src_address.get(address):
count, destinations = connections_per_src_address[address]
count += 1
destinations.add(destination)
connections_per_src_address[address] = ConnStatsEntry(count, destinations)
# compile connections-per-interface records
records = []
for key, entry in connections_per_src_address.items():
record = {'src_address': key, 'connection_count': entry.count, 'dst_addresses': ', '.join(entry.destinations)}
if add_router_id:
for router_key, router_value in router_entry.router_id.items():
record[router_key] = router_value
records.append(record)
return records
except Exception as exc:
print(f'Error getting IP connection stats info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}')
return None
ConnStatsEntry = namedtuple('ConnStatsEntry', ['count', 'destinations'])

23
mktxp/datasource/dhcp_ds.py
View File

@@ -20,12 +20,17 @@ class DHCPMetricsDataSource:
''' DHCP Metrics data provider
'''
@staticmethod
def metric_records(router_entry, *, metric_labels = None, add_router_id = True):
if metric_labels is None:
metric_labels = []
def metric_records(router_entry, *, metric_labels = None, add_router_id = True, dhcp_cache = True, translate = True, bound = False):
if metric_labels is None or dhcp_cache:
metric_labels = ['host_name', 'comment', 'active_address', 'address', 'mac_address', 'server', 'expires_after']
if dhcp_cache and router_entry.dhcp_records:
return router_entry.dhcp_records
try:
#dhcp_lease_records = router_entry.api_connection.router_api().get_resource('/ip/dhcp-server/lease').get(status='bound')
dhcp_lease_records = router_entry.api_connection.router_api().get_resource('/ip/dhcp-server/lease').call('print', {'active':''})
if bound:
dhcp_lease_records = router_entry.dhcp_entry.api_connection.router_api().get_resource('/ip/dhcp-server/lease').get(status='bound')
else:
dhcp_lease_records = router_entry.dhcp_entry.api_connection.router_api().get_resource('/ip/dhcp-server/lease').call('print', {'active':''})
# translation rules
translation_table = {}
@@ -33,12 +38,16 @@ class DHCPMetricsDataSource:
translation_table['comment'] = lambda c: c if c else ''
if 'host_name' in metric_labels:
translation_table['host_name'] = lambda c: c if c else ''
if 'expires_after' in metric_labels:
if 'expires_after' in metric_labels and translate:
translation_table['expires_after'] = lambda c: parse_mkt_uptime(c) if c else 0
if 'active_address' in metric_labels:
translation_table['active_address'] = lambda c: c if c else ''
return BaseDSProcessor.trimmed_records(router_entry, router_records = dhcp_lease_records, metric_labels = metric_labels, add_router_id = add_router_id, translation_table = translation_table)
records = BaseDSProcessor.trimmed_records(router_entry, router_records = dhcp_lease_records, metric_labels = metric_labels, add_router_id = add_router_id, translation_table = translation_table)
if dhcp_cache:
router_entry.dhcp_records = records
return records
except Exception as exc:
print(f'Error getting dhcp info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}')
return None

8
mktxp/datasource/wireless_ds.py
View File

@@ -52,11 +52,3 @@ class WirelessMetricsDataSource:
@staticmethod
def wifiwave2_installed(router_entry):
return WirelessMetricsDataSource.wireless_package(router_entry) == WirelessMetricsDataSource.WIFIWAVE2
@staticmethod
def dhcp_entry(router_entry):
if router_entry.dhcp_entry:
return router_entry.dhcp_entry
return router_entry

21
mktxp/flow/collector_handler.py
View File

@@ -1,15 +1,15 @@
# coding=utf8
# Copyright (c) 2020 Arseniy Kuznenowov
## Copyright (c) 2020 Arseniy Kuznetsov
##
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
## This program is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License
## as published by the Free Software Foundation; either version 2
## of the License, or (at your option) any later version.
##
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
## GNU General Public License for more details.
from concurrent.futures import ThreadPoolExecutor, as_completed
from timeit import default_timer
@@ -43,7 +43,7 @@ class CollectorHandler:
start = default_timer()
yield from collect_func(router_entry)
router_entry.time_spent[collector_ID] += default_timer() - start
router_entry.is_done()
def collect_router_entry_async(self, router_entry, scrape_timeout_event, total_scrape_timeout_event):
results = []
@@ -60,6 +60,7 @@ class CollectorHandler:
result = list(collect_func(router_entry))
results += result
router_entry.time_spent[collector_ID] += default_timer() - start
router_entry.is_done()
return results

44
mktxp/flow/collector_registry.py
View File

@@ -13,6 +13,7 @@
from collections import OrderedDict
from mktxp.cli.config.config import CollectorKeys
from mktxp.collector.dhcp_collector import DHCPCollector
from mktxp.collector.package_collector import PackageCollector
from mktxp.collector.connection_collector import IPConnectionCollector
@@ -36,7 +37,6 @@ from mktxp.collector.user_collector import UserCollector
from mktxp.collector.queue_collector import QueueTreeCollector
from mktxp.collector.queue_collector import QueueSimpleCollector
class CollectorRegistry:
''' MKTXP Collectors Registry
'''
@@ -46,33 +46,33 @@ class CollectorRegistry:
# bandwidth collector is not router-entry related, so registering directly
self.bandwidthCollector = BandwidthCollector()
self.register('IdentityCollector', IdentityCollector.collect)
self.register('SystemResourceCollector', SystemResourceCollector.collect)
self.register('HealthCollector', HealthCollector.collect)
self.register('PublicIPAddressCollector', PublicIPAddressCollector.collect)
self.register(CollectorKeys.IDENTITY_COLLECTOR, IdentityCollector.collect)
self.register(CollectorKeys.SYSTEM_RESOURCE_COLLECTOR, SystemResourceCollector.collect)
self.register(CollectorKeys.HEALTH_COLLECTOR, HealthCollector.collect)
self.register(CollectorKeys.PUBLIC_IP_ADDRESS_COLLECTOR, PublicIPAddressCollector.collect)
self.register('IPv6NeighborCollector', IPv6NeighborCollector.collect)
self.register(CollectorKeys.IPV6_NEIGHBOR_COLLECTOR, IPv6NeighborCollector.collect)
self.register('PackageCollector', PackageCollector.collect)
self.register('DHCPCollector', DHCPCollector.collect)
self.register('IPConnectionCollector', IPConnectionCollector.collect)
self.register('PoolCollector', PoolCollector.collect)
self.register('InterfaceCollector', InterfaceCollector.collect)
self.register(CollectorKeys.PACKAGE_COLLECTOR, PackageCollector.collect)
self.register(CollectorKeys.DHCP_COLLECTOR, DHCPCollector.collect)
self.register(CollectorKeys.IP_CONNECTION_COLLECTOR, IPConnectionCollector.collect)
self.register(CollectorKeys.POOL_COLLECTOR, PoolCollector.collect)
self.register(CollectorKeys.INTERFACE_COLLECTOR, InterfaceCollector.collect)
self.register('FirewallCollector', FirewallCollector.collect)
self.register('MonitorCollector', MonitorCollector.collect)
self.register('POECollector', POECollector.collect)
self.register('NetwatchCollector', NetwatchCollector.collect)
self.register('RouteCollector', RouteCollector.collect)
self.register(CollectorKeys.FIREWALL_COLLECTOR, FirewallCollector.collect)
self.register(CollectorKeys.MONITOR_COLLECTOR, MonitorCollector.collect)
self.register(CollectorKeys.POE_COLLECTOR, POECollector.collect)
self.register(CollectorKeys.NETWATCH_COLLECTOR, NetwatchCollector.collect)
self.register(CollectorKeys.ROUTE_COLLECTOR, RouteCollector.collect)
self.register('WLANCollector', WLANCollector.collect)
self.register('CapsmanCollector', CapsmanCollector.collect)
self.register(CollectorKeys.WLAN_COLLECTOR, WLANCollector.collect)
self.register(CollectorKeys.CAPSMAN_COLLECTOR, CapsmanCollector.collect)
self.register('UserCollector', UserCollector.collect)
self.register('QueueTreeCollector', QueueTreeCollector.collect)
self.register('QueueSimpleCollector', QueueSimpleCollector.collect)
self.register(CollectorKeys.USER_COLLECTOR, UserCollector.collect)
self.register(CollectorKeys.QUEUE_TREE_COLLECTOR, QueueTreeCollector.collect)
self.register(CollectorKeys.QUEUE_SIMPLE_COLLECTOR, QueueSimpleCollector.collect)
self.register('MKTXPCollector', MKTXPCollector.collect)
self.register(CollectorKeys.MKTXP_COLLECTOR, MKTXPCollector.collect)
def register(self, collector_ID, collect_func):
self.registered_collectors[collector_ID] = collect_func

9
mktxp/flow/processor/base_proc.py
View File

@@ -25,6 +25,7 @@ from mktxp.flow.router_entries_handler import RouterEntriesHandler
from mktxp.cli.output.capsman_out import CapsmanOutput
from mktxp.cli.output.wifi_out import WirelessOutput
from mktxp.cli.output.dhcp_out import DHCPOutput
from mktxp.cli.output.conn_stats_out import ConnectionsStatsOutput
class ExportProcessor:
@@ -64,3 +65,11 @@ class OutputProcessor:
router_entry = RouterEntriesHandler.router_entry(entry_name)
if router_entry:
DHCPOutput.clients_summary(router_entry)
@staticmethod
def conn_stats(entry_name):
router_entry = RouterEntriesHandler.router_entry(entry_name)
if router_entry:
ConnectionsStatsOutput.clients_summary(router_entry)

38
mktxp/flow/processor/output.py
View File

@@ -19,6 +19,7 @@ from texttable import Texttable
from humanize import naturaldelta
from mktxp.cli.config.config import config_handler
from mktxp.datasource.wireless_ds import WirelessMetricsDataSource
from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource
from math import floor, log
@@ -35,20 +36,13 @@ class BaseOutputProcessor:
OutputDHCPEntry = namedtuple('OutputDHCPEntry', ['host_name', 'server', 'mac_address', 'address', 'active_address', 'expires_after'])
OutputDHCPEntry.__new__.__defaults__ = ('',) * len(OutputDHCPEntry._fields)
@staticmethod
def augment_record(router_entry, registration_record, dhcp_lease_records):
dhcp_name = registration_record.get('mac_address')
dhcp_address = 'No DHCP Record'
if dhcp_lease_records:
try:
dhcp_lease_record = next((dhcp_lease_record for dhcp_lease_record in dhcp_lease_records if dhcp_lease_record.get('mac_address')==registration_record.get('mac_address')))
dhcp_name = BaseOutputProcessor.dhcp_name(router_entry, dhcp_lease_record)
dhcp_address = dhcp_lease_record.get('address', '')
except StopIteration:
pass
OutputConnStatsEntry = namedtuple('OutputConnStatsEntry', ['dhcp_name', 'src_address', 'connection_count', 'dst_addresses'])
OutputConnStatsEntry.__new__.__defaults__ = ('',) * len(OutputConnStatsEntry._fields)
registration_record['dhcp_name'] = dhcp_name
registration_record['dhcp_address'] = dhcp_address
@staticmethod
def augment_record(router_entry, registration_record, id_key = 'mac_address'):
BaseOutputProcessor.resolve_dhcp(router_entry, registration_record, id_key)
# split out tx/rx bytes
if registration_record.get('bytes'):
@@ -85,9 +79,25 @@ class BaseOutputProcessor:
if drop_comment:
del dhcp_lease_record['comment']
return dhcp_name
@staticmethod
def resolve_dhcp(router_entry, registration_record, id_key = 'mac_address', resolve_address = True):
if not router_entry.dhcp_records:
DHCPMetricsDataSource.metric_records(router_entry)
dhcp_name = registration_record.get(id_key)
dhcp_address = 'No DHCP Record'
dhcp_lease_record = router_entry.dhcp_record(dhcp_name)
if dhcp_lease_record:
dhcp_name = BaseOutputProcessor.dhcp_name(router_entry, dhcp_lease_record)
dhcp_address = dhcp_lease_record.get('address', '')
registration_record['dhcp_name'] = dhcp_name
if resolve_address:
registration_record['dhcp_address'] = dhcp_address
@staticmethod
def parse_rates(rate):
wifi_rates_rgx = config_handler.re_compiled.get('wifi_rates_rgx')

44
mktxp/flow/router_entries_handler.py
View File

@@ -20,24 +20,34 @@ class RouterEntriesHandler:
''' Handles RouterOS entries defined in MKTXP config
'''
def __init__(self):
self.router_entries = []
for router_name in config_handler.registered_entries():
router_entry = RouterEntriesHandler.router_entry(router_name, enabled_only = True)
if router_entry:
self.router_entries.append(router_entry)
self._router_entries = {}
for router_name in config_handler.registered_entries():
router_entry = RouterEntry(router_name)
if router_entry.config_entry.remote_dhcp_entry and config_handler.registered_entry(router_entry.config_entry.remote_dhcp_entry):
router_entry.dhcp_entry = RouterEntry(router_entry.config_entry.remote_dhcp_entry)
self._router_entries[router_name] = router_entry
@property
def router_entries(self):
return (entry for key, entry in self._router_entries.items() if entry.config_entry.enabled) \
if self._router_entries else None
def router_entry(self, entry_name, enabled_only = False):
entry = self._router_entries.get(entry_name)
if entry and (entry.config_entry.enabled or not enabled_only):
return entry
return None
@staticmethod
def router_entry(entry_name, enabled_only = False):
router_entry = None
for router_name in config_handler.registered_entries():
if router_name == entry_name:
config_entry = config_handler.config_entry(router_name)
if enabled_only and not config_entry.enabled:
break
router_entry = RouterEntry(router_name)
router_entry.dhcp_entry = RouterEntriesHandler.router_entry(config_entry.remote_dhcp_entry)
break
''' A static router entry initialiser
'''
config_entry = config_handler.config_entry(entry_name)
if enabled_only and not config_entry.enabled:
return None
router_entry = RouterEntry(entry_name)
if config_entry.remote_dhcp_entry and config_handler.registered_entry(config_entry.remote_dhcp_entry):
router_entry.dhcp_entry = RouterEntry(config_entry.remote_dhcp_entry)
return router_entry

89
mktxp/flow/router_entry.py
View File

@@ -12,7 +12,8 @@
## GNU General Public License for more details.
from mktxp.cli.config.config import config_handler, MKTXPConfigKeys
from collections import namedtuple
from mktxp.cli.config.config import config_handler, MKTXPConfigKeys, CollectorKeys
from mktxp.flow.router_connection import RouterAPIConnection
@@ -29,39 +30,71 @@ class RouterEntry:
}
self.wifi_package = None
self.dhcp_entry = None
self.time_spent = { CollectorKeys.IDENTITY_COLLECTOR: 0,
CollectorKeys.SYSTEM_RESOURCE_COLLECTOR: 0,
CollectorKeys.HEALTH_COLLECTOR: 0,
CollectorKeys.PUBLIC_IP_ADDRESS_COLLECTOR: 0,
CollectorKeys.IPV6_NEIGHBOR_COLLECTOR: 0,
CollectorKeys.PACKAGE_COLLECTOR: 0,
CollectorKeys.DHCP_COLLECTOR: 0,
CollectorKeys.POOL_COLLECTOR: 0,
CollectorKeys.IP_CONNECTION_COLLECTOR: 0,
CollectorKeys.INTERFACE_COLLECTOR: 0,
CollectorKeys.FIREWALL_COLLECTOR: 0,
CollectorKeys.MONITOR_COLLECTOR: 0,
CollectorKeys.POE_COLLECTOR: 0,
CollectorKeys.NETWATCH_COLLECTOR: 0,
CollectorKeys.ROUTE_COLLECTOR: 0,
CollectorKeys.WLAN_COLLECTOR: 0,
CollectorKeys.CAPSMAN_COLLECTOR: 0,
CollectorKeys.QUEUE_TREE_COLLECTOR: 0,
CollectorKeys.QUEUE_SIMPLE_COLLECTOR: 0,
CollectorKeys.USER_COLLECTOR: 0,
CollectorKeys.MKTXP_COLLECTOR: 0
}
self._dhcp_entry = None
self._dhcp_records = {}
@property
def dhcp_entry(self):
if self._dhcp_entry:
return self._dhcp_entry
return self
@dhcp_entry.setter
def dhcp_entry(self, dhcp_entry):
self._dhcp_entry = dhcp_entry
self.time_spent = { 'IdentityCollector': 0,
'SystemResourceCollector': 0,
'HealthCollector': 0,
'PublicIPAddressCollector': 0,
'IPv6NeighborCollector': 0,
'PackageCollector': 0,
'DHCPCollector': 0,
'PoolCollector': 0,
'IPConnectionCollector': 0,
'InterfaceCollector': 0,
'FirewallCollector': 0,
'MonitorCollector': 0,
'POECollector': 0,
'NetwatchCollector': 0,
'RouteCollector': 0,
'WLANCollector': 0,
'CapsmanCollector': 0,
'QueueTreeCollector': 0,
'QueueSimpleCollector': 0,
'UserCollector': 0,
'MKTXPCollector': 0
}
@property
def dhcp_records(self):
return (entry.record for key, entry in self._dhcp_records.items() if entry.type == 'mac_address') \
if self._dhcp_records else None
@dhcp_records.setter
def dhcp_records(self, dhcp_records):
for dhcp_record in dhcp_records:
if dhcp_record.get('mac_address'):
self._dhcp_records[dhcp_record.get('mac_address')] = DHCPCacheEntry('mac_address', dhcp_record)
if dhcp_record.get('address'):
dhcp_record['type'] = 'address'
self._dhcp_records[dhcp_record.get('address')] = DHCPCacheEntry('address', dhcp_record)
def dhcp_record(self, key):
if self._dhcp_records and self._dhcp_records.get(key):
return self._dhcp_records[key].record
return None
def is_ready(self):
self.is_done() #flush caches, just in case
is_ready = True
self.wifi_package = None
if not self.api_connection.is_connected():
is_ready = False
# let's get connected now
self.api_connection.connect()
if self.dhcp_entry:
self.dhcp_entry.api_connection.connect()
if self._dhcp_entry:
self._dhcp_entry.api_connection.connect()
return is_ready
def is_done(self):
self.wifi_package = None
self._dhcp_records = {}
DHCPCacheEntry = namedtuple('DHCPCacheEntry', ['type', 'record'])

2
setup.py
View File

@@ -20,7 +20,7 @@ with open(path.join(pkg_dir, 'README.md'), encoding='utf-8') as f:
setup(
name='mktxp',
version='1.1',
version='1.2',
url='https://github.com/akpw/mktxp',