KevinMidboe/mktxp-no-cli
1
0
Fork 0
mirror of https://github.com/KevinMidboe/mktxp-no-cli.git synced 2025-10-29 17:50:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

connections stats collector / cmd output, remote dhcp resolver, fixes / optimizations

Browse Source
This commit is contained in:
Arseniy Kuznetsov
2023-02-04 20:48:51 +01:00
parent 18ddbe3311
commit 9a381d028c
21 changed files with 351 additions and 161 deletions
Download Patch File Download Diff File Expand all files Collapse all files

73
mktxp/cli/config/config.py
View File

@@ -24,6 +24,29 @@ from mktxp.utils.utils import FSHelper
''' MKTXP conf file handling
'''
class CollectorKeys:
IDENTITY_COLLECTOR = 'IdentityCollector'
SYSTEM_RESOURCE_COLLECTOR = 'SystemResourceCollector'
HEALTH_COLLECTOR = 'HealthCollector'
PUBLIC_IP_ADDRESS_COLLECTOR = 'PublicIPAddressCollector'
IPV6_NEIGHBOR_COLLECTOR = 'IPv6NeighborCollector'
PACKAGE_COLLECTOR = 'PackageCollector'
DHCP_COLLECTOR = 'DHCPCollector'
POOL_COLLECTOR = 'PoolCollector'
IP_CONNECTION_COLLECTOR = 'IPConnectionCollector'
INTERFACE_COLLECTOR = 'InterfaceCollector'
FIREWALL_COLLECTOR = 'FirewallCollector'
MONITOR_COLLECTOR = 'MonitorCollector'
POE_COLLECTOR = 'POECollector'
NETWATCH_COLLECTOR = 'NetwatchCollector'
ROUTE_COLLECTOR = 'RouteCollector'
WLAN_COLLECTOR = 'WLANCollector'
CAPSMAN_COLLECTOR = 'CapsmanCollector'
QUEUE_TREE_COLLECTOR = 'QueueTreeCollector'
QUEUE_SIMPLE_COLLECTOR = 'QueueSimpleCollector'
USER_COLLECTOR = 'UserCollector'
MKTXP_COLLECTOR = 'MKTXPCollector'
class MKTXPConfigKeys:
''' MKTXP config file keys
@@ -44,6 +67,7 @@ class MKTXPConfigKeys:
FE_DHCP_LEASE_KEY = 'dhcp_lease'
FE_DHCP_POOL_KEY = 'pool'
FE_IP_CONNECTIONS_KEY = 'connections'
FE_CONNECTION_STATS_KEY = 'connection_stats'
FE_INTERFACE_KEY = 'interface'
FE_FIREWALL_KEY = 'firewall'
@@ -104,7 +128,7 @@ class MKTXPConfigKeys:
BOOLEAN_KEYS_NO = {ENABLED_KEY, SSL_KEY, NO_SSL_CERTIFICATE,
SSL_CERTIFICATE_VERIFY, FE_IPV6_FIREWALL_KEY, FE_IPV6_NEIGHBOR_KEY}
SSL_CERTIFICATE_VERIFY, FE_IPV6_FIREWALL_KEY, FE_IPV6_NEIGHBOR_KEY, FE_CONNECTION_STATS_KEY}
# Feature keys enabled by default
BOOLEAN_KEYS_YES = {FE_DHCP_KEY, FE_PACKAGE_KEY, FE_DHCP_LEASE_KEY, FE_DHCP_POOL_KEY, FE_IP_CONNECTIONS_KEY, FE_INTERFACE_KEY, FE_FIREWALL_KEY,
@@ -116,6 +140,7 @@ class MKTXPConfigKeys:
SYSTEM_BOOLEAN_KEYS_NO = {MKTXP_VERBOSE_MODE, MKTXP_FETCH_IN_PARALLEL}
STR_KEYS = (HOST_KEY, USER_KEY, PASSWD_KEY, FE_REMOTE_DHCP_ENTRY)
INT_KEYS = ()
MKTXP_INT_KEYS = (PORT_KEY, MKTXP_SOCKET_TIMEOUT, MKTXP_INITIAL_DELAY, MKTXP_MAX_DELAY,
MKTXP_INC_DIV, MKTXP_BANDWIDTH_TEST_INTERVAL, MKTXP_MIN_COLLECT_INTERVAL,
MKTXP_MAX_WORKER_THREADS, MKTXP_MAX_SCRAPE_DURATION, MKTXP_TOTAL_MAX_SCRAPE_DURATION)
@@ -130,7 +155,7 @@ class ConfigEntry:
MKTXPConfigKeys.SSL_KEY, MKTXPConfigKeys.NO_SSL_CERTIFICATE, MKTXPConfigKeys.SSL_CERTIFICATE_VERIFY,
MKTXPConfigKeys.FE_DHCP_KEY, MKTXPConfigKeys.FE_PACKAGE_KEY, MKTXPConfigKeys.FE_DHCP_LEASE_KEY, MKTXPConfigKeys.FE_DHCP_POOL_KEY, MKTXPConfigKeys.FE_INTERFACE_KEY,
MKTXPConfigKeys.FE_FIREWALL_KEY, MKTXPConfigKeys.FE_MONITOR_KEY, MKTXPConfigKeys.FE_ROUTE_KEY, MKTXPConfigKeys.FE_WIRELESS_KEY, MKTXPConfigKeys.FE_WIRELESS_CLIENTS_KEY,
MKTXPConfigKeys.FE_IP_CONNECTIONS_KEY, MKTXPConfigKeys.FE_CAPSMAN_KEY, MKTXPConfigKeys.FE_CAPSMAN_CLIENTS_KEY, MKTXPConfigKeys.FE_POE_KEY, MKTXPConfigKeys.FE_NETWATCH_KEY,
MKTXPConfigKeys.FE_IP_CONNECTIONS_KEY, MKTXPConfigKeys.FE_CONNECTION_STATS_KEY, MKTXPConfigKeys.FE_CAPSMAN_KEY, MKTXPConfigKeys.FE_CAPSMAN_CLIENTS_KEY, MKTXPConfigKeys.FE_POE_KEY, MKTXPConfigKeys.FE_NETWATCH_KEY,
MKTXPConfigKeys.MKTXP_USE_COMMENTS_OVER_NAMES, MKTXPConfigKeys.FE_PUBLIC_IP_KEY, MKTXPConfigKeys.FE_IPV6_FIREWALL_KEY, MKTXPConfigKeys.FE_IPV6_NEIGHBOR_KEY,
MKTXPConfigKeys.FE_USER_KEY, MKTXPConfigKeys.FE_QUEUE_KEY, MKTXPConfigKeys.FE_REMOTE_DHCP_ENTRY
])
@@ -235,17 +260,18 @@ class MKTXPConfigHandler:
def registered_entries(self):
''' All MKTXP registered entries
'''
registered_entries = [entry_name for entry_name in self.config.keys()]
if not registered_entries:
registered_entries = [MKTXPConfigKeys.NO_ENTRIES_REGISTERED]
return (entry_name for entry_name in self.config.keys())
return registered_entries
def registered_entry(self, entry_name):
''' A specific MKTXP registered entry by name
'''
return self.config.get(entry_name)
def config_entry(self, entry_name):
''' Given an entry name, reads and returns the entry info
'''
entry_reader = self._config_entry_reader(entry_name)
return ConfigEntry.MKTXPConfigEntry(**entry_reader)
return ConfigEntry.MKTXPConfigEntry(**entry_reader) if entry_reader else None
def system_entry(self):
''' MKTXP internal config entry
@@ -258,7 +284,10 @@ class MKTXPConfigHandler:
''' (Force-)Read conf data from disk
'''
self.config = ConfigObj(self.usr_conf_data_path)
self.config.preserve_comments = True
self._config = ConfigObj(self.mktxp_conf_path)
self._config.preserve_comments = True
def _create_os_path(self, os_path, resource_path):
if not os.path.exists(os_path):
@@ -272,7 +301,7 @@ class MKTXPConfigHandler:
new_keys = []
for key in MKTXPConfigKeys.BOOLEAN_KEYS_NO.union(MKTXPConfigKeys.BOOLEAN_KEYS_YES):
if self.config[entry_name].get(key):
if self.config[entry_name].get(key) is not None:
config_entry_reader[key] = self.config[entry_name].as_bool(key)
else:
config_entry_reader[key] = True if key in MKTXPConfigKeys.BOOLEAN_KEYS_YES else False
@@ -288,6 +317,13 @@ class MKTXPConfigHandler:
if key is MKTXPConfigKeys.PASSWD_KEY and type(config_entry_reader[key]) is list:
config_entry_reader[key] = ','.join(config_entry_reader[key])
for key in MKTXPConfigKeys.INT_KEYS:
if self.config[entry_name].get(key):
config_entry_reader[key] = self.config[entry_name].as_int(key)
else:
config_entry_reader[key] = self._default_value_for_key(key)
new_keys.append(key) # read from disk next time
# port
if self.config[entry_name].get(MKTXPConfigKeys.PORT_KEY):
config_entry_reader[MKTXPConfigKeys.PORT_KEY] = self.config[entry_name].as_int(
@@ -322,9 +358,8 @@ class MKTXPConfigHandler:
new_keys.append(key) # read from disk next time
for key in MKTXPConfigKeys.SYSTEM_BOOLEAN_KEYS_NO.union(MKTXPConfigKeys.SYSTEM_BOOLEAN_KEYS_YES):
if self._config[entry_name].get(key):
system_entry_reader[key] = self._config[entry_name].as_bool(
key)
if self._config[entry_name].get(key) is not None:
system_entry_reader[key] = self._config[entry_name].as_bool(key)
else:
system_entry_reader[key] = True if key in MKTXPConfigKeys.SYSTEM_BOOLEAN_KEYS_YES else False
new_keys.append(key) # read from disk next time
@@ -344,14 +379,14 @@ class MKTXPConfigHandler:
def _default_value_for_key(self, key, value=None):
return {
MKTXPConfigKeys.SSL_KEY: lambda value: MKTXPConfigKeys.DEFAULT_API_SSL_PORT if value else MKTXPConfigKeys.DEFAULT_API_PORT,
MKTXPConfigKeys.PORT_KEY: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_PORT,
MKTXPConfigKeys.FE_REMOTE_DHCP_ENTRY: lambda value: MKTXPConfigKeys.DEFAULT_FE_REMOTE_DHCP_ENTRY,
MKTXPConfigKeys.MKTXP_SOCKET_TIMEOUT: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_SOCKET_TIMEOUT,
MKTXPConfigKeys.MKTXP_INITIAL_DELAY: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_INITIAL_DELAY,
MKTXPConfigKeys.MKTXP_MAX_DELAY: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_MAX_DELAY,
MKTXPConfigKeys.MKTXP_INC_DIV: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_INC_DIV,
MKTXPConfigKeys.MKTXP_BANDWIDTH_TEST_INTERVAL: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_BANDWIDTH_TEST_INTERVAL,
MKTXPConfigKeys.MKTXP_MIN_COLLECT_INTERVAL: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_MIN_COLLECT_INTERVAL,
MKTXPConfigKeys.PORT_KEY: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_PORT,
MKTXPConfigKeys.FE_REMOTE_DHCP_ENTRY: lambda _: MKTXPConfigKeys.DEFAULT_FE_REMOTE_DHCP_ENTRY,
MKTXPConfigKeys.MKTXP_SOCKET_TIMEOUT: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_SOCKET_TIMEOUT,
MKTXPConfigKeys.MKTXP_INITIAL_DELAY: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_INITIAL_DELAY,
MKTXPConfigKeys.MKTXP_MAX_DELAY: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_MAX_DELAY,
MKTXPConfigKeys.MKTXP_INC_DIV: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_INC_DIV,
MKTXPConfigKeys.MKTXP_BANDWIDTH_TEST_INTERVAL: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_BANDWIDTH_TEST_INTERVAL,
MKTXPConfigKeys.MKTXP_MIN_COLLECT_INTERVAL: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_MIN_COLLECT_INTERVAL,
MKTXPConfigKeys.MKTXP_FETCH_IN_PARALLEL: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_FETCH_IN_PARALLEL,
MKTXPConfigKeys.MKTXP_MAX_WORKER_THREADS: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_MAX_WORKER_THREADS,
MKTXPConfigKeys.MKTXP_MAX_SCRAPE_DURATION: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_MAX_SCRAPE_DURATION,

3
mktxp/cli/config/mktxp.conf
View File

@@ -27,7 +27,10 @@
installed_packages = True # Installed packages
dhcp = True # DHCP general metrics
dhcp_lease = True # DHCP lease metrics
connections = True # IP connections metrics
connection_stats = False # Open IP connections metrics
pool = True # Pool metrics
interface = True # Interfaces traffic metrics

3
mktxp/cli/dispatch.py
View File

@@ -98,6 +98,9 @@ class MKTXPDispatcher:
elif args['dhcp_clients']:
OutputProcessor.dhcp_clients(args['entry_name'])
elif args['conn_stats']:
OutputProcessor.conn_stats(args['entry_name'])
else:
print("Select metric option(s) to print out, or run 'mktxp print -h' to find out more")

5
mktxp/cli/options.py
View File

@@ -146,6 +146,11 @@ Selected metrics info can be printed on the command line. For more information,
help = "DHCP clients metrics",
action = 'store_true')
optional_args_group.add_argument('-cn', '--conn_stats', dest='conn_stats',
help = "IP connections stats",
action = 'store_true')
# Options checking
def _check_args(self, args, parser):
''' Validation of supplied CLI arguments

8
mktxp/cli/output/capsman_out.py
View File

@@ -13,8 +13,6 @@
from mktxp.flow.processor.output import BaseOutputProcessor
from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource
from mktxp.datasource.wireless_ds import WirelessMetricsDataSource
from mktxp.datasource.capsman_ds import CapsmanRegistrationsMetricsDataSource
class CapsmanOutput:
@@ -29,13 +27,9 @@ class CapsmanOutput:
return
# translate / trim / augment registration records
dhcp_lease_labels = ['host_name', 'comment', 'address', 'mac_address']
dhcp_entry = WirelessMetricsDataSource.dhcp_entry(router_entry)
dhcp_lease_records = DHCPMetricsDataSource.metric_records(dhcp_entry, metric_labels = dhcp_lease_labels, add_router_id = False)
dhcp_rt_by_interface = {}
for registration_record in sorted(registration_records, key = lambda rt_record: rt_record['rx_signal'], reverse=True):
BaseOutputProcessor.augment_record(router_entry, registration_record, dhcp_lease_records)
BaseOutputProcessor.augment_record(router_entry, registration_record)
interface = registration_record['interface']
if interface in dhcp_rt_by_interface.keys():

49
mktxp/cli/output/conn_stats_out.py Normal file
View File

@@ -0,0 +1,49 @@
# coding=utf8
## Copyright (c) 2020 Arseniy Kuznetsov
##
## This program is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License
## as published by the Free Software Foundation; either version 2
## of the License, or (at your option) any later version.
##
## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
## GNU General Public License for more details.
from mktxp.flow.processor.output import BaseOutputProcessor
from mktxp.datasource.connection_ds import IPConnectionStatsDatasource
class ConnectionsStatsOutput:
''' Connections Stats Output
'''
@staticmethod
def clients_summary(router_entry):
connection_records = IPConnectionStatsDatasource.metric_records(router_entry, add_router_id = False)
if not connection_records:
print('No connection stats records')
return
conn_cnt = 0
output_records = []
for registration_record in sorted(connection_records, key = lambda rt_record: rt_record['connection_count'], reverse=True):
BaseOutputProcessor.resolve_dhcp(router_entry, registration_record, id_key = 'src_address', resolve_address = False)
output_records.append(registration_record)
conn_cnt += registration_record['connection_count']
output_records_cnt = 0
output_entry = BaseOutputProcessor.OutputConnStatsEntry
output_table = BaseOutputProcessor.output_table(output_entry)
for record in output_records:
output_table.add_row(output_entry(**record))
output_table.add_row(output_entry())
output_records_cnt += 1
print (output_table.draw())
print(f'Distinct source addresses: {output_records_cnt}')
print(f'Total open connections: {conn_cnt}', '\n')

4
mktxp/cli/output/dhcp_out.py
View File

@@ -22,13 +22,13 @@ class DHCPOutput:
@staticmethod
def clients_summary(router_entry):
dhcp_lease_labels = ['host_name', 'comment', 'active_address', 'address', 'mac_address', 'server', 'expires_after']
dhcp_lease_records = DHCPMetricsDataSource.metric_records(router_entry, metric_labels = dhcp_lease_labels, add_router_id = False)
dhcp_lease_records = DHCPMetricsDataSource.metric_records(router_entry, metric_labels = dhcp_lease_labels, add_router_id = False, translate = False, dhcp_cache = False)
if not dhcp_lease_records:
print('No DHCP registration records')
return
dhcp_by_server = {}
for dhcp_lease_record in sorted(dhcp_lease_records, key = lambda dhcp_record: dhcp_record['active_address'], reverse=True):
for dhcp_lease_record in sorted(dhcp_lease_records, key = lambda dhcp_record: dhcp_record['address'], reverse=True):
server = dhcp_lease_record.get('server', 'all')
if server == 'all':
dhcp_lease_record['server'] = server

8
mktxp/cli/output/wifi_out.py
View File

@@ -13,9 +13,7 @@
from mktxp.flow.processor.output import BaseOutputProcessor
from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource
from mktxp.datasource.wireless_ds import WirelessMetricsDataSource
from mktxp.flow.router_entries_handler import RouterEntriesHandler
class WirelessOutput:
@@ -30,15 +28,11 @@ class WirelessOutput:
return
# translate / trim / augment registration records
dhcp_lease_labels = ['host_name', 'comment', 'address', 'mac_address']
dhcp_entry = WirelessMetricsDataSource.dhcp_entry(router_entry)
dhcp_lease_records = DHCPMetricsDataSource.metric_records(dhcp_entry, metric_labels = dhcp_lease_labels, add_router_id = False)
dhcp_rt_by_interface = {}
key = lambda rt_record: rt_record['signal_strength'] if rt_record.get('signal_strength') else rt_record['interface']
for registration_record in sorted(registration_records, key = key, reverse=True):
BaseOutputProcessor.augment_record(router_entry, registration_record, dhcp_lease_records)
BaseOutputProcessor.augment_record(router_entry, registration_record)
interface = registration_record['interface']
if interface in dhcp_rt_by_interface.keys():