KevinMidboe/mktxp-no-cli
1
0
Fork 0
mirror of https://github.com/KevinMidboe/mktxp-no-cli.git synced 2025-10-29 17:50:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

remote dhcp info resolution (wireless/capsman), more wifiwave2 support

Browse Source
This commit is contained in:
Arseniy Kuznetsov
2023-01-14 17:33:04 +01:00
parent cbdaf51a95
commit e27305e736
14 changed files with 105 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@@ -331,7 +331,7 @@ mktxp is running as pid 36704
````
❯ mktxp -h
usage: MKTXP [-h] [--dir DIR] {info, edit, export, print, show, } ...
usage: MKTXP [-h] [--cfg-dir CFG_DIR] {info, edit, export, print, show, } ...
````
To learn more about individual commands, just run it with ```-h```:
For example, to learn everything about ````mktxp show````:

14
mktxp/cli/config/config.py
View File

@@ -62,6 +62,7 @@ class MKTXPConfigKeys:
FE_USER_KEY = 'user'
FE_QUEUE_KEY = 'queue'
FE_REMOTE_DHCP_ENTRY = 'remote_dhcp_entry'
MKTXP_SOCKET_TIMEOUT = 'socket_timeout'
MKTXP_INITIAL_DELAY = 'initial_delay_on_failure'
@@ -88,6 +89,7 @@ class MKTXPConfigKeys:
# Default values
DEFAULT_API_PORT = 8728
DEFAULT_API_SSL_PORT = 8729
DEFAULT_FE_REMOTE_DHCP_ENTRY = 'None'
DEFAULT_MKTXP_PORT = 49090
DEFAULT_MKTXP_SOCKET_TIMEOUT = 2
DEFAULT_MKTXP_INITIAL_DELAY = 120
@@ -113,7 +115,7 @@ class MKTXPConfigKeys:
SYSTEM_BOOLEAN_KEYS_YES = {MKTXP_BANDWIDTH_KEY}
SYSTEM_BOOLEAN_KEYS_NO = {MKTXP_VERBOSE_MODE, MKTXP_FETCH_IN_PARALLEL}
STR_KEYS = (HOST_KEY, USER_KEY, PASSWD_KEY)
STR_KEYS = (HOST_KEY, USER_KEY, PASSWD_KEY, FE_REMOTE_DHCP_ENTRY)
MKTXP_INT_KEYS = (PORT_KEY, MKTXP_SOCKET_TIMEOUT, MKTXP_INITIAL_DELAY, MKTXP_MAX_DELAY,
MKTXP_INC_DIV, MKTXP_BANDWIDTH_TEST_INTERVAL, MKTXP_MIN_COLLECT_INTERVAL,
MKTXP_MAX_WORKER_THREADS, MKTXP_MAX_SCRAPE_DURATION, MKTXP_TOTAL_MAX_SCRAPE_DURATION)
@@ -130,7 +132,7 @@ class ConfigEntry:
MKTXPConfigKeys.FE_FIREWALL_KEY, MKTXPConfigKeys.FE_MONITOR_KEY, MKTXPConfigKeys.FE_ROUTE_KEY, MKTXPConfigKeys.FE_WIRELESS_KEY, MKTXPConfigKeys.FE_WIRELESS_CLIENTS_KEY,
MKTXPConfigKeys.FE_IP_CONNECTIONS_KEY, MKTXPConfigKeys.FE_CAPSMAN_KEY, MKTXPConfigKeys.FE_CAPSMAN_CLIENTS_KEY, MKTXPConfigKeys.FE_POE_KEY, MKTXPConfigKeys.FE_NETWATCH_KEY,
MKTXPConfigKeys.MKTXP_USE_COMMENTS_OVER_NAMES, MKTXPConfigKeys.FE_PUBLIC_IP_KEY, MKTXPConfigKeys.FE_IPV6_FIREWALL_KEY, MKTXPConfigKeys.FE_IPV6_NEIGHBOR_KEY,
MKTXPConfigKeys.FE_USER_KEY, MKTXPConfigKeys.FE_QUEUE_KEY
MKTXPConfigKeys.FE_USER_KEY, MKTXPConfigKeys.FE_QUEUE_KEY, MKTXPConfigKeys.FE_REMOTE_DHCP_ENTRY
])
MKTXPSystemEntry = namedtuple('MKTXPSystemEntry', [MKTXPConfigKeys.PORT_KEY, MKTXPConfigKeys.MKTXP_SOCKET_TIMEOUT,
MKTXPConfigKeys.MKTXP_INITIAL_DELAY, MKTXPConfigKeys.MKTXP_MAX_DELAY,
@@ -277,7 +279,12 @@ class MKTXPConfigHandler:
new_keys.append(key) # read from disk next time
for key in MKTXPConfigKeys.STR_KEYS:
config_entry_reader[key] = self.config[entry_name][key]
if self.config[entry_name].get(key):
config_entry_reader[key] = self.config[entry_name].get(key)
else:
config_entry_reader[key] = self._default_value_for_key(key)
new_keys.append(key) # read from disk next time
if key is MKTXPConfigKeys.PASSWD_KEY and type(config_entry_reader[key]) is list:
config_entry_reader[key] = ','.join(config_entry_reader[key])
@@ -338,6 +345,7 @@ class MKTXPConfigHandler:
return {
MKTXPConfigKeys.SSL_KEY: lambda value: MKTXPConfigKeys.DEFAULT_API_SSL_PORT if value else MKTXPConfigKeys.DEFAULT_API_PORT,
MKTXPConfigKeys.PORT_KEY: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_PORT,
MKTXPConfigKeys.FE_REMOTE_DHCP_ENTRY: lambda value: MKTXPConfigKeys.DEFAULT_FE_REMOTE_DHCP_ENTRY,
MKTXPConfigKeys.MKTXP_SOCKET_TIMEOUT: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_SOCKET_TIMEOUT,
MKTXPConfigKeys.MKTXP_INITIAL_DELAY: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_INITIAL_DELAY,
MKTXPConfigKeys.MKTXP_MAX_DELAY: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_MAX_DELAY,

1
mktxp/cli/config/mktxp.conf
View File

@@ -47,5 +47,6 @@
user = True # Active Users metrics
queue = True # Queues metrics
remote_dhcp_entry = None # Alternative mktxp entry for DHCP info resolution (capsman/wireless)
use_comments_over_names = True # when available, forces using comments over the interfaces names

6
mktxp/cli/options.py
View File

@@ -66,8 +66,8 @@ Selected metrics info can be printed on the command line. For more information,
global_options_parser = ArgumentParser(add_help=False)
self.parse_global_options(global_options_parser)
namespace, _ = global_options_parser.parse_known_args()
if namespace.dir:
config_handler(CustomConfig(namespace.dir))
if namespace.cfg_dir:
config_handler(CustomConfig(namespace.cfg_dir))
else:
config_handler()
@@ -84,7 +84,7 @@ Selected metrics info can be printed on the command line. For more information,
def parse_global_options(self, parser):
''' Parses global options
'''
parser.add_argument('--dir', dest = 'dir',
parser.add_argument('--cfg-dir', dest = 'cfg_dir',
type = lambda d: self._is_valid_dir_path(parser, d),
help = 'MKTXP config files directory (optional)')

6
mktxp/cli/output/capsman_out.py
View File

@@ -14,9 +14,9 @@
from mktxp.flow.processor.output import BaseOutputProcessor
from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource
from mktxp.datasource.wireless_ds import WirelessMetricsDataSource
from mktxp.datasource.capsman_ds import CapsmanRegistrationsMetricsDataSource
class CapsmanOutput:
''' CAPsMAN CLI Output
'''
@@ -30,7 +30,8 @@ class CapsmanOutput:
# translate / trim / augment registration records
dhcp_lease_labels = ['host_name', 'comment', 'address', 'mac_address']
dhcp_lease_records = DHCPMetricsDataSource.metric_records(router_entry, metric_labels = dhcp_lease_labels, add_router_id = False)
dhcp_entry = WirelessMetricsDataSource.dhcp_entry(router_entry)
dhcp_lease_records = DHCPMetricsDataSource.metric_records(dhcp_entry, metric_labels = dhcp_lease_labels, add_router_id = False)
dhcp_rt_by_interface = {}
for registration_record in sorted(registration_records, key = lambda rt_record: rt_record['rx_signal'], reverse=True):
@@ -60,4 +61,3 @@ class CapsmanOutput:
print(f'{server} clients: {len(dhcp_rt_by_interface[server])}')
print(f'Total connected CAPsMAN clients: {output_records}', '\n')

7
mktxp/cli/output/wifi_out.py
View File

@@ -15,6 +15,7 @@
from mktxp.flow.processor.output import BaseOutputProcessor
from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource
from mktxp.datasource.wireless_ds import WirelessMetricsDataSource
from mktxp.flow.router_entries_handler import RouterEntriesHandler
class WirelessOutput:
@@ -30,7 +31,8 @@ class WirelessOutput:
# translate / trim / augment registration records
dhcp_lease_labels = ['host_name', 'comment', 'address', 'mac_address']
dhcp_lease_records = DHCPMetricsDataSource.metric_records(router_entry, metric_labels = dhcp_lease_labels, add_router_id = False)
dhcp_entry = WirelessMetricsDataSource.dhcp_entry(router_entry)
dhcp_lease_records = DHCPMetricsDataSource.metric_records(dhcp_entry, metric_labels = dhcp_lease_labels, add_router_id = False)
dhcp_rt_by_interface = {}
@@ -46,7 +48,8 @@ class WirelessOutput:
output_records = 0
registration_records = len(registration_records)
output_entry = BaseOutputProcessor.OutputWiFiEntry
output_entry = BaseOutputProcessor.OutputWiFiWave2Entry \
if WirelessMetricsDataSource.wifiwave2_installed(router_entry) else BaseOutputProcessor.OutputWiFiEntry
output_table = BaseOutputProcessor.output_table(output_entry)
for key in dhcp_rt_by_interface.keys():

5
mktxp/collector/capsman_collector.py
View File

@@ -17,6 +17,7 @@ from mktxp.flow.processor.output import BaseOutputProcessor
from mktxp.collector.base_collector import BaseCollector
from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource
from mktxp.datasource.capsman_ds import CapsmanCapsMetricsDataSource, CapsmanRegistrationsMetricsDataSource, CapsmanInterfacesDatasource
from mktxp.datasource.wireless_ds import WirelessMetricsDataSource
class CapsmanCollector(BaseCollector):
@@ -51,9 +52,11 @@ class CapsmanCollector(BaseCollector):
# the client info metrics
if router_entry.config_entry.capsman_clients:
# translate / trim / augment registration records
dhcp_lease_labels = ['mac_address', 'address', 'host_name', 'comment']
dhcp_lease_records = DHCPMetricsDataSource.metric_records(router_entry, metric_labels = dhcp_lease_labels)
dhcp_entry = WirelessMetricsDataSource.dhcp_entry(router_entry)
dhcp_lease_records = DHCPMetricsDataSource.metric_records(dhcp_entry, metric_labels = dhcp_lease_labels)
for registration_record in registration_records:
BaseOutputProcessor.augment_record(router_entry, registration_record, dhcp_lease_records)

3
mktxp/collector/wlan_collector.py
View File

@@ -54,7 +54,8 @@ class WLANCollector(BaseCollector):
registration_records = WirelessMetricsDataSource.metric_records(router_entry, metric_labels = registration_labels)
if registration_records:
dhcp_lease_labels = ['mac_address', 'address', 'host_name', 'comment']
dhcp_lease_records = DHCPMetricsDataSource.metric_records(router_entry, metric_labels = dhcp_lease_labels)
dhcp_entry = WirelessMetricsDataSource.dhcp_entry(router_entry)
dhcp_lease_records = DHCPMetricsDataSource.metric_records(dhcp_entry, metric_labels = dhcp_lease_labels)
for registration_record in registration_records:
BaseOutputProcessor.augment_record(router_entry, registration_record, dhcp_lease_records)

10
mktxp/datasource/capsman_ds.py
View File

@@ -19,14 +19,14 @@ from mktxp.datasource.wireless_ds import WirelessMetricsDataSource
class CapsmanInfo:
@staticmethod
def capsman_path(router_entry):
if WirelessMetricsDataSource.wireless_package(router_entry) == WirelessMetricsDataSource.WIFIWAVE2:
if WirelessMetricsDataSource.wifiwave2_installed(router_entry):
return '/interface/wifiwave2/capsman'
else:
return '/caps-man'
@staticmethod
def registration_table_path(router_entry):
if WirelessMetricsDataSource.wireless_package(router_entry) == WirelessMetricsDataSource.WIFIWAVE2:
if WirelessMetricsDataSource.wifiwave2_installed(router_entry):
return '/interface/wifiwave2/registration-table'
else:
return '/caps-man/registration-table'
@@ -57,6 +57,12 @@ class CapsmanRegistrationsMetricsDataSource:
try:
registration_table_path = CapsmanInfo.registration_table_path(router_entry)
registration_table_records = router_entry.api_connection.router_api().get_resource(f'{registration_table_path}').get()
# With wifiwave2, Mikrotik renamed the field 'rx_signal' to 'signal'
for record in registration_table_records:
if 'signal' in record:
record['rx_signal'] = record['signal']
return BaseDSProcessor.trimmed_records(router_entry, router_records = registration_table_records, metric_labels = metric_labels, add_router_id = add_router_id)
except Exception as exc:
print(f'Error getting CAPsMAN registration table info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}')

16
mktxp/datasource/wireless_ds.py
View File

@@ -22,7 +22,6 @@ class WirelessMetricsDataSource:
WIFIWAVE2 = 'wifiwave2'
WIRELESS = 'wireless'
@staticmethod
def metric_records(router_entry, *, metric_labels = None, add_router_id = True):
if metric_labels is None:
@@ -31,8 +30,7 @@ class WirelessMetricsDataSource:
wireless_package = WirelessMetricsDataSource.wireless_package(router_entry)
registration_table_records = router_entry.api_connection.router_api().get_resource(f'/interface/{wireless_package}/registration-table').get()
# Mikrotik renamed the field 'signal_strength' to 'signal' when using wifiwave2.
# Rename this field back to 'signal_strength' to preserve backwards compatibility
# With wifiwave2, Mikrotik renamed the field 'signal_strength' to 'signal'
for record in registration_table_records:
if 'signal' in record:
record['signal_strength'] = record['signal']
@@ -49,3 +47,15 @@ class WirelessMetricsDataSource:
ww2_installed = PackageMetricsDataSource.is_package_installed(router_entry, package_name = WirelessMetricsDataSource.WIFIWAVE2)
router_entry.wifi_package = WirelessMetricsDataSource.WIFIWAVE2 if ww2_installed else WirelessMetricsDataSource.WIRELESS
return router_entry.wifi_package
@staticmethod
def wifiwave2_installed(router_entry):
return WirelessMetricsDataSource.wireless_package(router_entry) == WirelessMetricsDataSource.WIFIWAVE2
@staticmethod
def dhcp_entry(router_entry):
if router_entry.dhcp_entry:
return router_entry.dhcp_entry
return router_entry

6
mktxp/flow/collector_handler.py
View File

@@ -35,9 +35,8 @@ class CollectorHandler:
Thus, the total runtime of this function scales linearly with the number of registered routers.
"""
for router_entry in self.entries_handler.router_entries:
if not router_entry.api_connection.is_connected():
if not router_entry.is_connected():
# let's pick up on things in the next run
router_entry.api_connection.connect()
continue
for collector_ID, collect_func in self.collector_registry.registered_collectors.items():
@@ -88,9 +87,8 @@ class CollectorHandler:
print(f'Hit overall timeout while scraping router entry: {router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME]}')
break
if not router_entry.api_connection.is_connected():
if not router_entry.is_connected():
# let's pick up on things in the next run
router_entry.api_connection.connect()
continue
# Duration of individual scrapes

37
mktxp/flow/processor/output.py
View File

@@ -18,6 +18,8 @@ from collections import namedtuple
from texttable import Texttable
from humanize import naturaldelta
from mktxp.cli.config.config import config_handler
from mktxp.datasource.wireless_ds import WirelessMetricsDataSource
from math import floor, log
class BaseOutputProcessor:
@@ -27,18 +29,23 @@ class BaseOutputProcessor:
OutputWiFiEntry = namedtuple('OutputWiFiEntry', ['dhcp_name', 'dhcp_address', 'mac_address', 'signal_strength', 'signal_to_noise', 'interface', 'tx_rate', 'rx_rate', 'uptime'])
OutputWiFiEntry.__new__.__defaults__ = ('',) * len(OutputWiFiEntry._fields)
OutputWiFiWave2Entry = namedtuple('OutputWiFiWave2Entry', ['dhcp_name', 'dhcp_address', 'mac_address', 'signal_strength', 'interface', 'tx_rate', 'rx_rate', 'uptime'])
OutputWiFiWave2Entry.__new__.__defaults__ = ('',) * len(OutputWiFiWave2Entry._fields)
OutputDHCPEntry = namedtuple('OutputDHCPEntry', ['host_name', 'server', 'mac_address', 'address', 'active_address', 'expires_after'])
OutputDHCPEntry.__new__.__defaults__ = ('',) * len(OutputDHCPEntry._fields)
@staticmethod
def augment_record(router_entry, registration_record, dhcp_lease_records):
try:
dhcp_lease_record = next((dhcp_lease_record for dhcp_lease_record in dhcp_lease_records if dhcp_lease_record.get('mac_address')==registration_record.get('mac_address')))
dhcp_name = BaseOutputProcessor.dhcp_name(router_entry, dhcp_lease_record)
dhcp_address = dhcp_lease_record.get('address', '')
except StopIteration:
dhcp_name = registration_record.get('mac_address')
dhcp_address = 'No DHCP Record'
dhcp_name = registration_record.get('mac_address')
dhcp_address = 'No DHCP Record'
if dhcp_lease_records:
try:
dhcp_lease_record = next((dhcp_lease_record for dhcp_lease_record in dhcp_lease_records if dhcp_lease_record.get('mac_address')==registration_record.get('mac_address')))
dhcp_name = BaseOutputProcessor.dhcp_name(router_entry, dhcp_lease_record)
dhcp_address = dhcp_lease_record.get('address', '')
except StopIteration:
pass
registration_record['dhcp_name'] = dhcp_name
registration_record['dhcp_address'] = dhcp_address
@@ -49,10 +56,13 @@ class BaseOutputProcessor:
registration_record['rx_bytes'] = registration_record['bytes'].split(',')[1]
del registration_record['bytes']
ww2_installed = WirelessMetricsDataSource.wifiwave2_installed(router_entry)
if registration_record.get('tx_rate'):
registration_record['tx_rate'] = BaseOutputProcessor.parse_rates(registration_record['tx_rate'])
registration_record['tx_rate'] = BaseOutputProcessor.parse_bitrates(registration_record['tx_rate']) \
if ww2_installed else BaseOutputProcessor.parse_rates(registration_record['tx_rate'])
if registration_record.get('rx_rate'):
registration_record['rx_rate'] = BaseOutputProcessor.parse_rates(registration_record['rx_rate'])
registration_record['rx_rate'] = BaseOutputProcessor.parse_bitrates(registration_record['rx_rate']) \
if ww2_installed else BaseOutputProcessor.parse_rates(registration_record['rx_rate'])
if registration_record.get('uptime'):
registration_record['uptime'] = naturaldelta(BaseOutputProcessor.parse_timedelta_seconds(registration_record['uptime']), months=True, minimum_unit='seconds')
@@ -87,6 +97,12 @@ class BaseOutputProcessor:
rc = wifi_rates_rgx.search(rate)
return f'{int(float(rc[1]))} {rc[2]}' if rc and len(rc.groups()) == 2 else rate
@staticmethod
def parse_bitrates(rate):
rate = int(rate)
power = floor(log(rate, 1000))
return f"{int(rate / 1000 ** power)} {['bps', 'Kbps', 'Mbps', 'Gbps'][int(power)]}"
@staticmethod
def parse_timedelta(time):
duration_interval_rgx = config_handler.re_compiled.get('duration_interval_rgx')
@@ -126,6 +142,3 @@ class BaseOutputProcessor:
table.header(outputEntry._fields)
table.set_cols_align(['l']+ ['c']*(len(outputEntry._fields)-1))
return table

15
mktxp/flow/router_entries_handler.py
View File

@@ -22,19 +22,22 @@ class RouterEntriesHandler:
def __init__(self):
self.router_entries = []
for router_name in config_handler.registered_entries():
entry = config_handler.config_entry(router_name)
if entry.enabled:
self.router_entries.append(RouterEntry(router_name))
router_entry = RouterEntriesHandler.router_entry(router_name, enabled_only = True)
if router_entry:
self.router_entries.append(router_entry)
@staticmethod
def router_entry(entry_name, enabled_only = False):
router_entry = None
for router_name in config_handler.registered_entries():
if router_name == entry_name:
if enabled_only:
entry = config_handler.config_entry(router_name)
if not entry.enabled:
config_entry = config_handler.config_entry(router_name)
if enabled_only and not config_entry.enabled:
break
router_entry = RouterEntry(router_name)
router_entry.dhcp_entry = RouterEntriesHandler.router_entry(config_entry.remote_dhcp_entry)
break
return router_entry

19
mktxp/flow/router_entry.py
View File

@@ -27,7 +27,10 @@ class RouterEntry:
MKTXPConfigKeys.ROUTERBOARD_NAME: self.router_name,
MKTXPConfigKeys.ROUTERBOARD_ADDRESS: self.config_entry.hostname
}
self.wifi_package = None
self.dhcp_entry = None
self.time_spent = { 'IdentityCollector': 0,
'SystemResourceCollector': 0,
'HealthCollector': 0,
@@ -49,4 +52,18 @@ class RouterEntry:
'QueueSimpleCollector': 0,
'UserCollector': 0,
'MKTXPCollector': 0
}
}
def is_connected(self):
connected = True
if not self.api_connection.is_connected():
connected = False
# let's get connected now
self.api_connection.connect()
if self.dhcp_entry:
self.dhcp_entry.api_connection.connect()
return connected