diff --git a/README.md b/README.md index 4cd0f75..b2c5a6e 100644 --- a/README.md +++ b/README.md @@ -331,7 +331,7 @@ mktxp is running as pid 36704 ```` ❯ mktxp -h -usage: MKTXP [-h] [--dir DIR] {info, edit, export, print, show, } ... +usage: MKTXP [-h] [--cfg-dir CFG_DIR] {info, edit, export, print, show, } ... ```` To learn more about individual commands, just run it with ```-h```: For example, to learn everything about ````mktxp show````: diff --git a/mktxp/cli/config/config.py b/mktxp/cli/config/config.py index 4568585..85dcf69 100755 --- a/mktxp/cli/config/config.py +++ b/mktxp/cli/config/config.py @@ -62,6 +62,7 @@ class MKTXPConfigKeys: FE_USER_KEY = 'user' FE_QUEUE_KEY = 'queue' + FE_REMOTE_DHCP_ENTRY = 'remote_dhcp_entry' MKTXP_SOCKET_TIMEOUT = 'socket_timeout' MKTXP_INITIAL_DELAY = 'initial_delay_on_failure' @@ -88,6 +89,7 @@ class MKTXPConfigKeys: # Default values DEFAULT_API_PORT = 8728 DEFAULT_API_SSL_PORT = 8729 + DEFAULT_FE_REMOTE_DHCP_ENTRY = 'None' DEFAULT_MKTXP_PORT = 49090 DEFAULT_MKTXP_SOCKET_TIMEOUT = 2 DEFAULT_MKTXP_INITIAL_DELAY = 120 @@ -113,7 +115,7 @@ class MKTXPConfigKeys: SYSTEM_BOOLEAN_KEYS_YES = {MKTXP_BANDWIDTH_KEY} SYSTEM_BOOLEAN_KEYS_NO = {MKTXP_VERBOSE_MODE, MKTXP_FETCH_IN_PARALLEL} - STR_KEYS = (HOST_KEY, USER_KEY, PASSWD_KEY) + STR_KEYS = (HOST_KEY, USER_KEY, PASSWD_KEY, FE_REMOTE_DHCP_ENTRY) MKTXP_INT_KEYS = (PORT_KEY, MKTXP_SOCKET_TIMEOUT, MKTXP_INITIAL_DELAY, MKTXP_MAX_DELAY, MKTXP_INC_DIV, MKTXP_BANDWIDTH_TEST_INTERVAL, MKTXP_MIN_COLLECT_INTERVAL, MKTXP_MAX_WORKER_THREADS, MKTXP_MAX_SCRAPE_DURATION, MKTXP_TOTAL_MAX_SCRAPE_DURATION) @@ -130,7 +132,7 @@ class ConfigEntry: MKTXPConfigKeys.FE_FIREWALL_KEY, MKTXPConfigKeys.FE_MONITOR_KEY, MKTXPConfigKeys.FE_ROUTE_KEY, MKTXPConfigKeys.FE_WIRELESS_KEY, MKTXPConfigKeys.FE_WIRELESS_CLIENTS_KEY, MKTXPConfigKeys.FE_IP_CONNECTIONS_KEY, MKTXPConfigKeys.FE_CAPSMAN_KEY, MKTXPConfigKeys.FE_CAPSMAN_CLIENTS_KEY, MKTXPConfigKeys.FE_POE_KEY, MKTXPConfigKeys.FE_NETWATCH_KEY, MKTXPConfigKeys.MKTXP_USE_COMMENTS_OVER_NAMES, MKTXPConfigKeys.FE_PUBLIC_IP_KEY, MKTXPConfigKeys.FE_IPV6_FIREWALL_KEY, MKTXPConfigKeys.FE_IPV6_NEIGHBOR_KEY, - MKTXPConfigKeys.FE_USER_KEY, MKTXPConfigKeys.FE_QUEUE_KEY + MKTXPConfigKeys.FE_USER_KEY, MKTXPConfigKeys.FE_QUEUE_KEY, MKTXPConfigKeys.FE_REMOTE_DHCP_ENTRY ]) MKTXPSystemEntry = namedtuple('MKTXPSystemEntry', [MKTXPConfigKeys.PORT_KEY, MKTXPConfigKeys.MKTXP_SOCKET_TIMEOUT, MKTXPConfigKeys.MKTXP_INITIAL_DELAY, MKTXPConfigKeys.MKTXP_MAX_DELAY, @@ -277,7 +279,12 @@ class MKTXPConfigHandler: new_keys.append(key) # read from disk next time for key in MKTXPConfigKeys.STR_KEYS: - config_entry_reader[key] = self.config[entry_name][key] + if self.config[entry_name].get(key): + config_entry_reader[key] = self.config[entry_name].get(key) + else: + config_entry_reader[key] = self._default_value_for_key(key) + new_keys.append(key) # read from disk next time + if key is MKTXPConfigKeys.PASSWD_KEY and type(config_entry_reader[key]) is list: config_entry_reader[key] = ','.join(config_entry_reader[key]) @@ -338,6 +345,7 @@ class MKTXPConfigHandler: return { MKTXPConfigKeys.SSL_KEY: lambda value: MKTXPConfigKeys.DEFAULT_API_SSL_PORT if value else MKTXPConfigKeys.DEFAULT_API_PORT, MKTXPConfigKeys.PORT_KEY: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_PORT, + MKTXPConfigKeys.FE_REMOTE_DHCP_ENTRY: lambda value: MKTXPConfigKeys.DEFAULT_FE_REMOTE_DHCP_ENTRY, MKTXPConfigKeys.MKTXP_SOCKET_TIMEOUT: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_SOCKET_TIMEOUT, MKTXPConfigKeys.MKTXP_INITIAL_DELAY: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_INITIAL_DELAY, MKTXPConfigKeys.MKTXP_MAX_DELAY: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_MAX_DELAY, diff --git a/mktxp/cli/config/mktxp.conf b/mktxp/cli/config/mktxp.conf index 22beb0e..cc124da 100644 --- a/mktxp/cli/config/mktxp.conf +++ b/mktxp/cli/config/mktxp.conf @@ -47,5 +47,6 @@ user = True # Active Users metrics queue = True # Queues metrics + remote_dhcp_entry = None # Alternative mktxp entry for DHCP info resolution (capsman/wireless) use_comments_over_names = True # when available, forces using comments over the interfaces names \ No newline at end of file diff --git a/mktxp/cli/options.py b/mktxp/cli/options.py index f0d7e5c..a67b7ac 100755 --- a/mktxp/cli/options.py +++ b/mktxp/cli/options.py @@ -66,8 +66,8 @@ Selected metrics info can be printed on the command line. For more information, global_options_parser = ArgumentParser(add_help=False) self.parse_global_options(global_options_parser) namespace, _ = global_options_parser.parse_known_args() - if namespace.dir: - config_handler(CustomConfig(namespace.dir)) + if namespace.cfg_dir: + config_handler(CustomConfig(namespace.cfg_dir)) else: config_handler() @@ -84,7 +84,7 @@ Selected metrics info can be printed on the command line. For more information, def parse_global_options(self, parser): ''' Parses global options ''' - parser.add_argument('--dir', dest = 'dir', + parser.add_argument('--cfg-dir', dest = 'cfg_dir', type = lambda d: self._is_valid_dir_path(parser, d), help = 'MKTXP config files directory (optional)') diff --git a/mktxp/cli/output/capsman_out.py b/mktxp/cli/output/capsman_out.py index db407e8..a8b571b 100644 --- a/mktxp/cli/output/capsman_out.py +++ b/mktxp/cli/output/capsman_out.py @@ -14,9 +14,9 @@ from mktxp.flow.processor.output import BaseOutputProcessor from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource +from mktxp.datasource.wireless_ds import WirelessMetricsDataSource from mktxp.datasource.capsman_ds import CapsmanRegistrationsMetricsDataSource - class CapsmanOutput: ''' CAPsMAN CLI Output ''' @@ -30,7 +30,8 @@ class CapsmanOutput: # translate / trim / augment registration records dhcp_lease_labels = ['host_name', 'comment', 'address', 'mac_address'] - dhcp_lease_records = DHCPMetricsDataSource.metric_records(router_entry, metric_labels = dhcp_lease_labels, add_router_id = False) + dhcp_entry = WirelessMetricsDataSource.dhcp_entry(router_entry) + dhcp_lease_records = DHCPMetricsDataSource.metric_records(dhcp_entry, metric_labels = dhcp_lease_labels, add_router_id = False) dhcp_rt_by_interface = {} for registration_record in sorted(registration_records, key = lambda rt_record: rt_record['rx_signal'], reverse=True): @@ -60,4 +61,3 @@ class CapsmanOutput: print(f'{server} clients: {len(dhcp_rt_by_interface[server])}') print(f'Total connected CAPsMAN clients: {output_records}', '\n') - diff --git a/mktxp/cli/output/wifi_out.py b/mktxp/cli/output/wifi_out.py index 0155c01..d489bb3 100644 --- a/mktxp/cli/output/wifi_out.py +++ b/mktxp/cli/output/wifi_out.py @@ -15,6 +15,7 @@ from mktxp.flow.processor.output import BaseOutputProcessor from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource from mktxp.datasource.wireless_ds import WirelessMetricsDataSource +from mktxp.flow.router_entries_handler import RouterEntriesHandler class WirelessOutput: @@ -30,7 +31,8 @@ class WirelessOutput: # translate / trim / augment registration records dhcp_lease_labels = ['host_name', 'comment', 'address', 'mac_address'] - dhcp_lease_records = DHCPMetricsDataSource.metric_records(router_entry, metric_labels = dhcp_lease_labels, add_router_id = False) + dhcp_entry = WirelessMetricsDataSource.dhcp_entry(router_entry) + dhcp_lease_records = DHCPMetricsDataSource.metric_records(dhcp_entry, metric_labels = dhcp_lease_labels, add_router_id = False) dhcp_rt_by_interface = {} @@ -46,7 +48,8 @@ class WirelessOutput: output_records = 0 registration_records = len(registration_records) - output_entry = BaseOutputProcessor.OutputWiFiEntry + output_entry = BaseOutputProcessor.OutputWiFiWave2Entry \ + if WirelessMetricsDataSource.wifiwave2_installed(router_entry) else BaseOutputProcessor.OutputWiFiEntry output_table = BaseOutputProcessor.output_table(output_entry) for key in dhcp_rt_by_interface.keys(): diff --git a/mktxp/collector/capsman_collector.py b/mktxp/collector/capsman_collector.py index 9c70dc5..e5a02d3 100644 --- a/mktxp/collector/capsman_collector.py +++ b/mktxp/collector/capsman_collector.py @@ -17,6 +17,7 @@ from mktxp.flow.processor.output import BaseOutputProcessor from mktxp.collector.base_collector import BaseCollector from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource from mktxp.datasource.capsman_ds import CapsmanCapsMetricsDataSource, CapsmanRegistrationsMetricsDataSource, CapsmanInterfacesDatasource +from mktxp.datasource.wireless_ds import WirelessMetricsDataSource class CapsmanCollector(BaseCollector): @@ -51,9 +52,11 @@ class CapsmanCollector(BaseCollector): # the client info metrics if router_entry.config_entry.capsman_clients: + # translate / trim / augment registration records dhcp_lease_labels = ['mac_address', 'address', 'host_name', 'comment'] - dhcp_lease_records = DHCPMetricsDataSource.metric_records(router_entry, metric_labels = dhcp_lease_labels) + dhcp_entry = WirelessMetricsDataSource.dhcp_entry(router_entry) + dhcp_lease_records = DHCPMetricsDataSource.metric_records(dhcp_entry, metric_labels = dhcp_lease_labels) for registration_record in registration_records: BaseOutputProcessor.augment_record(router_entry, registration_record, dhcp_lease_records) diff --git a/mktxp/collector/wlan_collector.py b/mktxp/collector/wlan_collector.py index d486f35..e80db8f 100644 --- a/mktxp/collector/wlan_collector.py +++ b/mktxp/collector/wlan_collector.py @@ -54,7 +54,8 @@ class WLANCollector(BaseCollector): registration_records = WirelessMetricsDataSource.metric_records(router_entry, metric_labels = registration_labels) if registration_records: dhcp_lease_labels = ['mac_address', 'address', 'host_name', 'comment'] - dhcp_lease_records = DHCPMetricsDataSource.metric_records(router_entry, metric_labels = dhcp_lease_labels) + dhcp_entry = WirelessMetricsDataSource.dhcp_entry(router_entry) + dhcp_lease_records = DHCPMetricsDataSource.metric_records(dhcp_entry, metric_labels = dhcp_lease_labels) for registration_record in registration_records: BaseOutputProcessor.augment_record(router_entry, registration_record, dhcp_lease_records) diff --git a/mktxp/datasource/capsman_ds.py b/mktxp/datasource/capsman_ds.py index 02587d6..2c702d0 100644 --- a/mktxp/datasource/capsman_ds.py +++ b/mktxp/datasource/capsman_ds.py @@ -19,14 +19,14 @@ from mktxp.datasource.wireless_ds import WirelessMetricsDataSource class CapsmanInfo: @staticmethod def capsman_path(router_entry): - if WirelessMetricsDataSource.wireless_package(router_entry) == WirelessMetricsDataSource.WIFIWAVE2: + if WirelessMetricsDataSource.wifiwave2_installed(router_entry): return '/interface/wifiwave2/capsman' else: return '/caps-man' @staticmethod def registration_table_path(router_entry): - if WirelessMetricsDataSource.wireless_package(router_entry) == WirelessMetricsDataSource.WIFIWAVE2: + if WirelessMetricsDataSource.wifiwave2_installed(router_entry): return '/interface/wifiwave2/registration-table' else: return '/caps-man/registration-table' @@ -57,6 +57,12 @@ class CapsmanRegistrationsMetricsDataSource: try: registration_table_path = CapsmanInfo.registration_table_path(router_entry) registration_table_records = router_entry.api_connection.router_api().get_resource(f'{registration_table_path}').get() + + # With wifiwave2, Mikrotik renamed the field 'rx_signal' to 'signal' + for record in registration_table_records: + if 'signal' in record: + record['rx_signal'] = record['signal'] + return BaseDSProcessor.trimmed_records(router_entry, router_records = registration_table_records, metric_labels = metric_labels, add_router_id = add_router_id) except Exception as exc: print(f'Error getting CAPsMAN registration table info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}') diff --git a/mktxp/datasource/wireless_ds.py b/mktxp/datasource/wireless_ds.py index 56cf995..8b29369 100644 --- a/mktxp/datasource/wireless_ds.py +++ b/mktxp/datasource/wireless_ds.py @@ -22,7 +22,6 @@ class WirelessMetricsDataSource: WIFIWAVE2 = 'wifiwave2' WIRELESS = 'wireless' - @staticmethod def metric_records(router_entry, *, metric_labels = None, add_router_id = True): if metric_labels is None: @@ -31,8 +30,7 @@ class WirelessMetricsDataSource: wireless_package = WirelessMetricsDataSource.wireless_package(router_entry) registration_table_records = router_entry.api_connection.router_api().get_resource(f'/interface/{wireless_package}/registration-table').get() - # Mikrotik renamed the field 'signal_strength' to 'signal' when using wifiwave2. - # Rename this field back to 'signal_strength' to preserve backwards compatibility + # With wifiwave2, Mikrotik renamed the field 'signal_strength' to 'signal' for record in registration_table_records: if 'signal' in record: record['signal_strength'] = record['signal'] @@ -49,3 +47,15 @@ class WirelessMetricsDataSource: ww2_installed = PackageMetricsDataSource.is_package_installed(router_entry, package_name = WirelessMetricsDataSource.WIFIWAVE2) router_entry.wifi_package = WirelessMetricsDataSource.WIFIWAVE2 if ww2_installed else WirelessMetricsDataSource.WIRELESS return router_entry.wifi_package + + @staticmethod + def wifiwave2_installed(router_entry): + return WirelessMetricsDataSource.wireless_package(router_entry) == WirelessMetricsDataSource.WIFIWAVE2 + + @staticmethod + def dhcp_entry(router_entry): + if router_entry.dhcp_entry: + return router_entry.dhcp_entry + return router_entry + + diff --git a/mktxp/flow/collector_handler.py b/mktxp/flow/collector_handler.py index b56688a..e06aceb 100644 --- a/mktxp/flow/collector_handler.py +++ b/mktxp/flow/collector_handler.py @@ -35,9 +35,8 @@ class CollectorHandler: Thus, the total runtime of this function scales linearly with the number of registered routers. """ for router_entry in self.entries_handler.router_entries: - if not router_entry.api_connection.is_connected(): + if not router_entry.is_connected(): # let's pick up on things in the next run - router_entry.api_connection.connect() continue for collector_ID, collect_func in self.collector_registry.registered_collectors.items(): @@ -88,9 +87,8 @@ class CollectorHandler: print(f'Hit overall timeout while scraping router entry: {router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME]}') break - if not router_entry.api_connection.is_connected(): + if not router_entry.is_connected(): # let's pick up on things in the next run - router_entry.api_connection.connect() continue # Duration of individual scrapes diff --git a/mktxp/flow/processor/output.py b/mktxp/flow/processor/output.py index 56ba4e0..3a3272a 100644 --- a/mktxp/flow/processor/output.py +++ b/mktxp/flow/processor/output.py @@ -18,6 +18,8 @@ from collections import namedtuple from texttable import Texttable from humanize import naturaldelta from mktxp.cli.config.config import config_handler +from mktxp.datasource.wireless_ds import WirelessMetricsDataSource +from math import floor, log class BaseOutputProcessor: @@ -27,18 +29,23 @@ class BaseOutputProcessor: OutputWiFiEntry = namedtuple('OutputWiFiEntry', ['dhcp_name', 'dhcp_address', 'mac_address', 'signal_strength', 'signal_to_noise', 'interface', 'tx_rate', 'rx_rate', 'uptime']) OutputWiFiEntry.__new__.__defaults__ = ('',) * len(OutputWiFiEntry._fields) + OutputWiFiWave2Entry = namedtuple('OutputWiFiWave2Entry', ['dhcp_name', 'dhcp_address', 'mac_address', 'signal_strength', 'interface', 'tx_rate', 'rx_rate', 'uptime']) + OutputWiFiWave2Entry.__new__.__defaults__ = ('',) * len(OutputWiFiWave2Entry._fields) + OutputDHCPEntry = namedtuple('OutputDHCPEntry', ['host_name', 'server', 'mac_address', 'address', 'active_address', 'expires_after']) OutputDHCPEntry.__new__.__defaults__ = ('',) * len(OutputDHCPEntry._fields) @staticmethod def augment_record(router_entry, registration_record, dhcp_lease_records): - try: - dhcp_lease_record = next((dhcp_lease_record for dhcp_lease_record in dhcp_lease_records if dhcp_lease_record.get('mac_address')==registration_record.get('mac_address'))) - dhcp_name = BaseOutputProcessor.dhcp_name(router_entry, dhcp_lease_record) - dhcp_address = dhcp_lease_record.get('address', '') - except StopIteration: - dhcp_name = registration_record.get('mac_address') - dhcp_address = 'No DHCP Record' + dhcp_name = registration_record.get('mac_address') + dhcp_address = 'No DHCP Record' + if dhcp_lease_records: + try: + dhcp_lease_record = next((dhcp_lease_record for dhcp_lease_record in dhcp_lease_records if dhcp_lease_record.get('mac_address')==registration_record.get('mac_address'))) + dhcp_name = BaseOutputProcessor.dhcp_name(router_entry, dhcp_lease_record) + dhcp_address = dhcp_lease_record.get('address', '') + except StopIteration: + pass registration_record['dhcp_name'] = dhcp_name registration_record['dhcp_address'] = dhcp_address @@ -49,10 +56,13 @@ class BaseOutputProcessor: registration_record['rx_bytes'] = registration_record['bytes'].split(',')[1] del registration_record['bytes'] + ww2_installed = WirelessMetricsDataSource.wifiwave2_installed(router_entry) if registration_record.get('tx_rate'): - registration_record['tx_rate'] = BaseOutputProcessor.parse_rates(registration_record['tx_rate']) + registration_record['tx_rate'] = BaseOutputProcessor.parse_bitrates(registration_record['tx_rate']) \ + if ww2_installed else BaseOutputProcessor.parse_rates(registration_record['tx_rate']) if registration_record.get('rx_rate'): - registration_record['rx_rate'] = BaseOutputProcessor.parse_rates(registration_record['rx_rate']) + registration_record['rx_rate'] = BaseOutputProcessor.parse_bitrates(registration_record['rx_rate']) \ + if ww2_installed else BaseOutputProcessor.parse_rates(registration_record['rx_rate']) if registration_record.get('uptime'): registration_record['uptime'] = naturaldelta(BaseOutputProcessor.parse_timedelta_seconds(registration_record['uptime']), months=True, minimum_unit='seconds') @@ -87,6 +97,12 @@ class BaseOutputProcessor: rc = wifi_rates_rgx.search(rate) return f'{int(float(rc[1]))} {rc[2]}' if rc and len(rc.groups()) == 2 else rate + @staticmethod + def parse_bitrates(rate): + rate = int(rate) + power = floor(log(rate, 1000)) + return f"{int(rate / 1000 ** power)} {['bps', 'Kbps', 'Mbps', 'Gbps'][int(power)]}" + @staticmethod def parse_timedelta(time): duration_interval_rgx = config_handler.re_compiled.get('duration_interval_rgx') @@ -126,6 +142,3 @@ class BaseOutputProcessor: table.header(outputEntry._fields) table.set_cols_align(['l']+ ['c']*(len(outputEntry._fields)-1)) return table - - - diff --git a/mktxp/flow/router_entries_handler.py b/mktxp/flow/router_entries_handler.py index 012cee6..f5f734b 100644 --- a/mktxp/flow/router_entries_handler.py +++ b/mktxp/flow/router_entries_handler.py @@ -22,19 +22,22 @@ class RouterEntriesHandler: def __init__(self): self.router_entries = [] for router_name in config_handler.registered_entries(): - entry = config_handler.config_entry(router_name) - if entry.enabled: - self.router_entries.append(RouterEntry(router_name)) + router_entry = RouterEntriesHandler.router_entry(router_name, enabled_only = True) + if router_entry: + self.router_entries.append(router_entry) @staticmethod def router_entry(entry_name, enabled_only = False): router_entry = None + for router_name in config_handler.registered_entries(): if router_name == entry_name: - if enabled_only: - entry = config_handler.config_entry(router_name) - if not entry.enabled: + config_entry = config_handler.config_entry(router_name) + if enabled_only and not config_entry.enabled: break + router_entry = RouterEntry(router_name) + router_entry.dhcp_entry = RouterEntriesHandler.router_entry(config_entry.remote_dhcp_entry) break + return router_entry diff --git a/mktxp/flow/router_entry.py b/mktxp/flow/router_entry.py index 9de4605..b501610 100644 --- a/mktxp/flow/router_entry.py +++ b/mktxp/flow/router_entry.py @@ -27,7 +27,10 @@ class RouterEntry: MKTXPConfigKeys.ROUTERBOARD_NAME: self.router_name, MKTXPConfigKeys.ROUTERBOARD_ADDRESS: self.config_entry.hostname } + self.wifi_package = None + self.dhcp_entry = None + self.time_spent = { 'IdentityCollector': 0, 'SystemResourceCollector': 0, 'HealthCollector': 0, @@ -49,4 +52,18 @@ class RouterEntry: 'QueueSimpleCollector': 0, 'UserCollector': 0, 'MKTXPCollector': 0 - } + } + + def is_connected(self): + connected = True + if not self.api_connection.is_connected(): + connected = False + # let's get connected now + self.api_connection.connect() + if self.dhcp_entry: + self.dhcp_entry.api_connection.connect() + + return connected + + +