remote dhcp info resolution (wireless/capsman), more wifiwave2 support

README.md

@@ -331,7 +331,7 @@ mktxp is running as pid 36704
 
 ````
 ❯ mktxp -h
-usage: MKTXP [-h] [--dir DIR] {info, edit, export, print, show, } ...
+usage: MKTXP [-h] [--cfg-dir CFG_DIR] {info, edit, export, print, show, } ...
 ````
 To learn more about individual commands, just run it with ```-h```:
 For example, to learn everything about ````mktxp show````:

mktxp/cli/config/config.py

@@ -62,6 +62,7 @@ class MKTXPConfigKeys:
 
     FE_USER_KEY = 'user'
     FE_QUEUE_KEY = 'queue'
+    FE_REMOTE_DHCP_ENTRY = 'remote_dhcp_entry'
 
     MKTXP_SOCKET_TIMEOUT = 'socket_timeout'
     MKTXP_INITIAL_DELAY = 'initial_delay_on_failure'
@@ -88,6 +89,7 @@ class MKTXPConfigKeys:
     # Default values
     DEFAULT_API_PORT = 8728
     DEFAULT_API_SSL_PORT = 8729
+    DEFAULT_FE_REMOTE_DHCP_ENTRY = 'None'
     DEFAULT_MKTXP_PORT = 49090
     DEFAULT_MKTXP_SOCKET_TIMEOUT = 2
     DEFAULT_MKTXP_INITIAL_DELAY = 120
@@ -113,7 +115,7 @@ class MKTXPConfigKeys:
     SYSTEM_BOOLEAN_KEYS_YES = {MKTXP_BANDWIDTH_KEY}
     SYSTEM_BOOLEAN_KEYS_NO = {MKTXP_VERBOSE_MODE, MKTXP_FETCH_IN_PARALLEL}
 
-    STR_KEYS = (HOST_KEY, USER_KEY, PASSWD_KEY)
+    STR_KEYS = (HOST_KEY, USER_KEY, PASSWD_KEY, FE_REMOTE_DHCP_ENTRY)
     MKTXP_INT_KEYS = (PORT_KEY, MKTXP_SOCKET_TIMEOUT, MKTXP_INITIAL_DELAY, MKTXP_MAX_DELAY,
                       MKTXP_INC_DIV, MKTXP_BANDWIDTH_TEST_INTERVAL, MKTXP_MIN_COLLECT_INTERVAL,
                       MKTXP_MAX_WORKER_THREADS, MKTXP_MAX_SCRAPE_DURATION, MKTXP_TOTAL_MAX_SCRAPE_DURATION)
@@ -130,7 +132,7 @@ class ConfigEntry:
                                                        MKTXPConfigKeys.FE_FIREWALL_KEY, MKTXPConfigKeys.FE_MONITOR_KEY, MKTXPConfigKeys.FE_ROUTE_KEY, MKTXPConfigKeys.FE_WIRELESS_KEY, MKTXPConfigKeys.FE_WIRELESS_CLIENTS_KEY,
                                                        MKTXPConfigKeys.FE_IP_CONNECTIONS_KEY, MKTXPConfigKeys.FE_CAPSMAN_KEY, MKTXPConfigKeys.FE_CAPSMAN_CLIENTS_KEY, MKTXPConfigKeys.FE_POE_KEY, MKTXPConfigKeys.FE_NETWATCH_KEY,
                                                        MKTXPConfigKeys.MKTXP_USE_COMMENTS_OVER_NAMES, MKTXPConfigKeys.FE_PUBLIC_IP_KEY, MKTXPConfigKeys.FE_IPV6_FIREWALL_KEY, MKTXPConfigKeys.FE_IPV6_NEIGHBOR_KEY,
-                                                       MKTXPConfigKeys.FE_USER_KEY, MKTXPConfigKeys.FE_QUEUE_KEY
+                                                       MKTXPConfigKeys.FE_USER_KEY, MKTXPConfigKeys.FE_QUEUE_KEY, MKTXPConfigKeys.FE_REMOTE_DHCP_ENTRY
                                                        ])
     MKTXPSystemEntry = namedtuple('MKTXPSystemEntry', [MKTXPConfigKeys.PORT_KEY, MKTXPConfigKeys.MKTXP_SOCKET_TIMEOUT,
                                                        MKTXPConfigKeys.MKTXP_INITIAL_DELAY, MKTXPConfigKeys.MKTXP_MAX_DELAY,
@@ -277,7 +279,12 @@ class MKTXPConfigHandler:
                 new_keys.append(key) # read from disk next time
 
         for key in MKTXPConfigKeys.STR_KEYS:
-            config_entry_reader[key] = self.config[entry_name][key]
+            if self.config[entry_name].get(key):
+                config_entry_reader[key] = self.config[entry_name].get(key)
+            else:
+                config_entry_reader[key] = self._default_value_for_key(key)
+                new_keys.append(key) # read from disk next time
+
             if key is MKTXPConfigKeys.PASSWD_KEY and type(config_entry_reader[key]) is list:
                 config_entry_reader[key] = ','.join(config_entry_reader[key])
 
@@ -338,6 +345,7 @@ class MKTXPConfigHandler:
         return {
             MKTXPConfigKeys.SSL_KEY: lambda value: MKTXPConfigKeys.DEFAULT_API_SSL_PORT if value else MKTXPConfigKeys.DEFAULT_API_PORT,
             MKTXPConfigKeys.PORT_KEY: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_PORT,
+            MKTXPConfigKeys.FE_REMOTE_DHCP_ENTRY:  lambda value: MKTXPConfigKeys.DEFAULT_FE_REMOTE_DHCP_ENTRY,
             MKTXPConfigKeys.MKTXP_SOCKET_TIMEOUT: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_SOCKET_TIMEOUT,
             MKTXPConfigKeys.MKTXP_INITIAL_DELAY: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_INITIAL_DELAY,
             MKTXPConfigKeys.MKTXP_MAX_DELAY: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_MAX_DELAY,

mktxp/cli/config/mktxp.conf

@@ -47,5 +47,6 @@
 
     user = True                     # Active Users metrics
     queue = True                    # Queues metrics
+    remote_dhcp_entry = None        # Alternative mktxp entry for DHCP info resolution (capsman/wireless)
 
     use_comments_over_names = True  # when available, forces using comments over the interfaces names 

mktxp/cli/options.py

@@ -66,8 +66,8 @@ Selected metrics info can be printed on the command line. For more information,
         global_options_parser = ArgumentParser(add_help=False)
         self.parse_global_options(global_options_parser)
         namespace, _ = global_options_parser.parse_known_args()    
-        if namespace.dir:
+        if namespace.cfg_dir:
-            config_handler(CustomConfig(namespace.dir))
+            config_handler(CustomConfig(namespace.cfg_dir))
         else:
             config_handler()
 
@@ -84,7 +84,7 @@ Selected metrics info can be printed on the command line. For more information,
     def parse_global_options(self, parser):
         ''' Parses global options
         '''
-        parser.add_argument('--dir', dest = 'dir', 
+        parser.add_argument('--cfg-dir', dest = 'cfg_dir', 
                     type = lambda d: self._is_valid_dir_path(parser, d),
                     help = 'MKTXP config files directory (optional)')
 

mktxp/cli/output/capsman_out.py

@@ -14,9 +14,9 @@
 
 from mktxp.flow.processor.output import BaseOutputProcessor
 from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource
+from mktxp.datasource.wireless_ds import WirelessMetricsDataSource
 from mktxp.datasource.capsman_ds import CapsmanRegistrationsMetricsDataSource
 
-
 class CapsmanOutput:
     ''' CAPsMAN CLI Output
     '''    
@@ -30,7 +30,8 @@ class CapsmanOutput:
 
         # translate / trim / augment registration records
         dhcp_lease_labels = ['host_name', 'comment', 'address', 'mac_address']
-        dhcp_lease_records = DHCPMetricsDataSource.metric_records(router_entry, metric_labels = dhcp_lease_labels, add_router_id = False)   
+        dhcp_entry = WirelessMetricsDataSource.dhcp_entry(router_entry)
+        dhcp_lease_records = DHCPMetricsDataSource.metric_records(dhcp_entry, metric_labels = dhcp_lease_labels, add_router_id = False)   
 
         dhcp_rt_by_interface = {}
         for registration_record in sorted(registration_records, key = lambda rt_record: rt_record['rx_signal'], reverse=True):
@@ -60,4 +61,3 @@ class CapsmanOutput:
             print(f'{server} clients: {len(dhcp_rt_by_interface[server])}')
         print(f'Total connected CAPsMAN clients: {output_records}', '\n')
 
-

mktxp/cli/output/wifi_out.py

@@ -15,6 +15,7 @@
 from mktxp.flow.processor.output import BaseOutputProcessor
 from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource
 from mktxp.datasource.wireless_ds import WirelessMetricsDataSource
+from mktxp.flow.router_entries_handler import RouterEntriesHandler
 
 
 class WirelessOutput:
@@ -30,7 +31,8 @@ class WirelessOutput:
 
         # translate / trim / augment registration records
         dhcp_lease_labels = ['host_name', 'comment', 'address', 'mac_address']
-        dhcp_lease_records = DHCPMetricsDataSource.metric_records(router_entry, metric_labels = dhcp_lease_labels, add_router_id = False)   
+        dhcp_entry = WirelessMetricsDataSource.dhcp_entry(router_entry)
+        dhcp_lease_records = DHCPMetricsDataSource.metric_records(dhcp_entry, metric_labels = dhcp_lease_labels, add_router_id = False)   
 
         dhcp_rt_by_interface = {}
 
@@ -46,7 +48,8 @@ class WirelessOutput:
 
         output_records = 0
         registration_records = len(registration_records)                
-        output_entry = BaseOutputProcessor.OutputWiFiEntry
+        output_entry = BaseOutputProcessor.OutputWiFiWave2Entry \
+                        if WirelessMetricsDataSource.wifiwave2_installed(router_entry) else BaseOutputProcessor.OutputWiFiEntry
         output_table = BaseOutputProcessor.output_table(output_entry)
         
         for key in dhcp_rt_by_interface.keys():

mktxp/collector/capsman_collector.py

@@ -17,6 +17,7 @@ from mktxp.flow.processor.output import BaseOutputProcessor
 from mktxp.collector.base_collector import BaseCollector
 from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource
 from mktxp.datasource.capsman_ds import CapsmanCapsMetricsDataSource, CapsmanRegistrationsMetricsDataSource, CapsmanInterfacesDatasource
+from mktxp.datasource.wireless_ds import WirelessMetricsDataSource
 
 
 class CapsmanCollector(BaseCollector):
@@ -51,9 +52,11 @@ class CapsmanCollector(BaseCollector):
 
             # the client info metrics
             if router_entry.config_entry.capsman_clients:
+
                 # translate / trim / augment registration records
                 dhcp_lease_labels = ['mac_address', 'address', 'host_name', 'comment']
-                dhcp_lease_records = DHCPMetricsDataSource.metric_records(router_entry, metric_labels = dhcp_lease_labels)
+                dhcp_entry = WirelessMetricsDataSource.dhcp_entry(router_entry)
+                dhcp_lease_records = DHCPMetricsDataSource.metric_records(dhcp_entry, metric_labels = dhcp_lease_labels)
                 for registration_record in registration_records:
                     BaseOutputProcessor.augment_record(router_entry, registration_record, dhcp_lease_records)
                     

mktxp/collector/wlan_collector.py

@@ -54,7 +54,8 @@ class WLANCollector(BaseCollector):
             registration_records = WirelessMetricsDataSource.metric_records(router_entry, metric_labels = registration_labels)
             if registration_records:
                 dhcp_lease_labels = ['mac_address', 'address', 'host_name', 'comment']
-                dhcp_lease_records = DHCPMetricsDataSource.metric_records(router_entry, metric_labels = dhcp_lease_labels)
+                dhcp_entry = WirelessMetricsDataSource.dhcp_entry(router_entry)            
+                dhcp_lease_records = DHCPMetricsDataSource.metric_records(dhcp_entry, metric_labels = dhcp_lease_labels)
       
                 for registration_record in registration_records:
                     BaseOutputProcessor.augment_record(router_entry, registration_record, dhcp_lease_records)                

mktxp/datasource/capsman_ds.py

@@ -19,14 +19,14 @@ from mktxp.datasource.wireless_ds import WirelessMetricsDataSource
 class CapsmanInfo:
     @staticmethod
     def capsman_path(router_entry):
-        if WirelessMetricsDataSource.wireless_package(router_entry) == WirelessMetricsDataSource.WIFIWAVE2:
+        if WirelessMetricsDataSource.wifiwave2_installed(router_entry):
             return '/interface/wifiwave2/capsman'
         else:
             return '/caps-man'
 
     @staticmethod
     def registration_table_path(router_entry):
-        if WirelessMetricsDataSource.wireless_package(router_entry) == WirelessMetricsDataSource.WIFIWAVE2:
+        if WirelessMetricsDataSource.wifiwave2_installed(router_entry):
             return '/interface/wifiwave2/registration-table'
         else:
             return '/caps-man/registration-table'
@@ -57,6 +57,12 @@ class CapsmanRegistrationsMetricsDataSource:
         try:
             registration_table_path = CapsmanInfo.registration_table_path(router_entry)
             registration_table_records = router_entry.api_connection.router_api().get_resource(f'{registration_table_path}').get()
+            
+            # With wifiwave2, Mikrotik renamed the field 'rx_signal' to 'signal' 
+            for record in registration_table_records:
+                if 'signal' in record:
+                    record['rx_signal'] = record['signal']
+
             return BaseDSProcessor.trimmed_records(router_entry, router_records = registration_table_records, metric_labels = metric_labels, add_router_id = add_router_id)
         except Exception as exc:
             print(f'Error getting CAPsMAN registration table info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}')

mktxp/datasource/wireless_ds.py

@@ -22,7 +22,6 @@ class WirelessMetricsDataSource:
     WIFIWAVE2 = 'wifiwave2'
     WIRELESS = 'wireless'
 
-
     @staticmethod
     def metric_records(router_entry, *, metric_labels = None, add_router_id = True):
         if metric_labels is None:
@@ -31,8 +30,7 @@ class WirelessMetricsDataSource:
             wireless_package = WirelessMetricsDataSource.wireless_package(router_entry)
             registration_table_records = router_entry.api_connection.router_api().get_resource(f'/interface/{wireless_package}/registration-table').get()
 
-            # Mikrotik renamed the field 'signal_strength' to 'signal' when using wifiwave2.
+            # With wifiwave2, Mikrotik renamed the field 'signal_strength' to 'signal' 
-            # Rename this field back to 'signal_strength' to preserve backwards compatibility
             for record in registration_table_records:
                 if 'signal' in record:
                     record['signal_strength'] = record['signal']
@@ -49,3 +47,15 @@ class WirelessMetricsDataSource:
             ww2_installed = PackageMetricsDataSource.is_package_installed(router_entry, package_name = WirelessMetricsDataSource.WIFIWAVE2)
             router_entry.wifi_package = WirelessMetricsDataSource.WIFIWAVE2 if ww2_installed else WirelessMetricsDataSource.WIRELESS
         return router_entry.wifi_package
+
+    @staticmethod
+    def wifiwave2_installed(router_entry):
+        return WirelessMetricsDataSource.wireless_package(router_entry) == WirelessMetricsDataSource.WIFIWAVE2
+
+    @staticmethod
+    def dhcp_entry(router_entry):
+        if router_entry.dhcp_entry:
+            return router_entry.dhcp_entry
+        return router_entry
+
+

mktxp/flow/collector_handler.py

@@ -35,9 +35,8 @@ class CollectorHandler:
         Thus, the total runtime of this function scales linearly with the number of registered routers.
         """
         for router_entry in self.entries_handler.router_entries:
-            if not router_entry.api_connection.is_connected():
+            if not router_entry.is_connected():
                 # let's pick up on things in the next run
-                router_entry.api_connection.connect()
                 continue
 
             for collector_ID, collect_func in self.collector_registry.registered_collectors.items():
@@ -88,9 +87,8 @@ class CollectorHandler:
                     print(f'Hit overall timeout while scraping router entry: {router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME]}')
                     break
 
-                if not router_entry.api_connection.is_connected():
+                if not router_entry.is_connected():
                     # let's pick up on things in the next run
-                    router_entry.api_connection.connect()
                     continue
                 
                 # Duration of individual scrapes

mktxp/flow/processor/output.py

@@ -18,6 +18,8 @@ from collections import namedtuple
 from texttable import Texttable
 from humanize import naturaldelta
 from mktxp.cli.config.config import config_handler
+from mktxp.datasource.wireless_ds import WirelessMetricsDataSource
+from math import floor, log
 
 
 class BaseOutputProcessor:
@@ -27,18 +29,23 @@ class BaseOutputProcessor:
     OutputWiFiEntry = namedtuple('OutputWiFiEntry', ['dhcp_name', 'dhcp_address', 'mac_address', 'signal_strength', 'signal_to_noise', 'interface', 'tx_rate', 'rx_rate', 'uptime'])
     OutputWiFiEntry.__new__.__defaults__ = ('',) * len(OutputWiFiEntry._fields)
 
+    OutputWiFiWave2Entry = namedtuple('OutputWiFiWave2Entry', ['dhcp_name', 'dhcp_address', 'mac_address', 'signal_strength', 'interface', 'tx_rate', 'rx_rate', 'uptime'])
+    OutputWiFiWave2Entry.__new__.__defaults__ = ('',) * len(OutputWiFiWave2Entry._fields)
+
     OutputDHCPEntry = namedtuple('OutputDHCPEntry', ['host_name', 'server', 'mac_address', 'address', 'active_address', 'expires_after'])
     OutputDHCPEntry.__new__.__defaults__ = ('',) * len(OutputDHCPEntry._fields)
 
     @staticmethod
     def augment_record(router_entry, registration_record, dhcp_lease_records):
+        dhcp_name = registration_record.get('mac_address')
+        dhcp_address = 'No DHCP Record'              
+        if dhcp_lease_records:
             try:
                 dhcp_lease_record = next((dhcp_lease_record for dhcp_lease_record in dhcp_lease_records if dhcp_lease_record.get('mac_address')==registration_record.get('mac_address')))
                 dhcp_name = BaseOutputProcessor.dhcp_name(router_entry, dhcp_lease_record)
                 dhcp_address = dhcp_lease_record.get('address', '')
             except StopIteration:
-            dhcp_name = registration_record.get('mac_address')
+                pass
-            dhcp_address = 'No DHCP Record'          
 
         registration_record['dhcp_name'] = dhcp_name
         registration_record['dhcp_address'] = dhcp_address
@@ -49,10 +56,13 @@ class BaseOutputProcessor:
             registration_record['rx_bytes'] = registration_record['bytes'].split(',')[1]
             del registration_record['bytes']
 
+        ww2_installed = WirelessMetricsDataSource.wifiwave2_installed(router_entry)
         if registration_record.get('tx_rate'):
-            registration_record['tx_rate'] = BaseOutputProcessor.parse_rates(registration_record['tx_rate'])
+            registration_record['tx_rate'] = BaseOutputProcessor.parse_bitrates(registration_record['tx_rate']) \
+                                                if ww2_installed else BaseOutputProcessor.parse_rates(registration_record['tx_rate'])
         if registration_record.get('rx_rate'):
-            registration_record['rx_rate'] = BaseOutputProcessor.parse_rates(registration_record['rx_rate'])
+            registration_record['rx_rate'] = BaseOutputProcessor.parse_bitrates(registration_record['rx_rate']) \
+                                                if ww2_installed else BaseOutputProcessor.parse_rates(registration_record['rx_rate'])
         if registration_record.get('uptime'):
             registration_record['uptime'] = naturaldelta(BaseOutputProcessor.parse_timedelta_seconds(registration_record['uptime']), months=True, minimum_unit='seconds')
 
@@ -87,6 +97,12 @@ class BaseOutputProcessor:
         rc = wifi_rates_rgx.search(rate)
         return f'{int(float(rc[1]))} {rc[2]}' if rc and len(rc.groups()) == 2 else rate
 
+    @staticmethod
+    def parse_bitrates(rate):
+        rate = int(rate)
+        power = floor(log(rate, 1000))
+        return f"{int(rate / 1000 ** power)} {['bps', 'Kbps', 'Mbps', 'Gbps'][int(power)]}"
+
     @staticmethod
     def parse_timedelta(time):
         duration_interval_rgx = config_handler.re_compiled.get('duration_interval_rgx')
@@ -126,6 +142,3 @@ class BaseOutputProcessor:
             table.header(outputEntry._fields)
             table.set_cols_align(['l']+ ['c']*(len(outputEntry._fields)-1))
         return table
-
-
-

mktxp/flow/router_entries_handler.py

@@ -22,19 +22,22 @@ class RouterEntriesHandler:
     def __init__(self):
         self.router_entries = []
         for router_name in config_handler.registered_entries():
-            entry = config_handler.config_entry(router_name)
+            router_entry = RouterEntriesHandler.router_entry(router_name, enabled_only = True)
-            if entry.enabled:
+            if router_entry:                
-                self.router_entries.append(RouterEntry(router_name))
+                self.router_entries.append(router_entry)
 
     @staticmethod
     def router_entry(entry_name, enabled_only = False):
         router_entry = None
+        
         for router_name in config_handler.registered_entries():
             if router_name == entry_name:
-                if enabled_only:
+                config_entry = config_handler.config_entry(router_name)
-                    entry = config_handler.config_entry(router_name)
+                if enabled_only and not config_entry.enabled:
-                    if not entry.enabled:
                         break
+                        
                 router_entry = RouterEntry(router_name)
+                router_entry.dhcp_entry = RouterEntriesHandler.router_entry(config_entry.remote_dhcp_entry)
                 break
+        
         return router_entry

mktxp/flow/router_entry.py

@@ -27,7 +27,10 @@ class RouterEntry:
             MKTXPConfigKeys.ROUTERBOARD_NAME: self.router_name,
             MKTXPConfigKeys.ROUTERBOARD_ADDRESS: self.config_entry.hostname
             }
+        
         self.wifi_package = None
+        self.dhcp_entry = None
+
         self.time_spent =  { 'IdentityCollector': 0,
                             'SystemResourceCollector': 0,
                             'HealthCollector': 0,
@@ -50,3 +53,17 @@ class RouterEntry:
                             'UserCollector': 0,                            
                             'MKTXPCollector': 0
                             }                
+
+    def is_connected(self):
+        connected = True
+        if not self.api_connection.is_connected():
+            connected = False                        
+            # let's get connected now
+            self.api_connection.connect()
+            if self.dhcp_entry:
+                self.dhcp_entry.api_connection.connect()
+
+        return connected
+
+
+