connections stats collector / cmd output, remote dhcp resolver, fixes / optimizations

mktxp/cli/config/config.py

@@ -24,6 +24,29 @@ from mktxp.utils.utils import FSHelper
 ''' MKTXP conf file handling
 '''
 
+class CollectorKeys:
+    IDENTITY_COLLECTOR = 'IdentityCollector'
+    SYSTEM_RESOURCE_COLLECTOR = 'SystemResourceCollector'
+    HEALTH_COLLECTOR = 'HealthCollector'
+    PUBLIC_IP_ADDRESS_COLLECTOR = 'PublicIPAddressCollector'
+    IPV6_NEIGHBOR_COLLECTOR = 'IPv6NeighborCollector'
+    PACKAGE_COLLECTOR = 'PackageCollector'
+    DHCP_COLLECTOR = 'DHCPCollector'
+    POOL_COLLECTOR = 'PoolCollector'
+    IP_CONNECTION_COLLECTOR = 'IPConnectionCollector'
+    INTERFACE_COLLECTOR = 'InterfaceCollector'
+    FIREWALL_COLLECTOR = 'FirewallCollector'
+    MONITOR_COLLECTOR = 'MonitorCollector'
+    POE_COLLECTOR = 'POECollector'
+    NETWATCH_COLLECTOR = 'NetwatchCollector'
+    ROUTE_COLLECTOR = 'RouteCollector'
+    WLAN_COLLECTOR = 'WLANCollector'
+    CAPSMAN_COLLECTOR = 'CapsmanCollector'
+    QUEUE_TREE_COLLECTOR = 'QueueTreeCollector'
+    QUEUE_SIMPLE_COLLECTOR = 'QueueSimpleCollector'
+    USER_COLLECTOR = 'UserCollector'
+    MKTXP_COLLECTOR = 'MKTXPCollector'
+
 
 class MKTXPConfigKeys:
     ''' MKTXP config file keys
@@ -44,6 +67,7 @@ class MKTXPConfigKeys:
     FE_DHCP_LEASE_KEY = 'dhcp_lease'
     FE_DHCP_POOL_KEY = 'pool'
     FE_IP_CONNECTIONS_KEY = 'connections'
+    FE_CONNECTION_STATS_KEY = 'connection_stats'
     FE_INTERFACE_KEY = 'interface'
     FE_FIREWALL_KEY = 'firewall'
 
@@ -104,7 +128,7 @@ class MKTXPConfigKeys:
 
 
     BOOLEAN_KEYS_NO = {ENABLED_KEY, SSL_KEY, NO_SSL_CERTIFICATE,
-                       SSL_CERTIFICATE_VERIFY, FE_IPV6_FIREWALL_KEY, FE_IPV6_NEIGHBOR_KEY}
+                       SSL_CERTIFICATE_VERIFY, FE_IPV6_FIREWALL_KEY, FE_IPV6_NEIGHBOR_KEY, FE_CONNECTION_STATS_KEY}
 
     # Feature keys enabled by default
     BOOLEAN_KEYS_YES = {FE_DHCP_KEY, FE_PACKAGE_KEY, FE_DHCP_LEASE_KEY, FE_DHCP_POOL_KEY, FE_IP_CONNECTIONS_KEY, FE_INTERFACE_KEY, FE_FIREWALL_KEY,
@@ -116,6 +140,7 @@ class MKTXPConfigKeys:
     SYSTEM_BOOLEAN_KEYS_NO = {MKTXP_VERBOSE_MODE, MKTXP_FETCH_IN_PARALLEL}
 
     STR_KEYS = (HOST_KEY, USER_KEY, PASSWD_KEY, FE_REMOTE_DHCP_ENTRY)
+    INT_KEYS =  ()
     MKTXP_INT_KEYS = (PORT_KEY, MKTXP_SOCKET_TIMEOUT, MKTXP_INITIAL_DELAY, MKTXP_MAX_DELAY,
                       MKTXP_INC_DIV, MKTXP_BANDWIDTH_TEST_INTERVAL, MKTXP_MIN_COLLECT_INTERVAL,
                       MKTXP_MAX_WORKER_THREADS, MKTXP_MAX_SCRAPE_DURATION, MKTXP_TOTAL_MAX_SCRAPE_DURATION)
@@ -130,7 +155,7 @@ class ConfigEntry:
                                                        MKTXPConfigKeys.SSL_KEY, MKTXPConfigKeys.NO_SSL_CERTIFICATE, MKTXPConfigKeys.SSL_CERTIFICATE_VERIFY,
                                                        MKTXPConfigKeys.FE_DHCP_KEY, MKTXPConfigKeys.FE_PACKAGE_KEY, MKTXPConfigKeys.FE_DHCP_LEASE_KEY, MKTXPConfigKeys.FE_DHCP_POOL_KEY, MKTXPConfigKeys.FE_INTERFACE_KEY,
                                                        MKTXPConfigKeys.FE_FIREWALL_KEY, MKTXPConfigKeys.FE_MONITOR_KEY, MKTXPConfigKeys.FE_ROUTE_KEY, MKTXPConfigKeys.FE_WIRELESS_KEY, MKTXPConfigKeys.FE_WIRELESS_CLIENTS_KEY,
-                                                       MKTXPConfigKeys.FE_IP_CONNECTIONS_KEY, MKTXPConfigKeys.FE_CAPSMAN_KEY, MKTXPConfigKeys.FE_CAPSMAN_CLIENTS_KEY, MKTXPConfigKeys.FE_POE_KEY, MKTXPConfigKeys.FE_NETWATCH_KEY,
+                                                       MKTXPConfigKeys.FE_IP_CONNECTIONS_KEY, MKTXPConfigKeys.FE_CONNECTION_STATS_KEY, MKTXPConfigKeys.FE_CAPSMAN_KEY, MKTXPConfigKeys.FE_CAPSMAN_CLIENTS_KEY, MKTXPConfigKeys.FE_POE_KEY, MKTXPConfigKeys.FE_NETWATCH_KEY,
                                                        MKTXPConfigKeys.MKTXP_USE_COMMENTS_OVER_NAMES, MKTXPConfigKeys.FE_PUBLIC_IP_KEY, MKTXPConfigKeys.FE_IPV6_FIREWALL_KEY, MKTXPConfigKeys.FE_IPV6_NEIGHBOR_KEY,
                                                        MKTXPConfigKeys.FE_USER_KEY, MKTXPConfigKeys.FE_QUEUE_KEY, MKTXPConfigKeys.FE_REMOTE_DHCP_ENTRY
                                                        ])
@@ -235,17 +260,18 @@ class MKTXPConfigHandler:
     def registered_entries(self):
         ''' All MKTXP registered entries
         '''
-        registered_entries = [entry_name for entry_name in self.config.keys()]
+        return (entry_name for entry_name in self.config.keys())
-        if not registered_entries:
-            registered_entries = [MKTXPConfigKeys.NO_ENTRIES_REGISTERED]
 
-        return registered_entries
+    def registered_entry(self, entry_name):
+        ''' A specific MKTXP registered entry by name
+        '''
+        return self.config.get(entry_name)
 
     def config_entry(self, entry_name):
         ''' Given an entry name, reads and returns the entry info
         '''
         entry_reader = self._config_entry_reader(entry_name)
-        return ConfigEntry.MKTXPConfigEntry(**entry_reader)
+        return ConfigEntry.MKTXPConfigEntry(**entry_reader) if entry_reader else None
 
     def system_entry(self):
         ''' MKTXP internal config entry
@@ -258,7 +284,10 @@ class MKTXPConfigHandler:
         ''' (Force-)Read conf data from disk
         '''
         self.config = ConfigObj(self.usr_conf_data_path)
+        self.config.preserve_comments = True
+
         self._config = ConfigObj(self.mktxp_conf_path)
+        self._config.preserve_comments = True
 
     def _create_os_path(self, os_path, resource_path):
         if not os.path.exists(os_path):
@@ -272,7 +301,7 @@ class MKTXPConfigHandler:
         new_keys = []
 
         for key in MKTXPConfigKeys.BOOLEAN_KEYS_NO.union(MKTXPConfigKeys.BOOLEAN_KEYS_YES):
-            if self.config[entry_name].get(key):
+            if self.config[entry_name].get(key) is not None:
                 config_entry_reader[key] = self.config[entry_name].as_bool(key)
             else:
                 config_entry_reader[key] = True if key in MKTXPConfigKeys.BOOLEAN_KEYS_YES else False
@@ -288,6 +317,13 @@ class MKTXPConfigHandler:
             if key is MKTXPConfigKeys.PASSWD_KEY and type(config_entry_reader[key]) is list:
                 config_entry_reader[key] = ','.join(config_entry_reader[key])
 
+        for key in MKTXPConfigKeys.INT_KEYS:
+            if self.config[entry_name].get(key):
+                config_entry_reader[key] = self.config[entry_name].as_int(key)
+            else:
+                config_entry_reader[key] = self._default_value_for_key(key)
+                new_keys.append(key) # read from disk next time
+
         # port
         if self.config[entry_name].get(MKTXPConfigKeys.PORT_KEY):
             config_entry_reader[MKTXPConfigKeys.PORT_KEY] = self.config[entry_name].as_int(
@@ -322,9 +358,8 @@ class MKTXPConfigHandler:
                 new_keys.append(key) # read from disk next time
 
         for key in MKTXPConfigKeys.SYSTEM_BOOLEAN_KEYS_NO.union(MKTXPConfigKeys.SYSTEM_BOOLEAN_KEYS_YES):
-            if self._config[entry_name].get(key):
+            if self._config[entry_name].get(key) is not None:
-                system_entry_reader[key] = self._config[entry_name].as_bool(
+                system_entry_reader[key] = self._config[entry_name].as_bool(key)
-                    key)
             else:
                 system_entry_reader[key] = True if key in MKTXPConfigKeys.SYSTEM_BOOLEAN_KEYS_YES else False
                 new_keys.append(key) # read from disk next time
@@ -344,14 +379,14 @@ class MKTXPConfigHandler:
     def _default_value_for_key(self, key, value=None):
         return {
             MKTXPConfigKeys.SSL_KEY: lambda value: MKTXPConfigKeys.DEFAULT_API_SSL_PORT if value else MKTXPConfigKeys.DEFAULT_API_PORT,
-            MKTXPConfigKeys.PORT_KEY: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_PORT,
+            MKTXPConfigKeys.PORT_KEY: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_PORT,
-            MKTXPConfigKeys.FE_REMOTE_DHCP_ENTRY:  lambda value: MKTXPConfigKeys.DEFAULT_FE_REMOTE_DHCP_ENTRY,
+            MKTXPConfigKeys.FE_REMOTE_DHCP_ENTRY:  lambda _: MKTXPConfigKeys.DEFAULT_FE_REMOTE_DHCP_ENTRY,
-            MKTXPConfigKeys.MKTXP_SOCKET_TIMEOUT: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_SOCKET_TIMEOUT,
+            MKTXPConfigKeys.MKTXP_SOCKET_TIMEOUT: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_SOCKET_TIMEOUT,
-            MKTXPConfigKeys.MKTXP_INITIAL_DELAY: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_INITIAL_DELAY,
+            MKTXPConfigKeys.MKTXP_INITIAL_DELAY: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_INITIAL_DELAY,
-            MKTXPConfigKeys.MKTXP_MAX_DELAY: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_MAX_DELAY,
+            MKTXPConfigKeys.MKTXP_MAX_DELAY: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_MAX_DELAY,
-            MKTXPConfigKeys.MKTXP_INC_DIV: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_INC_DIV,
+            MKTXPConfigKeys.MKTXP_INC_DIV: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_INC_DIV,
-            MKTXPConfigKeys.MKTXP_BANDWIDTH_TEST_INTERVAL: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_BANDWIDTH_TEST_INTERVAL,
+            MKTXPConfigKeys.MKTXP_BANDWIDTH_TEST_INTERVAL: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_BANDWIDTH_TEST_INTERVAL,
-            MKTXPConfigKeys.MKTXP_MIN_COLLECT_INTERVAL: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_MIN_COLLECT_INTERVAL,
+            MKTXPConfigKeys.MKTXP_MIN_COLLECT_INTERVAL: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_MIN_COLLECT_INTERVAL,
             MKTXPConfigKeys.MKTXP_FETCH_IN_PARALLEL: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_FETCH_IN_PARALLEL,
             MKTXPConfigKeys.MKTXP_MAX_WORKER_THREADS: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_MAX_WORKER_THREADS,
             MKTXPConfigKeys.MKTXP_MAX_SCRAPE_DURATION: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_MAX_SCRAPE_DURATION,

mktxp/cli/config/mktxp.conf

@@ -27,7 +27,10 @@
     installed_packages = True       # Installed packages
     dhcp = True                     # DHCP general metrics
     dhcp_lease = True               # DHCP lease metrics
+
     connections = True              # IP connections metrics
+    connection_stats = False        # Open IP connections metrics 
+
     pool = True                     # Pool metrics
     interface = True                # Interfaces traffic metrics
     

mktxp/cli/dispatch.py

@@ -98,6 +98,9 @@ class MKTXPDispatcher:
         elif args['dhcp_clients']:
             OutputProcessor.dhcp_clients(args['entry_name'])
 
+        elif args['conn_stats']:
+            OutputProcessor.conn_stats(args['entry_name'])
+
         else:
             print("Select metric option(s) to print out, or run 'mktxp print -h' to find out more")
 

mktxp/cli/options.py

@@ -146,6 +146,11 @@ Selected metrics info can be printed on the command line. For more information,
                 help = "DHCP clients metrics",
                 action = 'store_true')
 
+        optional_args_group.add_argument('-cn', '--conn_stats', dest='conn_stats',
+                help = "IP connections stats",
+                action = 'store_true')
+
+
     # Options checking
     def _check_args(self, args, parser):
         ''' Validation of supplied CLI arguments

mktxp/cli/output/capsman_out.py

@@ -13,8 +13,6 @@
 
 
 from mktxp.flow.processor.output import BaseOutputProcessor
-from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource
-from mktxp.datasource.wireless_ds import WirelessMetricsDataSource
 from mktxp.datasource.capsman_ds import CapsmanRegistrationsMetricsDataSource
 
 class CapsmanOutput:
@@ -29,13 +27,9 @@ class CapsmanOutput:
             return 
 
         # translate / trim / augment registration records
-        dhcp_lease_labels = ['host_name', 'comment', 'address', 'mac_address']
-        dhcp_entry = WirelessMetricsDataSource.dhcp_entry(router_entry)
-        dhcp_lease_records = DHCPMetricsDataSource.metric_records(dhcp_entry, metric_labels = dhcp_lease_labels, add_router_id = False)   
-
         dhcp_rt_by_interface = {}
         for registration_record in sorted(registration_records, key = lambda rt_record: rt_record['rx_signal'], reverse=True):
-            BaseOutputProcessor.augment_record(router_entry, registration_record, dhcp_lease_records)
+            BaseOutputProcessor.augment_record(router_entry, registration_record)
 
             interface = registration_record['interface']
             if interface in dhcp_rt_by_interface.keys():

mktxp/cli/output/conn_stats_out.py

@@ -0,0 +1,49 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from mktxp.flow.processor.output import BaseOutputProcessor
+from mktxp.datasource.connection_ds import IPConnectionStatsDatasource
+
+
+class ConnectionsStatsOutput:
+    ''' Connections Stats Output
+    '''    
+    @staticmethod
+    def clients_summary(router_entry):
+        connection_records = IPConnectionStatsDatasource.metric_records(router_entry, add_router_id = False)
+        if not connection_records:
+            print('No connection stats records')
+            return 
+
+        conn_cnt = 0
+        output_records = []
+        for registration_record in sorted(connection_records, key = lambda rt_record: rt_record['connection_count'], reverse=True):
+            BaseOutputProcessor.resolve_dhcp(router_entry, registration_record, id_key = 'src_address', resolve_address = False)        
+            output_records.append(registration_record)
+            conn_cnt += registration_record['connection_count']
+
+        output_records_cnt = 0
+        output_entry = BaseOutputProcessor.OutputConnStatsEntry
+        output_table = BaseOutputProcessor.output_table(output_entry)
+        
+        for record in output_records:
+            output_table.add_row(output_entry(**record))
+            output_table.add_row(output_entry())
+            output_records_cnt += 1
+                
+        print (output_table.draw())
+
+        print(f'Distinct source addresses: {output_records_cnt}')
+        print(f'Total open connections: {conn_cnt}', '\n')
+

mktxp/cli/output/dhcp_out.py

@@ -22,13 +22,13 @@ class DHCPOutput:
     @staticmethod
     def clients_summary(router_entry):
         dhcp_lease_labels = ['host_name', 'comment', 'active_address', 'address', 'mac_address', 'server', 'expires_after']
-        dhcp_lease_records = DHCPMetricsDataSource.metric_records(router_entry, metric_labels = dhcp_lease_labels, add_router_id = False)
+        dhcp_lease_records = DHCPMetricsDataSource.metric_records(router_entry, metric_labels = dhcp_lease_labels, add_router_id = False, translate = False, dhcp_cache = False)
         if not dhcp_lease_records:
             print('No DHCP registration records')
             return 
 
         dhcp_by_server = {}
-        for dhcp_lease_record in sorted(dhcp_lease_records, key = lambda dhcp_record: dhcp_record['active_address'], reverse=True):
+        for dhcp_lease_record in sorted(dhcp_lease_records, key = lambda dhcp_record: dhcp_record['address'], reverse=True):
             server = dhcp_lease_record.get('server', 'all')
             if server == 'all':
                 dhcp_lease_record['server'] = server

mktxp/cli/output/wifi_out.py

@@ -13,9 +13,7 @@
 
 
 from mktxp.flow.processor.output import BaseOutputProcessor
-from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource
 from mktxp.datasource.wireless_ds import WirelessMetricsDataSource
-from mktxp.flow.router_entries_handler import RouterEntriesHandler
 
 
 class WirelessOutput:
@@ -30,15 +28,11 @@ class WirelessOutput:
             return 
 
         # translate / trim / augment registration records
-        dhcp_lease_labels = ['host_name', 'comment', 'address', 'mac_address']
-        dhcp_entry = WirelessMetricsDataSource.dhcp_entry(router_entry)
-        dhcp_lease_records = DHCPMetricsDataSource.metric_records(dhcp_entry, metric_labels = dhcp_lease_labels, add_router_id = False)   
-
         dhcp_rt_by_interface = {}
 
         key = lambda rt_record: rt_record['signal_strength'] if rt_record.get('signal_strength') else rt_record['interface']
         for registration_record in sorted(registration_records, key = key, reverse=True):
-            BaseOutputProcessor.augment_record(router_entry, registration_record, dhcp_lease_records)
+            BaseOutputProcessor.augment_record(router_entry, registration_record)
 
             interface = registration_record['interface']
             if interface in dhcp_rt_by_interface.keys():

mktxp/collector/capsman_collector.py

@@ -15,7 +15,6 @@
 from mktxp.cli.config.config import MKTXPConfigKeys
 from mktxp.flow.processor.output import BaseOutputProcessor
 from mktxp.collector.base_collector import BaseCollector
-from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource
 from mktxp.datasource.capsman_ds import CapsmanCapsMetricsDataSource, CapsmanRegistrationsMetricsDataSource, CapsmanInterfacesDatasource
 from mktxp.datasource.wireless_ds import WirelessMetricsDataSource
 
@@ -53,11 +52,8 @@ class CapsmanCollector(BaseCollector):
             if router_entry.config_entry.capsman_clients:
 
                 # translate / trim / augment registration records
-                dhcp_lease_labels = ['mac_address', 'address', 'host_name', 'comment']
-                dhcp_entry = WirelessMetricsDataSource.dhcp_entry(router_entry)
-                dhcp_lease_records = DHCPMetricsDataSource.metric_records(dhcp_entry, metric_labels = dhcp_lease_labels)
                 for registration_record in registration_records:
-                    BaseOutputProcessor.augment_record(router_entry, registration_record, dhcp_lease_records)
+                    BaseOutputProcessor.augment_record(router_entry, registration_record)
                     
                 tx_byte_metrics = BaseCollector.counter_collector('capsman_clients_tx_bytes', 'Number of sent packet bytes', registration_records, 'tx_bytes', ['dhcp_name'])
                 yield tx_byte_metrics

mktxp/collector/connection_collector.py

@@ -13,7 +13,8 @@
 
 
 from mktxp.collector.base_collector import BaseCollector
-from mktxp.datasource.connection_ds import IPConnectionDatasource
+from mktxp.flow.processor.output import BaseOutputProcessor
+from mktxp.datasource.connection_ds import IPConnectionDatasource, IPConnectionStatsDatasource
 
 
 class IPConnectionCollector(BaseCollector):
@@ -21,11 +22,20 @@ class IPConnectionCollector(BaseCollector):
     '''    
     @staticmethod
     def collect(router_entry):
-        if not router_entry.config_entry.connections:
+        if router_entry.config_entry.connections:       
-           return
-           
             connection_records = IPConnectionDatasource.metric_records(router_entry)
             if connection_records:
-
                 connection_metrics = BaseCollector.gauge_collector('ip_connections_total', 'Number of IP connections', connection_records, 'count',)
                 yield connection_metrics
+
+        if router_entry.config_entry.connection_stats:
+            connection_stats_records = IPConnectionStatsDatasource.metric_records(router_entry)            
+
+            for connection_stat_record in connection_stats_records:
+                BaseOutputProcessor.augment_record(router_entry, connection_stat_record, id_key = 'src_address')
+
+            connection_stats_labels = ['src_address', 'dst_addresses', 'dhcp_name']                
+            connection_stats_metrics_gauge = BaseCollector.gauge_collector('connection_stats', 'Open connection stats', 
+                                                                connection_stats_records, 'connection_count', connection_stats_labels)
+            yield connection_stats_metrics_gauge
+

mktxp/collector/wlan_collector.py

@@ -14,7 +14,6 @@
 
 from mktxp.flow.processor.output import BaseOutputProcessor
 from mktxp.collector.base_collector import BaseCollector
-from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource
 from mktxp.datasource.wireless_ds import WirelessMetricsDataSource
 from mktxp.datasource.interface_ds import InterfaceMonitorMetricsDataSource
 
@@ -52,12 +51,8 @@ class WLANCollector(BaseCollector):
             registration_labels = ['interface', 'ssid', 'mac_address', 'tx_rate', 'rx_rate', 'uptime', 'bytes', 'signal_to_noise', 'tx_ccq', 'signal_strength', 'signal']
             registration_records = WirelessMetricsDataSource.metric_records(router_entry, metric_labels = registration_labels)
             if registration_records:
-                dhcp_lease_labels = ['mac_address', 'address', 'host_name', 'comment']
-                dhcp_entry = WirelessMetricsDataSource.dhcp_entry(router_entry)            
-                dhcp_lease_records = DHCPMetricsDataSource.metric_records(dhcp_entry, metric_labels = dhcp_lease_labels)
-      
                 for registration_record in registration_records:
-                    BaseOutputProcessor.augment_record(router_entry, registration_record, dhcp_lease_records)
+                    BaseOutputProcessor.augment_record(router_entry, registration_record)
 
                 tx_byte_metrics = BaseCollector.counter_collector('wlan_clients_tx_bytes', 'Number of sent packet bytes', registration_records, 'tx_bytes', ['dhcp_name'])
                 yield tx_byte_metrics

mktxp/datasource/connection_ds.py

@@ -12,6 +12,7 @@
 ## GNU General Public License for more details.
 
 
+from collections import namedtuple
 from mktxp.datasource.base_ds import BaseDSProcessor
 
 
@@ -33,3 +34,44 @@ class IPConnectionDatasource:
         except Exception as exc:
             print(f'Error getting IP connection info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}')
             return None
+
+
+class IPConnectionStatsDatasource:
+    ''' IP connections stats data provider
+    '''             
+    @staticmethod
+    def metric_records(router_entry, *, metric_labels = None, add_router_id = True):
+        if metric_labels is None:
+            metric_labels = []        
+        try:
+            connection_records = router_entry.api_connection.router_api().get_resource('/ip/firewall/connection/').call('print', \
+                                                                                            {'proplist':'src-address,dst-address,protocol'})
+             # calculate number of connections per src-address
+            connections_per_src_address = {}
+            for connection_record in connection_records:
+                #address, port = (connection_record['src-address'].split(':') + [None])[:2]
+                address = connection_record['src-address'].split(':')[0]
+                destination = f"{connection_record.get('dst-address')}({connection_record.get('protocol')})"
+
+                count, destinations = 0, set()
+                if connections_per_src_address.get(address):
+                    count, destinations = connections_per_src_address[address]
+                count += 1
+                destinations.add(destination)
+                connections_per_src_address[address] = ConnStatsEntry(count, destinations)
+
+            # compile connections-per-interface records
+            records = []
+            for key, entry in connections_per_src_address.items():
+                record = {'src_address': key, 'connection_count': entry.count, 'dst_addresses': ', '.join(entry.destinations)}
+                if add_router_id: 
+                    for router_key, router_value in router_entry.router_id.items():
+                        record[router_key] = router_value
+                records.append(record)
+            return records 
+        except Exception as exc:
+            print(f'Error getting IP connection stats info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}')
+            return None
+
+
+ConnStatsEntry = namedtuple('ConnStatsEntry', ['count', 'destinations'])

mktxp/datasource/dhcp_ds.py

@@ -20,12 +20,17 @@ class DHCPMetricsDataSource:
     ''' DHCP Metrics data provider
     '''             
     @staticmethod
-    def metric_records(router_entry, *, metric_labels = None, add_router_id = True):
+    def metric_records(router_entry, *, metric_labels = None, add_router_id = True, dhcp_cache = True, translate = True, bound = False):
-        if metric_labels is None:
+        if metric_labels is None or dhcp_cache:
-            metric_labels = []                
+            metric_labels = ['host_name', 'comment', 'active_address', 'address', 'mac_address', 'server', 'expires_after']  
+
+        if dhcp_cache and router_entry.dhcp_records:
+            return router_entry.dhcp_records
         try:
-            #dhcp_lease_records = router_entry.api_connection.router_api().get_resource('/ip/dhcp-server/lease').get(status='bound')
+            if bound:
-            dhcp_lease_records = router_entry.api_connection.router_api().get_resource('/ip/dhcp-server/lease').call('print', {'active':''})
+                dhcp_lease_records = router_entry.dhcp_entry.api_connection.router_api().get_resource('/ip/dhcp-server/lease').get(status='bound')
+            else:
+                dhcp_lease_records = router_entry.dhcp_entry.api_connection.router_api().get_resource('/ip/dhcp-server/lease').call('print', {'active':''})
 
             # translation rules
             translation_table = {}
@@ -33,12 +38,16 @@ class DHCPMetricsDataSource:
                 translation_table['comment'] = lambda c: c if c else ''           
             if 'host_name' in metric_labels:
                 translation_table['host_name'] = lambda c: c if c else ''           
-            if 'expires_after' in metric_labels:
+            if 'expires_after' in metric_labels and translate:
                 translation_table['expires_after'] = lambda c: parse_mkt_uptime(c) if c else 0       
             if 'active_address' in metric_labels:
                 translation_table['active_address'] = lambda c: c if c else ''        
 
-            return BaseDSProcessor.trimmed_records(router_entry, router_records = dhcp_lease_records, metric_labels = metric_labels, add_router_id = add_router_id, translation_table = translation_table)
+            records = BaseDSProcessor.trimmed_records(router_entry, router_records = dhcp_lease_records, metric_labels = metric_labels, add_router_id = add_router_id, translation_table = translation_table)
+            if dhcp_cache:
+                router_entry.dhcp_records = records
+            return records
+            
         except Exception as exc:
             print(f'Error getting dhcp info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}')
             return None

mktxp/datasource/wireless_ds.py

@@ -52,11 +52,3 @@ class WirelessMetricsDataSource:
     @staticmethod
     def wifiwave2_installed(router_entry):
         return WirelessMetricsDataSource.wireless_package(router_entry) == WirelessMetricsDataSource.WIFIWAVE2
-
-    @staticmethod
-    def dhcp_entry(router_entry):
-        if router_entry.dhcp_entry:
-            return router_entry.dhcp_entry
-        return router_entry
-
-

mktxp/flow/collector_handler.py

@@ -1,15 +1,15 @@
 # coding=utf8
-# Copyright (c) 2020 Arseniy Kuznenowov
+## Copyright (c) 2020 Arseniy Kuznetsov
 ##
-# This program is free software; you can redistribute it and/or
+## This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
+## modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
+## as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
+## of the License, or (at your option) any later version.
 ##
-# This program is distributed in the hope that it will be useful,
+## This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+## GNU General Public License for more details.
 
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from timeit import default_timer
@@ -43,7 +43,7 @@ class CollectorHandler:
                 start = default_timer()
                 yield from collect_func(router_entry)
                 router_entry.time_spent[collector_ID] += default_timer() - start
-
+                router_entry.is_done()
 
     def collect_router_entry_async(self, router_entry, scrape_timeout_event, total_scrape_timeout_event):
         results = []
@@ -60,6 +60,7 @@ class CollectorHandler:
             result = list(collect_func(router_entry))
             results += result
             router_entry.time_spent[collector_ID] += default_timer() - start
+            router_entry.is_done()
 
         return results
 

mktxp/flow/collector_registry.py

@@ -13,6 +13,7 @@
 
 
 from collections import OrderedDict
+from mktxp.cli.config.config import CollectorKeys
 from mktxp.collector.dhcp_collector import DHCPCollector
 from mktxp.collector.package_collector import PackageCollector
 from mktxp.collector.connection_collector import IPConnectionCollector
@@ -36,7 +37,6 @@ from mktxp.collector.user_collector import UserCollector
 from mktxp.collector.queue_collector import QueueTreeCollector
 from mktxp.collector.queue_collector import QueueSimpleCollector
 
-
 class CollectorRegistry:
     ''' MKTXP Collectors Registry
     '''
@@ -46,33 +46,33 @@ class CollectorRegistry:
         # bandwidth collector is not router-entry related, so registering directly
         self.bandwidthCollector = BandwidthCollector()
 
-        self.register('IdentityCollector', IdentityCollector.collect)
+        self.register(CollectorKeys.IDENTITY_COLLECTOR, IdentityCollector.collect)
-        self.register('SystemResourceCollector', SystemResourceCollector.collect)
+        self.register(CollectorKeys.SYSTEM_RESOURCE_COLLECTOR, SystemResourceCollector.collect)
-        self.register('HealthCollector', HealthCollector.collect)
+        self.register(CollectorKeys.HEALTH_COLLECTOR, HealthCollector.collect)
-        self.register('PublicIPAddressCollector', PublicIPAddressCollector.collect)
+        self.register(CollectorKeys.PUBLIC_IP_ADDRESS_COLLECTOR, PublicIPAddressCollector.collect)
 
-        self.register('IPv6NeighborCollector', IPv6NeighborCollector.collect)
+        self.register(CollectorKeys.IPV6_NEIGHBOR_COLLECTOR, IPv6NeighborCollector.collect)
 
-        self.register('PackageCollector', PackageCollector.collect)
+        self.register(CollectorKeys.PACKAGE_COLLECTOR, PackageCollector.collect)
-        self.register('DHCPCollector', DHCPCollector.collect)
+        self.register(CollectorKeys.DHCP_COLLECTOR, DHCPCollector.collect)
-        self.register('IPConnectionCollector', IPConnectionCollector.collect)
+        self.register(CollectorKeys.IP_CONNECTION_COLLECTOR, IPConnectionCollector.collect)
-        self.register('PoolCollector', PoolCollector.collect)
+        self.register(CollectorKeys.POOL_COLLECTOR, PoolCollector.collect)
-        self.register('InterfaceCollector', InterfaceCollector.collect)
+        self.register(CollectorKeys.INTERFACE_COLLECTOR, InterfaceCollector.collect)
 
-        self.register('FirewallCollector', FirewallCollector.collect)
+        self.register(CollectorKeys.FIREWALL_COLLECTOR, FirewallCollector.collect)
-        self.register('MonitorCollector', MonitorCollector.collect)
+        self.register(CollectorKeys.MONITOR_COLLECTOR, MonitorCollector.collect)
-        self.register('POECollector', POECollector.collect)
+        self.register(CollectorKeys.POE_COLLECTOR, POECollector.collect)
-        self.register('NetwatchCollector', NetwatchCollector.collect)
+        self.register(CollectorKeys.NETWATCH_COLLECTOR, NetwatchCollector.collect)
-        self.register('RouteCollector', RouteCollector.collect)
+        self.register(CollectorKeys.ROUTE_COLLECTOR, RouteCollector.collect)
 
-        self.register('WLANCollector', WLANCollector.collect)
+        self.register(CollectorKeys.WLAN_COLLECTOR, WLANCollector.collect)
-        self.register('CapsmanCollector', CapsmanCollector.collect)
+        self.register(CollectorKeys.CAPSMAN_COLLECTOR, CapsmanCollector.collect)
 
-        self.register('UserCollector', UserCollector.collect)
+        self.register(CollectorKeys.USER_COLLECTOR, UserCollector.collect)
-        self.register('QueueTreeCollector', QueueTreeCollector.collect)
+        self.register(CollectorKeys.QUEUE_TREE_COLLECTOR, QueueTreeCollector.collect)
-        self.register('QueueSimpleCollector', QueueSimpleCollector.collect)
+        self.register(CollectorKeys.QUEUE_SIMPLE_COLLECTOR, QueueSimpleCollector.collect)
 
-        self.register('MKTXPCollector', MKTXPCollector.collect)
+        self.register(CollectorKeys.MKTXP_COLLECTOR, MKTXPCollector.collect)
 
     def register(self, collector_ID, collect_func):
         self.registered_collectors[collector_ID] = collect_func

mktxp/flow/processor/base_proc.py

@@ -25,6 +25,7 @@ from mktxp.flow.router_entries_handler import RouterEntriesHandler
 from mktxp.cli.output.capsman_out import CapsmanOutput
 from mktxp.cli.output.wifi_out import WirelessOutput
 from mktxp.cli.output.dhcp_out import DHCPOutput
+from mktxp.cli.output.conn_stats_out import ConnectionsStatsOutput
 
 
 class ExportProcessor:
@@ -64,3 +65,11 @@ class OutputProcessor:
         router_entry = RouterEntriesHandler.router_entry(entry_name)
         if router_entry:
             DHCPOutput.clients_summary(router_entry)
+
+    @staticmethod
+    def conn_stats(entry_name):
+        router_entry = RouterEntriesHandler.router_entry(entry_name)
+        if router_entry:
+            ConnectionsStatsOutput.clients_summary(router_entry)
+
+            

mktxp/flow/processor/output.py

@@ -19,6 +19,7 @@ from texttable import Texttable
 from humanize import naturaldelta
 from mktxp.cli.config.config import config_handler
 from mktxp.datasource.wireless_ds import WirelessMetricsDataSource
+from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource
 from math import floor, log
 
 
@@ -35,20 +36,13 @@ class BaseOutputProcessor:
     OutputDHCPEntry = namedtuple('OutputDHCPEntry', ['host_name', 'server', 'mac_address', 'address', 'active_address', 'expires_after'])
     OutputDHCPEntry.__new__.__defaults__ = ('',) * len(OutputDHCPEntry._fields)
 
-    @staticmethod
+    OutputConnStatsEntry = namedtuple('OutputConnStatsEntry', ['dhcp_name', 'src_address', 'connection_count', 'dst_addresses'])
-    def augment_record(router_entry, registration_record, dhcp_lease_records):
+    OutputConnStatsEntry.__new__.__defaults__ = ('',) * len(OutputConnStatsEntry._fields)
-        dhcp_name = registration_record.get('mac_address')
-        dhcp_address = 'No DHCP Record'              
-        if dhcp_lease_records:
-            try:
-                dhcp_lease_record = next((dhcp_lease_record for dhcp_lease_record in dhcp_lease_records if dhcp_lease_record.get('mac_address')==registration_record.get('mac_address')))
-                dhcp_name = BaseOutputProcessor.dhcp_name(router_entry, dhcp_lease_record)
-                dhcp_address = dhcp_lease_record.get('address', '')
-            except StopIteration:
-                pass
 
-        registration_record['dhcp_name'] = dhcp_name
+
-        registration_record['dhcp_address'] = dhcp_address
+    @staticmethod
+    def augment_record(router_entry, registration_record, id_key = 'mac_address'):
+        BaseOutputProcessor.resolve_dhcp(router_entry, registration_record, id_key)
 
         # split out tx/rx bytes
         if registration_record.get('bytes'):
@@ -88,6 +82,22 @@ class BaseOutputProcessor:
 
         return dhcp_name
 
+    @staticmethod
+    def resolve_dhcp(router_entry, registration_record, id_key = 'mac_address', resolve_address = True):
+        if not router_entry.dhcp_records:
+            DHCPMetricsDataSource.metric_records(router_entry)
+        dhcp_name = registration_record.get(id_key)
+        dhcp_address = 'No DHCP Record'              
+
+        dhcp_lease_record = router_entry.dhcp_record(dhcp_name)
+        if dhcp_lease_record:
+            dhcp_name = BaseOutputProcessor.dhcp_name(router_entry, dhcp_lease_record)
+            dhcp_address = dhcp_lease_record.get('address', '')
+
+        registration_record['dhcp_name'] = dhcp_name
+        if resolve_address:
+            registration_record['dhcp_address'] = dhcp_address
+
     @staticmethod
     def parse_rates(rate):
         wifi_rates_rgx = config_handler.re_compiled.get('wifi_rates_rgx')

mktxp/flow/router_entries_handler.py

@@ -20,24 +20,34 @@ class RouterEntriesHandler:
     ''' Handles RouterOS entries defined in MKTXP config 
     '''         
     def __init__(self):
-        self.router_entries = []
+        self._router_entries = {}            
         for router_name in config_handler.registered_entries():            
-            router_entry = RouterEntriesHandler.router_entry(router_name, enabled_only = True)
+            router_entry = RouterEntry(router_name)
-            if router_entry:                
+            if router_entry.config_entry.remote_dhcp_entry and config_handler.registered_entry(router_entry.config_entry.remote_dhcp_entry):
-                self.router_entries.append(router_entry)
+                router_entry.dhcp_entry = RouterEntry(router_entry.config_entry.remote_dhcp_entry)
+            self._router_entries[router_name] = router_entry
+
+    @property
+    def router_entries(self):
+        return (entry for key, entry in  self._router_entries.items() if entry.config_entry.enabled) \
+                                                                                if self._router_entries else None   
+
+    def router_entry(self, entry_name, enabled_only = False):
+        entry = self._router_entries.get(entry_name)                                                                                
+        if entry and (entry.config_entry.enabled or not enabled_only):
+            return entry
+        return None
 
     @staticmethod
     def router_entry(entry_name, enabled_only = False):
-        router_entry = None
+        ''' A static router entry initialiser
-        
+        '''
-        for router_name in config_handler.registered_entries():
+        config_entry = config_handler.config_entry(entry_name)
-            if router_name == entry_name:
-                config_entry = config_handler.config_entry(router_name)
         if enabled_only and not config_entry.enabled:
-                        break
+            return None
 
-                router_entry = RouterEntry(router_name)
+        router_entry = RouterEntry(entry_name)
-                router_entry.dhcp_entry = RouterEntriesHandler.router_entry(config_entry.remote_dhcp_entry)
+        if config_entry.remote_dhcp_entry and config_handler.registered_entry(config_entry.remote_dhcp_entry):
-                break
+            router_entry.dhcp_entry = RouterEntry(config_entry.remote_dhcp_entry)        
 
         return router_entry

mktxp/flow/router_entry.py

@@ -12,7 +12,8 @@
 ## GNU General Public License for more details.
 
 
-from mktxp.cli.config.config import config_handler, MKTXPConfigKeys
+from collections import namedtuple
+from mktxp.cli.config.config import config_handler, MKTXPConfigKeys, CollectorKeys
 from mktxp.flow.router_connection import RouterAPIConnection
 
 
@@ -29,39 +30,71 @@ class RouterEntry:
             }
         
         self.wifi_package = None
-        self.dhcp_entry = None
+        self.time_spent =  { CollectorKeys.IDENTITY_COLLECTOR: 0,
-
+                            CollectorKeys.SYSTEM_RESOURCE_COLLECTOR: 0,
-        self.time_spent =  { 'IdentityCollector': 0,
+                            CollectorKeys.HEALTH_COLLECTOR: 0,
-                            'SystemResourceCollector': 0,
+                            CollectorKeys.PUBLIC_IP_ADDRESS_COLLECTOR: 0,
-                            'HealthCollector': 0,
+                            CollectorKeys.IPV6_NEIGHBOR_COLLECTOR: 0,
-                            'PublicIPAddressCollector': 0,
+                            CollectorKeys.PACKAGE_COLLECTOR: 0,
-                            'IPv6NeighborCollector': 0,
+                            CollectorKeys.DHCP_COLLECTOR: 0,
-                            'PackageCollector': 0,
+                            CollectorKeys.POOL_COLLECTOR: 0,
-                            'DHCPCollector': 0,
+                            CollectorKeys.IP_CONNECTION_COLLECTOR: 0,
-                            'PoolCollector': 0,
+                            CollectorKeys.INTERFACE_COLLECTOR: 0,
-                            'IPConnectionCollector': 0,
+                            CollectorKeys.FIREWALL_COLLECTOR: 0,
-                            'InterfaceCollector': 0,
+                            CollectorKeys.MONITOR_COLLECTOR: 0,
-                            'FirewallCollector': 0,
+                            CollectorKeys.POE_COLLECTOR: 0,
-                            'MonitorCollector': 0,
+                            CollectorKeys.NETWATCH_COLLECTOR: 0,
-                            'POECollector': 0,
+                            CollectorKeys.ROUTE_COLLECTOR: 0,
-                            'NetwatchCollector': 0,
+                            CollectorKeys.WLAN_COLLECTOR: 0,
-                            'RouteCollector': 0,
+                            CollectorKeys.CAPSMAN_COLLECTOR: 0,
-                            'WLANCollector': 0,
+                            CollectorKeys.QUEUE_TREE_COLLECTOR: 0,
-                            'CapsmanCollector': 0,
+                            CollectorKeys.QUEUE_SIMPLE_COLLECTOR: 0,                            
-                            'QueueTreeCollector': 0,
+                            CollectorKeys.USER_COLLECTOR: 0,                            
-                            'QueueSimpleCollector': 0,                            
+                            CollectorKeys.MKTXP_COLLECTOR: 0
-                            'UserCollector': 0,                            
-                            'MKTXPCollector': 0
                             }         
+        self._dhcp_entry = None
+        self._dhcp_records = {}
+                                    
+    @property
+    def dhcp_entry(self):
+        if self._dhcp_entry:
+            return self._dhcp_entry
+        return self
+    @dhcp_entry.setter
+    def dhcp_entry(self, dhcp_entry):
+        self._dhcp_entry = dhcp_entry
+
+    @property
+    def dhcp_records(self):
+        return (entry.record for key, entry in  self._dhcp_records.items() if entry.type == 'mac_address') \
+                                                                                if self._dhcp_records else None   
+    @dhcp_records.setter
+    def dhcp_records(self, dhcp_records):
+        for dhcp_record in dhcp_records:
+            if dhcp_record.get('mac_address'):
+                self._dhcp_records[dhcp_record.get('mac_address')] = DHCPCacheEntry('mac_address', dhcp_record)
+            if dhcp_record.get('address'):
+                dhcp_record['type'] = 'address'
+                self._dhcp_records[dhcp_record.get('address')] = DHCPCacheEntry('address', dhcp_record)
+
+    def dhcp_record(self, key):
+        if self._dhcp_records and self._dhcp_records.get(key):
+            return self._dhcp_records[key].record
+        return None
 
     def is_ready(self):
+        self.is_done() #flush caches, just in case
         is_ready = True
-        self.wifi_package = None 
         if not self.api_connection.is_connected():
             is_ready = False                        
             # let's get connected now
             self.api_connection.connect()
-            if self.dhcp_entry:
+            if self._dhcp_entry:
-                self.dhcp_entry.api_connection.connect()
+                self._dhcp_entry.api_connection.connect()
-
         return is_ready
+
+    def is_done(self):
+        self.wifi_package = None 
+        self._dhcp_records = {}
+
+DHCPCacheEntry = namedtuple('DHCPCacheEntry', ['type', 'record'])

setup.py

@@ -20,7 +20,7 @@ with open(path.join(pkg_dir, 'README.md'), encoding='utf-8') as f:
 
 setup(
     name='mktxp',
-    version='1.1',
+    version='1.2',
 
     url='https://github.com/akpw/mktxp',