POE collector, bandwidth metrics on/off switch, fixes/optimizations

mktxp/cli/config/_mktxp.conf

@@ -14,9 +14,10 @@
 [MKTXP]
     port = 49090
     socket_timeout = 2
+    
     initial_delay_on_failure = 120
     max_delay_on_failure = 900
     delay_inc_div = 5
-    bandwidth_test_interval = 420
 
-	collectors = IdentityCollector, SystemResourceCollector, HealthCollector, DHCPCollector, PoolCollector, InterfaceCollector, FirewallCollector, MonitorCollector, RouteCollector, WLANCollector, CapsmanCollector, MKTXPCollector
+	bandwidth = True                # Turns metrics bandwidth metrics collection on / off    
+	bandwidth_test_interval = 420   # Interval for colllecting bandwidth metrics

mktxp/cli/config/config.py

@@ -47,14 +47,15 @@ class MKTXPConfigKeys:
 	FE_WIRELESS_CLIENTS_KEY = 'wireless_clients'
 	FE_CAPSMAN_KEY = 'capsman'
 	FE_CAPSMAN_CLIENTS_KEY = 'capsman_clients'
+	FE_POE_KEY = 'poe'
 
 	MKTXP_SOCKET_TIMEOUT = 'socket_timeout'    
 	MKTXP_SOCKET_TIMEOUT = 'socket_timeout'   
 	MKTXP_INITIAL_DELAY = 'initial_delay_on_failure'
 	MKTXP_MAX_DELAY = 'max_delay_on_failure'
 	MKTXP_INC_DIV = 'delay_inc_div'
+	MKTXP_BANDWIDTH_KEY = 'bandwidth'
 	MKTXP_BANDWIDTH_TEST_INTERVAL = 'bandwidth_test_interval'
-	MKTXP_COLLECTORS = 'collectors'
 
 	# UnRegistered entries placeholder
 	NO_ENTRIES_REGISTERED = 'NoEntriesRegistered'
@@ -75,13 +76,15 @@ class MKTXPConfigKeys:
 	DEFAULT_MKTXP_MAX_DELAY = 900
 	DEFAULT_MKTXP_INC_DIV = 5
 	DEFAULT_MKTXP_BANDWIDTH_TEST_INTERVAL = 420
-	DEFAULT_MKTXP_COLLECTORS = ['IdentityCollector', 'SystemResourceCollector', 'HealthCollector', 'DHCPCollector', 'PoolCollector', 
-								'InterfaceCollector', 'FirewallCollector', 'MonitorCollector', 'RouteCollector', 'WLANCollector', 'CapsmanCollector', 'MKTXPCollector']
 
-	BOOLEAN_KEYS = (ENABLED_KEY, SSL_KEY, NO_SSL_CERTIFICATE, SSL_CERTIFICATE_VERIFY, 
+	BOOLEAN_KEYS_NO = {ENABLED_KEY, SSL_KEY, NO_SSL_CERTIFICATE, SSL_CERTIFICATE_VERIFY}
-					  FE_DHCP_KEY, FE_DHCP_LEASE_KEY, FE_DHCP_POOL_KEY, FE_INTERFACE_KEY, FE_FIREWALL_KEY,
+	
+	# Feature keys enabled by default
+	BOOLEAN_KEYS_YES = {FE_DHCP_KEY, FE_DHCP_LEASE_KEY, FE_DHCP_POOL_KEY, FE_INTERFACE_KEY, FE_FIREWALL_KEY,
 					  FE_MONITOR_KEY, FE_ROUTE_KEY, MKTXP_USE_COMMENTS_OVER_NAMES, 
-					  FE_WIRELESS_KEY, FE_WIRELESS_CLIENTS_KEY, FE_CAPSMAN_KEY, FE_CAPSMAN_CLIENTS_KEY)
+					  FE_WIRELESS_KEY, FE_WIRELESS_CLIENTS_KEY, FE_CAPSMAN_KEY, FE_CAPSMAN_CLIENTS_KEY, FE_POE_KEY}
+
+	SYSTEM_BOOLEAN_KEYS_YES = (MKTXP_BANDWIDTH_KEY,)
 
 	STR_KEYS = (HOST_KEY, USER_KEY, PASSWD_KEY)
 	MKTXP_INT_KEYS = (PORT_KEY, MKTXP_SOCKET_TIMEOUT, MKTXP_INITIAL_DELAY, MKTXP_MAX_DELAY, MKTXP_INC_DIV, MKTXP_BANDWIDTH_TEST_INTERVAL)
@@ -97,11 +100,11 @@ class ConfigEntry:
 
 												 MKTXPConfigKeys.FE_DHCP_KEY, MKTXPConfigKeys.FE_DHCP_LEASE_KEY, MKTXPConfigKeys.FE_DHCP_POOL_KEY, MKTXPConfigKeys.FE_INTERFACE_KEY, MKTXPConfigKeys.FE_FIREWALL_KEY,
 												 MKTXPConfigKeys.FE_MONITOR_KEY, MKTXPConfigKeys.FE_ROUTE_KEY, MKTXPConfigKeys.FE_WIRELESS_KEY, MKTXPConfigKeys.FE_WIRELESS_CLIENTS_KEY,
-												 MKTXPConfigKeys.FE_CAPSMAN_KEY, MKTXPConfigKeys.FE_CAPSMAN_CLIENTS_KEY, MKTXPConfigKeys.MKTXP_USE_COMMENTS_OVER_NAMES
+												 MKTXPConfigKeys.FE_CAPSMAN_KEY, MKTXPConfigKeys.FE_CAPSMAN_CLIENTS_KEY, MKTXPConfigKeys.FE_POE_KEY, MKTXPConfigKeys.MKTXP_USE_COMMENTS_OVER_NAMES
 												 ])
 	MKTXPSystemEntry = namedtuple('MKTXPSystemEntry', [MKTXPConfigKeys.PORT_KEY, MKTXPConfigKeys.MKTXP_SOCKET_TIMEOUT,
 												  MKTXPConfigKeys.MKTXP_INITIAL_DELAY, MKTXPConfigKeys.MKTXP_MAX_DELAY,
-												  MKTXPConfigKeys.MKTXP_INC_DIV, MKTXPConfigKeys.MKTXP_BANDWIDTH_TEST_INTERVAL, MKTXPConfigKeys.MKTXP_COLLECTORS])
+												  MKTXPConfigKeys.MKTXP_INC_DIV, MKTXPConfigKeys.MKTXP_BANDWIDTH_KEY, MKTXPConfigKeys.MKTXP_BANDWIDTH_TEST_INTERVAL])
 
 
 class OSConfig(metaclass = ABCMeta):
@@ -202,11 +205,11 @@ class MKTXPConfigHandler:
 	def _config_entry_reader(self, entry_name):
 		config_entry_reader = {}
 		write_needed = False
-		for key in MKTXPConfigKeys.BOOLEAN_KEYS:
+		for key in MKTXPConfigKeys.BOOLEAN_KEYS_NO.union(MKTXPConfigKeys.BOOLEAN_KEYS_YES):
 			if self.config[entry_name].get(key):
 				config_entry_reader[key] = self.config[entry_name].as_bool(key)
 			else:
-				config_entry_reader[key] = False
+				config_entry_reader[key] = True if key in MKTXPConfigKeys.BOOLEAN_KEYS_YES else False
 				write_needed = True # read from disk next time                
 
 		for key in MKTXPConfigKeys.STR_KEYS:
@@ -237,11 +240,11 @@ class MKTXPConfigHandler:
 				system_entry_reader[key] = self._default_value_for_key(key)
 				write_needed = True # read from disk next time                
 
-		# Collectors
+		for key in MKTXPConfigKeys.SYSTEM_BOOLEAN_KEYS_YES:
-		if self._config[entry_name].get(MKTXPConfigKeys.MKTXP_COLLECTORS):
+			if self._config[entry_name].get(key):
-			system_entry_reader[MKTXPConfigKeys.MKTXP_COLLECTORS] = self._config[entry_name][MKTXPConfigKeys.MKTXP_COLLECTORS]
+				system_entry_reader[key] = self._config[entry_name].as_bool(key)
 			else:
-			system_entry_reader[MKTXPConfigKeys.MKTXP_COLLECTORS] = self._default_value_for_key(MKTXPConfigKeys.MKTXP_COLLECTORS)
+				system_entry_reader[key] = True
 				write_needed = True # read from disk next time                
 			
 		if write_needed:
@@ -258,13 +261,9 @@ class MKTXPConfigHandler:
 				MKTXPConfigKeys.MKTXP_INITIAL_DELAY: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_INITIAL_DELAY,
 				MKTXPConfigKeys.MKTXP_MAX_DELAY: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_MAX_DELAY,
 				MKTXPConfigKeys.MKTXP_INC_DIV: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_INC_DIV,
-				MKTXPConfigKeys.MKTXP_BANDWIDTH_TEST_INTERVAL: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_BANDWIDTH_TEST_INTERVAL,
+				MKTXPConfigKeys.MKTXP_BANDWIDTH_TEST_INTERVAL: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_BANDWIDTH_TEST_INTERVAL
-				MKTXPConfigKeys.MKTXP_COLLECTORS: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_COLLECTORS
 				}[key](value)
 
-
 # Simplest possible Singleton impl
 config_handler = MKTXPConfigHandler()
 
-
-

mktxp/cli/config/mktxp.conf

@@ -30,6 +30,7 @@
     interface = True                # Interfaces traffic metrics
     firewall = True                 # Firewall rules traffic metrics
     monitor = True                  # Interface monitor metrics
+    poe = True                      # POE metrics
     route = True                    # Routes metrics
     wireless = True                 # WLAN general metrics
     wireless_clients = True         # WLAN clients metrics

mktxp/collector/bandwidth_collector.py

@@ -33,6 +33,9 @@ class BandwidthCollector(BaseCollector):
         self.last_call_timestamp = 0        
     
     def collect(self):
+        if not config_handler.system_entry().bandwidth:
+            return
+
         if result_list:      
             result_dict = result_list[0]
             bandwidth_records = [{'direction': key, 'bandwidth': str(result_dict[key])} for key in ('download', 'upload')]     

mktxp/collector/monitor_collector.py

@@ -25,7 +25,7 @@ class MonitorCollector(BaseCollector):
         if not router_entry.config_entry.monitor:
             return
 
-        monitor_labels = ('status', 'rate', 'full_duplex', 'name')
+        monitor_labels = ['status', 'rate', 'full_duplex', 'name']
         monitor_records = InterfaceMonitorMetricsDataSource.metric_records(router_entry, metric_labels = monitor_labels, include_comments = True)   
         if monitor_records:
             # translate records to appropriate values

mktxp/collector/poe_collector.py

@@ -0,0 +1,34 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from mktxp.collector.base_collector import BaseCollector
+from mktxp.flow.processor.output import BaseOutputProcessor
+from mktxp.datasource.poe_ds import POEMetricsDataSource
+
+
+class POECollector(BaseCollector):
+    ''' POE Metrics collector
+    '''    
+    @staticmethod
+    def collect(router_entry):
+        if not router_entry.config_entry.poe:
+            return
+
+        poe_labels = ['name', 'poe_out', 'poe_priority', 'poe_voltage', 'poe_out_status', 'poe_out_voltage', 'poe_out_current', 'poe_out_power']
+        poe_records = POEMetricsDataSource.metric_records(router_entry, include_comments = True, metric_labels = poe_labels)  
+
+        if poe_records:
+            poe_metrics = BaseCollector.info_collector('poe', 'POE Metrics', poe_records, poe_labels)
+            yield poe_metrics
+            

mktxp/datasource/poe_ds.py

@@ -0,0 +1,53 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from mktxp.datasource.base_ds import BaseDSProcessor
+
+
+class POEMetricsDataSource:
+    ''' POE Metrics data provider
+    '''             
+    @staticmethod
+    def metric_records(router_entry, *, include_comments = False, metric_labels = []):
+        try:
+            poe_records = router_entry.api_connection.router_api().get_resource('/interface/ethernet/poe').get()
+            for int_num, poe_record in enumerate(poe_records):
+                poe_monitor_record = router_entry.api_connection.router_api().get_resource('/interface/ethernet/poe').call('monitor', {'once':'', 'numbers':f'{int_num}'})
+                if poe_monitor_record[0].get('poe_out_status'):
+                    poe_record['poe_out_status'] = poe_monitor_record[0]['poe_out_status']
+
+                if poe_monitor_record[0].get('poe_out_voltage'):
+                    poe_record['poe_out_voltage'] = poe_monitor_record[0]['poe_out_voltage']
+
+                if poe_monitor_record[0].get('poe_out_current'):
+                    poe_record['poe_out_current'] = poe_monitor_record[0]['poe_out_current']
+
+                if poe_monitor_record[0].get('poe_out_power'):
+                    poe_record['poe_out_power'] = poe_monitor_record[0]['poe_out_power']
+
+            if include_comments:
+                interfaces = router_entry.api_connection.router_api().get_resource('/interface/ethernet').get()
+                comment = lambda interface: interface['comment'] if interface.get('comment') else ''
+                for poe_record in poe_records:
+                    comment = [comment(interface) for interface in interfaces if interface['name'] == poe_record['name']][0]                    
+                    if comment:
+                        # combines name with comment
+                        poe_record['name'] = comment if router_entry.config_entry.use_comments_over_names else \
+                                                                                                    f"{poe_record['name']} ({comment})"                                
+                                                                                                    
+            return BaseDSProcessor.trimmed_records(router_entry, router_records = poe_records, metric_labels = metric_labels)
+        except Exception as exc:
+            print(f'Error getting PoE info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}')
+            return None
+

mktxp/flow/collector_registry.py

@@ -19,6 +19,7 @@ from mktxp.collector.interface_collector import InterfaceCollector
 from mktxp.collector.health_collector import HealthCollector
 from mktxp.collector.identity_collector import IdentityCollector
 from mktxp.collector.monitor_collector import MonitorCollector
+from mktxp.collector.poe_collector import POECollector
 from mktxp.collector.pool_collector import PoolCollector
 from mktxp.collector.resource_collector import SystemResourceCollector
 from mktxp.collector.route_collector import RouteCollector
@@ -48,6 +49,7 @@ class CollectorRegistry:
 
         self.register('FirewallCollector', FirewallCollector.collect)
         self.register('MonitorCollector', MonitorCollector.collect)
+        self.register('POECollector', POECollector.collect)
         self.register('RouteCollector', RouteCollector.collect)
 
         self.register('WLANCollector', WLANCollector.collect)
@@ -58,3 +60,4 @@ class CollectorRegistry:
     def register(self, collector_ID, collect_func):
         self.registered_collectors[collector_ID] = collect_func
 
+

mktxp/flow/router_entry.py

@@ -35,6 +35,7 @@ class RouterEntry:
                             'InterfaceCollector': 0,
                             'FirewallCollector': 0,
                             'MonitorCollector': 0,
+                            'POECollector': 0,
                             'RouteCollector': 0,
                             'WLANCollector': 0,
                             'CapsmanCollector': 0,

setup.py

@@ -20,7 +20,7 @@ with open(path.join(pkg_dir, 'README.md'), encoding='utf-8') as f:
 
 setup(
     name='mktxp',
-    version='0.28',
+    version='0.29',
 
     url='https://github.com/akpw/mktxp',