Collector registry, fixes/optimizations

mktxp/cli/config/_mktxp.conf

@@ -12,10 +12,11 @@
 
 
 [MKTXP]
-    port = 49090					# default metrics HTTP server port
+    port = 49090
-    bandwidth_test_interval = 420	# Interval between periodic bandwidth tests
+    socket_timeout = 2
-
+    initial_delay_on_failure = 120
-    socket_timeout = 2				# Socket connection timeout
+    max_delay_on_failure = 900
-    initial_delay_on_failure = 120	# Delay untill next connection attempt to a RouterOS Device
+    delay_inc_div = 5
-    max_delay_on_failure = 900		# Max delay untill next connection attempt to a RouterOS Device
+    bandwidth_test_interval = 420
-    delay_inc_div = 5				# Delay increment factor
+    
+	collectors = IdentityCollector, SystemResourceCollector, HealthCollector, DHCPCollector, PoolCollector, InterfaceCollector, FirewallCollector, MonitorCollector, RouteCollector, WLANCollector, CapsmanCollector, MKTXPCollector

mktxp/cli/config/config.py

@@ -54,6 +54,7 @@ class MKTXPConfigKeys:
 	MKTXP_MAX_DELAY = 'max_delay_on_failure'
 	MKTXP_INC_DIV = 'delay_inc_div'
 	MKTXP_BANDWIDTH_TEST_INTERVAL = 'bandwidth_test_interval'
+	MKTXP_COLLECTORS = 'collectors'
 
 	# UnRegistered entries placeholder
 	NO_ENTRIES_REGISTERED = 'NoEntriesRegistered'
@@ -74,6 +75,8 @@ class MKTXPConfigKeys:
 	DEFAULT_MKTXP_MAX_DELAY = 900
 	DEFAULT_MKTXP_INC_DIV = 5
 	DEFAULT_MKTXP_BANDWIDTH_TEST_INTERVAL = 420
+	DEFAULT_MKTXP_COLLECTORS = ['IdentityCollector', 'SystemResourceCollector', 'HealthCollector', 'DHCPCollector', 'PoolCollector', 
+								'InterfaceCollector', 'FirewallCollector', 'MonitorCollector', 'RouteCollector', 'WLANCollector', 'CapsmanCollector', 'MKTXPCollector']
 
 	BOOLEAN_KEYS = (ENABLED_KEY, SSL_KEY, NO_SSL_CERTIFICATE, SSL_CERTIFICATE_VERIFY, 
 					  FE_DHCP_KEY, FE_DHCP_LEASE_KEY, FE_DHCP_POOL_KEY, FE_INTERFACE_KEY, FE_FIREWALL_KEY,
@@ -88,17 +91,17 @@ class MKTXPConfigKeys:
 
 
 class ConfigEntry:
-	MKTXPEntry = namedtuple('MKTXPEntry', [MKTXPConfigKeys.ENABLED_KEY, MKTXPConfigKeys.HOST_KEY, MKTXPConfigKeys.PORT_KEY, 
+	MKTXPConfigEntry = namedtuple('MKTXPConfigEntry', [MKTXPConfigKeys.ENABLED_KEY, MKTXPConfigKeys.HOST_KEY, MKTXPConfigKeys.PORT_KEY,
-						 MKTXPConfigKeys.USER_KEY, MKTXPConfigKeys.PASSWD_KEY, 
+												 MKTXPConfigKeys.USER_KEY, MKTXPConfigKeys.PASSWD_KEY,
-						 MKTXPConfigKeys.SSL_KEY, MKTXPConfigKeys.NO_SSL_CERTIFICATE, MKTXPConfigKeys.SSL_CERTIFICATE_VERIFY,
+												 MKTXPConfigKeys.SSL_KEY, MKTXPConfigKeys.NO_SSL_CERTIFICATE, MKTXPConfigKeys.SSL_CERTIFICATE_VERIFY,
-						 
+
-						 MKTXPConfigKeys.FE_DHCP_KEY, MKTXPConfigKeys.FE_DHCP_LEASE_KEY, MKTXPConfigKeys.FE_DHCP_POOL_KEY, MKTXPConfigKeys.FE_INTERFACE_KEY, MKTXPConfigKeys.FE_FIREWALL_KEY,
+												 MKTXPConfigKeys.FE_DHCP_KEY, MKTXPConfigKeys.FE_DHCP_LEASE_KEY, MKTXPConfigKeys.FE_DHCP_POOL_KEY, MKTXPConfigKeys.FE_INTERFACE_KEY, MKTXPConfigKeys.FE_FIREWALL_KEY,
-						 MKTXPConfigKeys.FE_MONITOR_KEY, MKTXPConfigKeys.FE_ROUTE_KEY, MKTXPConfigKeys.FE_WIRELESS_KEY, MKTXPConfigKeys.FE_WIRELESS_CLIENTS_KEY,
+												 MKTXPConfigKeys.FE_MONITOR_KEY, MKTXPConfigKeys.FE_ROUTE_KEY, MKTXPConfigKeys.FE_WIRELESS_KEY, MKTXPConfigKeys.FE_WIRELESS_CLIENTS_KEY,
-						 MKTXPConfigKeys.FE_CAPSMAN_KEY, MKTXPConfigKeys.FE_CAPSMAN_CLIENTS_KEY, MKTXPConfigKeys.MKTXP_USE_COMMENTS_OVER_NAMES
+												 MKTXPConfigKeys.FE_CAPSMAN_KEY, MKTXPConfigKeys.FE_CAPSMAN_CLIENTS_KEY, MKTXPConfigKeys.MKTXP_USE_COMMENTS_OVER_NAMES
-						 ])
+												 ])
-	_MKTXPEntry = namedtuple('_MKTXPEntry', [MKTXPConfigKeys.PORT_KEY, MKTXPConfigKeys.MKTXP_SOCKET_TIMEOUT, 
+	MKTXPSystemEntry = namedtuple('MKTXPSystemEntry', [MKTXPConfigKeys.PORT_KEY, MKTXPConfigKeys.MKTXP_SOCKET_TIMEOUT,
-											MKTXPConfigKeys.MKTXP_INITIAL_DELAY, MKTXPConfigKeys.MKTXP_MAX_DELAY, 
+												  MKTXPConfigKeys.MKTXP_INITIAL_DELAY, MKTXPConfigKeys.MKTXP_MAX_DELAY,
-											MKTXPConfigKeys.MKTXP_INC_DIV, MKTXPConfigKeys.MKTXP_BANDWIDTH_TEST_INTERVAL])
+												  MKTXPConfigKeys.MKTXP_INC_DIV, MKTXPConfigKeys.MKTXP_BANDWIDTH_TEST_INTERVAL, MKTXPConfigKeys.MKTXP_COLLECTORS])
 
 
 class OSConfig(metaclass = ABCMeta):
@@ -158,9 +161,32 @@ class MKTXPConfigHandler:
 		self._create_os_path(self.mktxp_conf_path, 'mktxp/cli/config/_mktxp.conf')
 
 		self.re_compiled = {}
-		
+
 		self._read_from_disk()
 
+	# MKTXP entries
+	def registered_entries(self):
+		''' All MKTXP registered entries
+		'''
+		registered_entries = [entry_name for entry_name in self.config.keys()]
+		if not registered_entries:
+			registered_entries = [MKTXPConfigKeys.NO_ENTRIES_REGISTERED]
+
+		return registered_entries
+
+	def config_entry(self, entry_name):
+		''' Given an entry name, reads and returns the entry info
+		'''
+		entry_reader = self._config_entry_reader(entry_name)
+		return ConfigEntry.MKTXPConfigEntry(**entry_reader)
+
+	def system_entry(self):
+		''' MKTXP internal config entry
+		'''
+		_entry_reader = self._system_entry_reader()
+		return ConfigEntry.MKTXPSystemEntry(**_entry_reader)
+
+	# Helpers
 	def _read_from_disk(self):
 		''' (Force-)Read conf data from disk
 		'''
@@ -173,112 +199,67 @@ class MKTXPConfigHandler:
 			lookup_path = resource_filename(Requirement.parse("mktxp"), resource_path)
 			shutil.copy(lookup_path, os_path)
 
-
+	def _config_entry_reader(self, entry_name):
-	# MKTXP entries
+		config_entry_reader = {}
-	##############
-	def register_entry(self, entry_name, entry_args, quiet = False):
-		''' Registers MKTXP conf entry
-		'''
-		if entry_name in self.registered_entries():
-			if not quiet:
-				print('"{0}": entry name already registered'.format(entry_name))
-			return False
-		else:
-			self.config[entry_name] = entry_args
-			self.config.write()
-			if not quiet:
-				print('Entry registered: {0}'.format(entry_name))
-			return True
-
-	def unregister_entry(self, entry_name, quiet = False):
-		''' Un-registers MKTXP conf entry
-		'''
-		if self.config[entry_name]:
-			del(self.config[entry_name])
-			self.config.write()
-			if not quiet:
-				print('Unregistered entry: {}'.format(entry_name))
-			return True
-		else:
-			if not quiet:
-				print('Entry is not registered: {}'.format(entry_name))
-			return False
-
-	def registered_entries(self):
-		''' All MKTXP registered entries
-		'''
-		registered_entries = [entry_name for entry_name in self.config.keys()]
-		if not registered_entries:
-			registered_entries = [MKTXPConfigKeys.NO_ENTRIES_REGISTERED]
-
-		return registered_entries
-
-	def entry(self, entry_name):
-		''' Given an entry name, reads and returns the entry info
-		'''
-		entry_reader = self.entry_reader(entry_name)
-		return ConfigEntry.MKTXPEntry(**entry_reader)
-
-	def _entry(self):
-		''' MKTXP internal config entry
-		'''
-		_entry_reader = self._entry_reader()
-		return ConfigEntry._MKTXPEntry(**_entry_reader)
-
-
-	# Helpers
-	def entry_reader(self, entry_name):
-		entry_reader = {}
 		write_needed = False
 		for key in MKTXPConfigKeys.BOOLEAN_KEYS:
 			if self.config[entry_name].get(key):
-				entry_reader[key] = self.config[entry_name].as_bool(key)
+				config_entry_reader[key] = self.config[entry_name].as_bool(key)
 			else:
-				entry_reader[key] = False
+				config_entry_reader[key] = False
 				write_needed = True # read from disk next time                
 
 		for key in MKTXPConfigKeys.STR_KEYS:
-			entry_reader[key] = self.config[entry_name][key]
+			config_entry_reader[key] = self.config[entry_name][key]
 
 		# port
 		if self.config[entry_name].get(MKTXPConfigKeys.PORT_KEY):
-			entry_reader[MKTXPConfigKeys.PORT_KEY] = self.config[entry_name].as_int(MKTXPConfigKeys.PORT_KEY)
+			config_entry_reader[MKTXPConfigKeys.PORT_KEY] = self.config[entry_name].as_int(MKTXPConfigKeys.PORT_KEY)
 		else:
-			entry_reader[MKTXPConfigKeys.PORT_KEY] = self._default_value_for_key(MKTXPConfigKeys.SSL_KEY, entry_reader[MKTXPConfigKeys.SSL_KEY])
+			config_entry_reader[MKTXPConfigKeys.PORT_KEY] = self._default_value_for_key(MKTXPConfigKeys.SSL_KEY, config_entry_reader[MKTXPConfigKeys.SSL_KEY])
 			write_needed = True # read from disk next time                
 
 		if write_needed:
-			self.config[entry_name] = entry_reader
+			self.config[entry_name] = config_entry_reader
 			self.config.write()
 
-		return entry_reader
+		return config_entry_reader
 
-	def _entry_reader(self):
+	def _system_entry_reader(self):
-		_entry_reader = {}
+		system_entry_reader = {}
 		entry_name = MKTXPConfigKeys.MKTXP_CONFIG_ENTRY_NAME
 		write_needed = False
+
 		for key in MKTXPConfigKeys.MKTXP_INT_KEYS:
 			if self._config[entry_name].get(key):
-				_entry_reader[key] = self._config[entry_name].as_int(key)
+				system_entry_reader[key] = self._config[entry_name].as_int(key)
 			else:
-				_entry_reader[key] = self._default_value_for_key(key)
+				system_entry_reader[key] = self._default_value_for_key(key)
 				write_needed = True # read from disk next time                
 			
+		# Collectors
+		if self._config[entry_name].get(MKTXPConfigKeys.MKTXP_COLLECTORS):
+			system_entry_reader[MKTXPConfigKeys.MKTXP_COLLECTORS] = self._config[entry_name][MKTXPConfigKeys.MKTXP_COLLECTORS]
+		else:
+			system_entry_reader[MKTXPConfigKeys.MKTXP_COLLECTORS] = self._default_value_for_key(MKTXPConfigKeys.MKTXP_COLLECTORS)
+			write_needed = True # read from disk next time 
+
 		if write_needed:
-			self._config[entry_name] = _entry_reader
+			self._config[entry_name] = system_entry_reader
 			self._config.write()
 			
-		return _entry_reader
+		return system_entry_reader
 
 	def _default_value_for_key(self, key, value = None):
 		return {
 				MKTXPConfigKeys.SSL_KEY: lambda value: MKTXPConfigKeys.DEFAULT_API_SSL_PORT if value else MKTXPConfigKeys.DEFAULT_API_PORT,
-				MKTXPConfigKeys.PORT_KEY: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_PORT,
+				MKTXPConfigKeys.PORT_KEY: MKTXPConfigKeys.DEFAULT_MKTXP_PORT,
-				MKTXPConfigKeys.MKTXP_SOCKET_TIMEOUT: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_SOCKET_TIMEOUT,
+				MKTXPConfigKeys.MKTXP_SOCKET_TIMEOUT: MKTXPConfigKeys.DEFAULT_MKTXP_SOCKET_TIMEOUT,
-				MKTXPConfigKeys.MKTXP_INITIAL_DELAY: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_INITIAL_DELAY,
+				MKTXPConfigKeys.MKTXP_INITIAL_DELAY: MKTXPConfigKeys.DEFAULT_MKTXP_INITIAL_DELAY,
-				MKTXPConfigKeys.MKTXP_MAX_DELAY: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_MAX_DELAY,
+				MKTXPConfigKeys.MKTXP_MAX_DELAY: MKTXPConfigKeys.DEFAULT_MKTXP_MAX_DELAY,
-				MKTXPConfigKeys.MKTXP_INC_DIV: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_INC_DIV,
+				MKTXPConfigKeys.MKTXP_INC_DIV: MKTXPConfigKeys.DEFAULT_MKTXP_INC_DIV,
-				MKTXPConfigKeys.MKTXP_BANDWIDTH_TEST_INTERVAL: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_BANDWIDTH_TEST_INTERVAL
+				MKTXPConfigKeys.MKTXP_BANDWIDTH_TEST_INTERVAL: MKTXPConfigKeys.DEFAULT_MKTXP_BANDWIDTH_TEST_INTERVAL,
+				MKTXPConfigKeys.MKTXP_COLLECTORS: MKTXPConfigKeys.DEFAULT_MKTXP_COLLECTORS
 				}[key](value)
 
 

mktxp/cli/dispatch.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!.usr/bin/env python
 # coding=utf8
 ## Copyright (c) 2020 Arseniy Kuznetsov
 ##
@@ -16,7 +16,7 @@
 import subprocess
 from mktxp.cli.config.config import config_handler
 from mktxp.cli.options import MKTXPOptionsParser, MKTXPCommands
-from mktxp.processor.mktxp import MKTXPProcessor, MKTXPCLIProcessor
+from mktxp.flow.processor.base_proc import ExportProcessor, OutputProcessor
 
 
 class MKTXPDispatcher:
@@ -60,12 +60,11 @@ class MKTXPDispatcher:
         if args['config']:
             print(f'MKTXP data config: {config_handler.usr_conf_data_path}')
             print(f'MKTXP internal config: {config_handler.mktxp_conf_path}')
-
         else:
             for entryname in config_handler.registered_entries():
                 if args['entry_name'] and entryname != args['entry_name']:
                     continue
-                entry = config_handler.entry(entryname)
+                entry = config_handler.config_entry(entryname)
                 print(f'[{entryname}]')
                 divider_fields = set(['username', 'use_ssl', 'dhcp'])
                 for field in entry._fields:
@@ -87,20 +86,20 @@ class MKTXPDispatcher:
             subprocess.check_call([editor, config_handler.usr_conf_data_path])
        
     def start_export(self, args):
-        MKTXPProcessor.start()
+        ExportProcessor.start()
 
     def print(self, args):
         if not (args['wifi_clients'] or args['capsman_clients']):
             print("Select metric option(s) to print out, or run 'mktxp print -h' to find out more")
 
         if args['wifi_clients']:
-            MKTXPCLIProcessor.wifi_clients(args['entry_name'])
+            OutputProcessor.wifi_clients(args['entry_name'])
 
         if args['capsman_clients']:
-            MKTXPCLIProcessor.capsman_clients(args['entry_name'])
+            OutputProcessor.capsman_clients(args['entry_name'])
 
         if args['dhcp_clients']:
-            MKTXPCLIProcessor.dhcp_clients(args['entry_name'])
+            OutputProcessor.dhcp_clients(args['entry_name'])
             
 
 def main():

mktxp/cli/options.py

@@ -150,7 +150,7 @@ Selected metrics info can be printed on the command line. For more information,
                 args['entry_name'] = UniquePartialMatchList(config_handler.registered_entries()).find(args['entry_name'])
 
         if args['sub_cmd'] == MKTXPCommands.PRINT:
-            if not config_handler.entry(args['entry_name']).enabled:
+            if not config_handler.config_entry(args['entry_name']).enabled:
                 print(f"Can not print metrics for disabled RouterOS entry: {args['entry_name']}\nRun 'mktxp edit' to review and enable it in the configuration file first")
                 parser.exit()
 

mktxp/cli/output/capsman_out.py

@@ -12,7 +12,7 @@
 ## GNU General Public License for more details.
 
 
-from mktxp.processor.output import BaseOutputProcessor
+from mktxp.flow.processor.output import BaseOutputProcessor
 from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource
 from mktxp.datasource.capsman_ds import CapsmanRegistrationsMetricsDataSource
 

mktxp/cli/output/dhcp_out.py

@@ -12,7 +12,7 @@
 ## GNU General Public License for more details.
 
 
-from mktxp.processor.output import BaseOutputProcessor
+from mktxp.flow.processor.output import BaseOutputProcessor
 from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource
 
 

mktxp/cli/output/wifi_out.py

@@ -12,7 +12,7 @@
 ## GNU General Public License for more details.
 
 
-from mktxp.processor.output import BaseOutputProcessor
+from mktxp.flow.processor.output import BaseOutputProcessor
 from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource
 from mktxp.datasource.wireless_ds import WirelessMetricsDataSource
 

mktxp/collector/bandwidth_collector.py

@@ -46,7 +46,7 @@ class BandwidthCollector(BaseCollector):
             yield latency_metrics
 
         ts =  datetime.now().timestamp()       
-        if (ts - self.last_call_timestamp) > config_handler._entry().bandwidth_test_interval:            
+        if (ts - self.last_call_timestamp) > config_handler.system_entry().bandwidth_test_interval:
             self.pool.apply_async(BandwidthCollector.bandwidth_worker, callback=get_result)            
             self.last_call_timestamp = ts
 

mktxp/collector/capsman_collector.py

@@ -13,7 +13,7 @@
 
 
 from mktxp.cli.config.config import MKTXPConfigKeys
-from mktxp.processor.output import BaseOutputProcessor
+from mktxp.flow.processor.output import BaseOutputProcessor
 from mktxp.collector.base_collector import BaseCollector
 from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource
 from mktxp.datasource.capsman_ds import CapsmanCapsMetricsDataSource, CapsmanRegistrationsMetricsDataSource
@@ -24,6 +24,9 @@ class CapsmanCollector(BaseCollector):
     '''    
     @staticmethod
     def collect(router_entry):
+        if not router_entry.config_entry.capsman:
+            return
+
         remote_caps_labels = ['identity', 'version', 'base_mac', 'board', 'base_mac']
         remote_caps_records = CapsmanCapsMetricsDataSource.metric_records(router_entry, metric_labels = remote_caps_labels)
         if remote_caps_records:

mktxp/collector/dhcp_collector.py

@@ -22,6 +22,9 @@ class DHCPCollector(BaseCollector):
     '''    
     @staticmethod
     def collect(router_entry):
+        if not router_entry.config_entry.dhcp:
+            return
+
         dhcp_lease_labels = ['active_address', 'address', 'mac_address', 'host_name', 'comment', 'server', 'expires_after']
         dhcp_lease_records = DHCPMetricsDataSource.metric_records(router_entry, metric_labels = dhcp_lease_labels)   
         if dhcp_lease_records:

mktxp/collector/firewall_collector.py

@@ -22,19 +22,22 @@ class FirewallCollector(BaseCollector):
     '''    
     @staticmethod
     def collect(router_entry):
+        if not router_entry.config_entry.firewall:
+            return
+
         # initialize all pool counts, including those currently not used
-        firewall_labels = ['chain', 'action', 'bytes', 'comment']
+        firewall_labels = ['chain', 'action', 'bytes', 'comment', 'log']
         
         firewall_filter_records = FirewallMetricsDataSource.metric_records(router_entry, metric_labels = firewall_labels)   
         if firewall_filter_records:           
             metris_records = [FirewallCollector.metric_record(router_entry, record) for record in firewall_filter_records]
-            firewall_filter_metrics = BaseCollector.counter_collector('firewall_filter', 'Total amount of bytes matched by firewall rules', metris_records, 'bytes', ['name'])
+            firewall_filter_metrics = BaseCollector.counter_collector('firewall_filter', 'Total amount of bytes matched by firewall rules', metris_records, 'bytes', ['name', 'log'])
             yield firewall_filter_metrics
 
         firewall_raw_records = FirewallMetricsDataSource.metric_records(router_entry, metric_labels = firewall_labels, raw = True)        
         if firewall_raw_records:      
             metris_records = [FirewallCollector.metric_record(router_entry, record) for record in firewall_raw_records]     
-            firewall_raw_metrics = BaseCollector.counter_collector('firewall_raw', 'Total amount of bytes matched by raw firewall rules', metris_records, 'bytes', ['name'])
+            firewall_raw_metrics = BaseCollector.counter_collector('firewall_raw', 'Total amount of bytes matched by raw firewall rules', metris_records, 'bytes', ['name', 'log'])
             yield firewall_raw_metrics
 
     # Helpers
@@ -44,4 +47,4 @@ class FirewallCollector(BaseCollector):
         bytes = firewall_record['bytes']
         return {MKTXPConfigKeys.ROUTERBOARD_NAME: router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
                 MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
-                'name': name, 'bytes': bytes}
+                'name': name, 'log': firewall_record['log'], 'bytes': bytes}

mktxp/collector/interface_collector.py

@@ -21,6 +21,9 @@ class InterfaceCollector(BaseCollector):
     '''        
     @staticmethod
     def collect(router_entry):
+        if not router_entry.config_entry.interface:
+            return
+            
         interface_traffic_labels = ['name', 'comment', 'rx_byte', 'tx_byte', 'rx_packet', 'tx_packet', 'rx_error', 'tx_error', 'rx_drop', 'tx_drop']
         interface_traffic_records = InterfaceTrafficMetricsDataSource.metric_records(router_entry, metric_labels = interface_traffic_labels)   
 

mktxp/collector/monitor_collector.py

@@ -13,7 +13,7 @@
 
 
 from mktxp.collector.base_collector import BaseCollector
-from mktxp.processor.output import BaseOutputProcessor
+from mktxp.flow.processor.output import BaseOutputProcessor
 from mktxp.datasource.interface_ds import InterfaceMonitorMetricsDataSource
 
 
@@ -22,6 +22,9 @@ class MonitorCollector(BaseCollector):
     '''    
     @staticmethod
     def collect(router_entry):
+        if not router_entry.config_entry.monitor:
+            return
+
         monitor_labels = ('status', 'rate', 'full_duplex', 'name')
         monitor_records = InterfaceMonitorMetricsDataSource.metric_records(router_entry, metric_labels = monitor_labels, include_comments = True)   
         if monitor_records:

mktxp/collector/pool_collector.py

@@ -22,6 +22,9 @@ class PoolCollector(BaseCollector):
     '''    
     @staticmethod
     def collect(router_entry):
+        if not router_entry.config_entry.pool:
+            return
+
         # initialize all pool counts, including those currently not used
         pool_records = PoolMetricsDataSource.metric_records(router_entry, metric_labels = ['name'])   
         if pool_records:

mktxp/collector/resource_collector.py

@@ -13,7 +13,7 @@
 
 
 from mktxp.collector.base_collector import BaseCollector
-from mktxp.processor.output import BaseOutputProcessor
+from mktxp.flow.processor.output import BaseOutputProcessor
 from mktxp.datasource.system_resource_ds import SystemResourceMetricsDataSource
 
 

mktxp/collector/route_collector.py

@@ -22,6 +22,9 @@ class RouteCollector(BaseCollector):
     '''        
     @staticmethod
     def collect(router_entry):
+        if not router_entry.config_entry.route:
+            return
+
         route_labels = ['connect', 'dynamic', 'static', 'bgp', 'ospf']
         route_records = RouteMetricsDataSource.metric_records(router_entry, metric_labels = route_labels)   
         if route_records:       

mktxp/collector/wlan_collector.py

@@ -12,7 +12,7 @@
 ## GNU General Public License for more details.
 
 
-from mktxp.processor.output import BaseOutputProcessor
+from mktxp.flow.processor.output import BaseOutputProcessor
 from mktxp.collector.base_collector import BaseCollector
 from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource
 from mktxp.datasource.wireless_ds import WirelessMetricsDataSource
@@ -24,6 +24,9 @@ class WLANCollector(BaseCollector):
     '''    
     @staticmethod
     def collect(router_entry):
+        if not router_entry.config_entry.wireless:
+            return
+
         monitor_labels = ['channel', 'noise_floor', 'overall_tx_ccq', 'registered_clients']
         monitor_records = InterfaceMonitorMetricsDataSource.metric_records(router_entry, metric_labels = monitor_labels, kind = 'wireless')   
         if monitor_records:

mktxp/datasource/firewall_ds.py

@@ -29,7 +29,9 @@ class FirewallMetricsDataSource:
             # translation rules
             translation_table = {}
             if 'comment' in metric_labels:
-                translation_table['comment'] = lambda c: c if c else ''           
+                translation_table['comment'] = lambda value: value if value else ''           
+            if 'log' in metric_labels:
+                translation_table['log'] = lambda value: '1' if value == 'true' else '0'           
 
             return BaseDSProcessor.trimmed_records(router_entry, router_records = firewall_records, metric_labels = metric_labels, translation_table = translation_table)
         except Exception as exc:

mktxp/flow/collector_handler.py

@@ -0,0 +1,38 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from timeit import default_timer
+
+
+class CollectorHandler:
+    ''' MKTXP Collectors Handler
+    '''
+    def __init__(self, entries_handler, collector_registry):
+        self.entries_handler = entries_handler
+        self.collector_registry = collector_registry
+
+    def collect(self):
+        yield from self.collector_registry.bandwidthCollector.collect()
+
+        for router_entry in self.entries_handler.router_entries:
+            if not router_entry.api_connection.is_connected():
+                # let's pick up on things in the next run
+                router_entry.api_connection.connect()
+                continue
+
+            for collector_ID, collect_func in self.collector_registry.registered_collectors.items():                
+                start = default_timer()
+                yield from collect_func(router_entry)
+                router_entry.time_spent[collector_ID] += default_timer() - start
+

mktxp/flow/collector_registry.py

@@ -0,0 +1,60 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from collections import OrderedDict
+from mktxp.cli.config.config import MKTXPConfigKeys
+from mktxp.collector.dhcp_collector import DHCPCollector
+from mktxp.collector.interface_collector import InterfaceCollector
+from mktxp.collector.health_collector import HealthCollector
+from mktxp.collector.identity_collector import IdentityCollector
+from mktxp.collector.monitor_collector import MonitorCollector
+from mktxp.collector.pool_collector import PoolCollector
+from mktxp.collector.resource_collector import SystemResourceCollector
+from mktxp.collector.route_collector import RouteCollector
+from mktxp.collector.wlan_collector import WLANCollector
+from mktxp.collector.capsman_collector import CapsmanCollector
+from mktxp.collector.bandwidth_collector import BandwidthCollector
+from mktxp.collector.firewall_collector import FirewallCollector
+from mktxp.collector.mktxp_collector import MKTXPCollector
+
+
+class CollectorRegistry:
+    ''' MKTXP Collectors Registry
+    '''
+    def __init__(self):
+        self.registered_collectors = OrderedDict()
+
+        # bandwidth collector is not router-entry related, so registering directly
+        self.bandwidthCollector = BandwidthCollector()
+
+        self.register('IdentityCollector', IdentityCollector.collect)
+        self.register('SystemResourceCollector', SystemResourceCollector.collect)
+        self.register('HealthCollector', HealthCollector.collect)
+
+        self.register('DHCPCollector', DHCPCollector.collect)
+        self.register('PoolCollector', PoolCollector.collect)
+        self.register('InterfaceCollector', InterfaceCollector.collect)
+
+        self.register('FirewallCollector', FirewallCollector.collect)
+        self.register('MonitorCollector', MonitorCollector.collect)
+        self.register('RouteCollector', RouteCollector.collect)
+
+        self.register('WLANCollector', WLANCollector.collect)
+        self.register('CapsmanCollector', CapsmanCollector.collect)
+
+        self.register('MKTXPCollector', MKTXPCollector.collect)
+
+    def register(self, collector_ID, collect_func):
+        self.registered_collectors[collector_ID] = collect_func
+

mktxp/flow/collectors_handler.py

@@ -1,104 +0,0 @@
-# coding=utf8
-## Copyright (c) 2020 Arseniy Kuznetsov
-##
-## This program is free software; you can redistribute it and/or
-## modify it under the terms of the GNU General Public License
-## as published by the Free Software Foundation; either version 2
-## of the License, or (at your option) any later version.
-##
-## This program is distributed in the hope that it will be useful,
-## but WITHOUT ANY WARRANTY; without even the implied warranty of
-## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-## GNU General Public License for more details.
-
-
-from timeit import default_timer
-from mktxp.collector.dhcp_collector import DHCPCollector
-from mktxp.collector.interface_collector import InterfaceCollector
-from mktxp.collector.health_collector import HealthCollector
-from mktxp.collector.identity_collector import IdentityCollector
-from mktxp.collector.monitor_collector import MonitorCollector
-from mktxp.collector.pool_collector import PoolCollector
-from mktxp.collector.resource_collector import SystemResourceCollector
-from mktxp.collector.route_collector import RouteCollector
-from mktxp.collector.wlan_collector import WLANCollector
-from mktxp.collector.capsman_collector import CapsmanCollector
-from mktxp.collector.bandwidth_collector import BandwidthCollector
-from mktxp.collector.firewall_collector import FirewallCollector
-from mktxp.collector.mktxp_collector import MKTXPCollector
-
-
-class CollectorsHandler:
-    ''' MKTXP Collectors Handler
-    '''
-    def __init__(self, entries_handler):
-        self.entries_handler = entries_handler
-        self.bandwidthCollector = BandwidthCollector()
-
-    def collect(self):
-        # process mktxp internal metrics
-        yield from self.bandwidthCollector.collect()
-
-        for router_entry in self.entries_handler.router_entries:
-            if not router_entry.api_connection.is_connected():
-                # let's pick up on things in the next run
-                router_entry.api_connection.connect()
-                continue
-
-            start = default_timer()
-            yield from IdentityCollector.collect(router_entry)
-            router_entry.time_spent['IdentityCollector'] += default_timer() - start
-
-            start = default_timer()
-            yield from SystemResourceCollector.collect(router_entry)
-            router_entry.time_spent['SystemResourceCollector'] += default_timer() - start
-
-            start = default_timer()
-            yield from HealthCollector.collect(router_entry)
-            router_entry.time_spent['HealthCollector'] += default_timer() - start
-
-            if router_entry.config_entry.dhcp:
-                start = default_timer()
-                yield from DHCPCollector.collect(router_entry)                
-                router_entry.time_spent['DHCPCollector'] += default_timer() - start
-
-            if router_entry.config_entry.pool:
-                start = default_timer()
-                yield from PoolCollector.collect(router_entry)
-                router_entry.time_spent['PoolCollector'] += default_timer() - start
-            
-            if router_entry.config_entry.interface:
-                start = default_timer()
-                yield from InterfaceCollector.collect(router_entry)
-                router_entry.time_spent['InterfaceCollector'] += default_timer() - start
-
-            if router_entry.config_entry.firewall:
-                start = default_timer()
-                yield from FirewallCollector.collect(router_entry)
-                router_entry.time_spent['FirewallCollector'] += default_timer() - start
-            
-            if router_entry.config_entry.monitor:
-                start = default_timer()
-                yield from MonitorCollector.collect(router_entry)
-                router_entry.time_spent['MonitorCollector'] += default_timer() - start
-            
-            if router_entry.config_entry.route:
-                start = default_timer()
-                yield from RouteCollector.collect(router_entry)
-                router_entry.time_spent['RouteCollector'] += default_timer() - start
-       
-            if router_entry.config_entry.wireless:
-                start = default_timer()
-                yield from WLANCollector.collect(router_entry)
-                router_entry.time_spent['WLANCollector'] += default_timer() - start
-
-            if router_entry.config_entry.capsman:
-                start = default_timer()
-                yield from CapsmanCollector.collect(router_entry)
-                router_entry.time_spent['CapsmanCollector'] += default_timer() - start
-            
-            yield from MKTXPCollector.collect(router_entry)
-
-
-
-

mktxp/flow/processor/base_proc.py

@@ -16,8 +16,10 @@ from http.server import HTTPServer
 from datetime import datetime
 from prometheus_client.core import REGISTRY
 from prometheus_client import MetricsHandler
+
 from mktxp.cli.config.config import config_handler
-from mktxp.flow.collectors_handler import CollectorsHandler
+from mktxp.flow.collector_handler import CollectorHandler
+from mktxp.flow.collector_registry import CollectorRegistry
 from mktxp.flow.router_entries_handler import RouterEntriesHandler
 
 from mktxp.cli.output.capsman_out import CapsmanOutput
@@ -25,17 +27,16 @@ from mktxp.cli.output.wifi_out import WirelessOutput
 from mktxp.cli.output.dhcp_out import DHCPOutput
 
 
-class MKTXPProcessor:
+class ExportProcessor:
     ''' Base Export Processing
     '''    
     @staticmethod
     def start():
-        router_entries_handler = RouterEntriesHandler()
+        REGISTRY.register(CollectorHandler(RouterEntriesHandler(), CollectorRegistry()))
-        REGISTRY.register(CollectorsHandler(router_entries_handler))
+        ExportProcessor.run(port=config_handler.system_entry().port)
-        MKTXPProcessor.run(port=config_handler._entry().port)
 
     @staticmethod
-    def run(server_class=HTTPServer, handler_class=MetricsHandler, port = None):
+    def run(server_class=HTTPServer, handler_class=MetricsHandler, port=None):
         server_address = ('', port)
         httpd = server_class(server_address, handler_class)
         current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
@@ -43,7 +44,7 @@ class MKTXPProcessor:
         httpd.serve_forever()
 
 
-class MKTXPCLIProcessor:
+class OutputProcessor:
     ''' Base CLI Processing
     '''    
     @staticmethod

mktxp/flow/router_connection.py

@@ -46,7 +46,7 @@ class RouterAPIConnection:
                 ssl_verify = self.config_entry.ssl_certificate_verify,
                 ssl_context = ctx)
         
-        self.connection.socket_timeout = config_handler._entry().socket_timeout
+        self.connection.socket_timeout = config_handler.system_entry().socket_timeout
         self.api = None
 
     def is_connected(self):
@@ -91,7 +91,7 @@ class RouterAPIConnection:
         return False
 
     def _connect_delay(self):
-        mktxp_entry = config_handler._entry()
+        mktxp_entry = config_handler.system_entry()
         connect_delay = (1 + self.successive_failure_count / mktxp_entry.delay_inc_div) * mktxp_entry.initial_delay_on_failure
         return connect_delay if connect_delay < mktxp_entry.max_delay_on_failure else mktxp_entry.max_delay_on_failure
 

mktxp/flow/router_entries_handler.py

@@ -22,7 +22,7 @@ class RouterEntriesHandler:
     def __init__(self):
         self.router_entries = []
         for router_name in config_handler.registered_entries():
-            entry = config_handler.entry(router_name)
+            entry = config_handler.config_entry(router_name)
             if entry.enabled:
                 self.router_entries.append(RouterEntry(router_name))
 
@@ -32,7 +32,7 @@ class RouterEntriesHandler:
         for router_name in config_handler.registered_entries():
             if router_name == entry_name:
                 if enabled_only:
-                    entry = config_handler.entry(router_name)
+                    entry = config_handler.config_entry(router_name)
                     if not entry.enabled:
                         break
                 router_entry = RouterEntry(router_name)

mktxp/flow/router_entry.py

@@ -21,7 +21,7 @@ class RouterEntry:
     '''                 
     def __init__(self, router_name):
         self.router_name = router_name
-        self.config_entry  = config_handler.entry(router_name)
+        self.config_entry  = config_handler.config_entry(router_name)
         self.api_connection = RouterAPIConnection(router_name, self.config_entry)
         self.router_id = {
             MKTXPConfigKeys.ROUTERBOARD_NAME: self.router_name,
@@ -37,5 +37,6 @@ class RouterEntry:
                             'MonitorCollector': 0,
                             'RouteCollector': 0,
                             'WLANCollector': 0,
-                            'CapsmanCollector': 0
+                            'CapsmanCollector': 0,
+                            'MKTXPCollector': 0
                             }            

setup.py

@@ -20,7 +20,7 @@ with open(path.join(pkg_dir, 'README.md'), encoding='utf-8') as f:
 
 setup(
     name='mktxp',
-    version='0.26',
+    version='0.28',
 
     url='https://github.com/akpw/mktxp',