DS refactor, fixes/optimizations

mktxp/basep.py

@@ -11,24 +11,27 @@
 ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 ## GNU General Public License for more details.
 
+
 from http.server import HTTPServer
 from datetime import datetime
 from prometheus_client.core import REGISTRY
 from prometheus_client import MetricsHandler, start_http_server
 from mktxp.cli.config.config import config_handler
 from mktxp.collectors_handler import CollectorsHandler
-from mktxp.metrics_handler import RouterMetricsHandler
+from mktxp.router_entries_handler import RouterEntriesHandler
 
 from mktxp.cli.output.capsman_out import CapsmanOutput
 from mktxp.cli.output.wifi_out import WirelessOutput
+from mktxp.cli.output.dhcp_out import DHCPOutput
+
 
 class MKTXPProcessor:
     ''' Base Export Processing
     '''    
     @staticmethod
     def start():
-        router_metrics_handler = RouterMetricsHandler()
+        router_entries_handler = RouterEntriesHandler()
-        REGISTRY.register(CollectorsHandler(router_metrics_handler))
+        REGISTRY.register(CollectorsHandler(router_entries_handler))
         MKTXPProcessor.run(port=config_handler._entry().port)
 
     @staticmethod
@@ -45,13 +48,18 @@ class MKTXPCLIProcessor:
     '''    
     @staticmethod
     def capsman_clients(entry_name):
-        router_metric = RouterMetricsHandler.router_metric(entry_name)
+        router_entry = RouterEntriesHandler.router_entry(entry_name)
-        if router_metric:
+        if router_entry:
-            CapsmanOutput.clients_summary(router_metric)
+            CapsmanOutput.clients_summary(router_entry)
         
     @staticmethod
     def wifi_clients(entry_name):
-        router_metric = RouterMetricsHandler.router_metric(entry_name)
+        router_entry = RouterEntriesHandler.router_entry(entry_name)
-        if router_metric:
+        if router_entry:
-            WirelessOutput.clients_summary(router_metric)
+            WirelessOutput.clients_summary(router_entry)
         
+    @staticmethod
+    def dhcp_clients(entry_name):
+        router_entry = RouterEntriesHandler.router_entry(entry_name)
+        if router_entry:
+            DHCPOutput.clients_summary(router_entry)

mktxp/cli/config/_mktxp.conf

@@ -12,9 +12,10 @@
 
 
 [MKTXP]
-    port = 49090
+    port = 49090		# default metrics HTTP server port
-    socket_timeout = 2
+    bandwidth_test_interval = 420	# Interval between periodic bandwidth tests
-    initial_delay_on_failure = 120
+
-    max_delay_on_failure = 900
+    socket_timeout = 2	# Socket connection timeout
-    delay_inc_div = 5
+    initial_delay_on_failure = 120	# Delay untill next connection attempt to a RouterOS Device
-    bandwidth_test_interval = 420
+    max_delay_on_failure = 900		# Max delay untill next connection attempt to a RouterOS Device
+    delay_inc_div = 5				# Delay increment factor

mktxp/cli/dispatch.py

@@ -12,6 +12,7 @@
 ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 ## GNU General Public License for more details.
 
+
 import sys
 import subprocess
 import mktxp.cli.checks.chk_pv
@@ -20,6 +21,7 @@ from mktxp.cli.options import MKTXPOptionsParser, MKTXPCommands
 from mktxp.cli.config.config import config_handler, ConfigEntry
 from mktxp.basep import MKTXPProcessor, MKTXPCLIProcessor
 
+
 class MKTXPDispatcher:
     ''' Base MKTXP Commands Dispatcher
     '''
@@ -92,7 +94,7 @@ class MKTXPDispatcher:
 
     def print(self, args):
         if not (args['wifi_clients'] or args['capsman_clients']):
-            print("Select metric option(s) to print out, or run 'mktxp print' -h to find out more")
+            print("Select metric option(s) to print out, or run 'mktxp print -h' to find out more")
 
         if args['wifi_clients']:
             MKTXPCLIProcessor.wifi_clients(args['entry_name'])
@@ -100,6 +102,10 @@ class MKTXPDispatcher:
         if args['capsman_clients']:
             MKTXPCLIProcessor.capsman_clients(args['entry_name'])
 
+        if args['dhcp_clients']:
+            MKTXPCLIProcessor.dhcp_clients(args['entry_name'])
+
+
 
 def main():
     MKTXPDispatcher().dispatch()

mktxp/cli/options.py

@@ -11,6 +11,7 @@
 ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 ## GNU General Public License for more details.
 
+
 import os
 import pkg_resources
 from argparse import ArgumentParser, HelpFormatter
@@ -132,6 +133,9 @@ Selected metrics info can be printed on the command line. For more information,
                 help = "WiFi clients metrics",
                 action = 'store_true')
 
+        optional_args_group.add_argument('-dc', '--dhcp_clients', dest='dhcp_clients',
+                help = "DHCP clients metrics",
+                action = 'store_true')
 
     # Options checking
     def _check_args(self, args, parser):

mktxp/cli/output/base_out.py

@@ -21,16 +21,17 @@ from mktxp.cli.config.config import config_handler
 class BaseOutputProcessor:
     OutputCapsmanEntry = namedtuple('OutputCapsmanEntry', ['dhcp_name', 'dhcp_address', 'mac_address', 'rx_signal', 'interface', 'ssid', 'tx_rate', 'rx_rate', 'uptime'])
     OutputWiFiEntry = namedtuple('OutputWiFiEntry', ['dhcp_name', 'dhcp_address', 'mac_address', 'signal_strength', 'signal_to_noise', 'interface', 'tx_rate', 'rx_rate', 'uptime'])
+    OutputDHCPEntry = namedtuple('OutputDHCPEntry', ['host_name', 'comment', 'address', 'active_address', 'mac_address', 'server', 'expires_after'])
 
     @staticmethod
-    def augment_record(router_metric, registration_record, dhcp_lease_records):
+    def augment_record(router_entry, registration_record, dhcp_lease_records):
         try:
             dhcp_lease_record = next((dhcp_lease_record for dhcp_lease_record in dhcp_lease_records if dhcp_lease_record['mac_address']==registration_record['mac_address']))
             dhcp_name = dhcp_lease_record.get('host_name')
             dhcp_comment = dhcp_lease_record.get('comment')
 
             if dhcp_name and dhcp_comment:
-                dhcp_name = f'{dhcp_name[0:20]} ({dhcp_comment[0:20]})' if not router_metric.router_entry.use_comments_over_names else dhcp_comment
+                dhcp_name = f'{dhcp_name[0:20]} ({dhcp_comment[0:20]})' if not router_entry.config_entry.use_comments_over_names else dhcp_comment
             elif dhcp_comment:
                 dhcp_name = dhcp_comment
             else:
@@ -49,8 +50,11 @@ class BaseOutputProcessor:
             registration_record['rx_bytes'] = registration_record['bytes'].split(',')[1]
             del registration_record['bytes']
 
+        if registration_record.get('tx_rate'):
             registration_record['tx_rate'] = BaseOutputProcessor.parse_rates(registration_record['tx_rate'])
+        if registration_record.get('rx_rate'):
             registration_record['rx_rate'] = BaseOutputProcessor.parse_rates(registration_record['rx_rate'])
+        if registration_record.get('uptime'):
             registration_record['uptime'] = naturaldelta(BaseOutputProcessor.parse_timedelta_seconds(registration_record['uptime']), months=True, minimum_unit='seconds', when=None)
 
         if registration_record.get('signal_strength'):

mktxp/cli/output/capsman_out.py

@@ -11,27 +11,31 @@
 ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 ## GNU General Public License for more details.
 
+
 from tabulate import tabulate   
 from mktxp.cli.output.base_out import BaseOutputProcessor
+from mktxp.datasources.dhcp_ds import DHCPMetricsDataSource
+from mktxp.datasources.capsman_ds import CapsmanRegistrationsMetricsDataSource
+
 
 class CapsmanOutput:
     ''' CAPsMAN CLI Output
     '''    
     @staticmethod
-    def clients_summary(router_metric):
+    def clients_summary(router_entry):
         registration_labels = ['interface', 'ssid', 'mac_address', 'rx_signal', 'uptime', 'tx_rate', 'rx_rate']
-        registration_records = router_metric.capsman_registration_table_records(registration_labels, False)
+        registration_records = CapsmanRegistrationsMetricsDataSource.metric_records(router_entry, metric_labels = registration_labels, add_router_id = False)
         if not registration_records:
             print('No CAPsMAN registration records')
             return 
 
         # translate / trim / augment registration records
         dhcp_lease_labels = ['host_name', 'comment', 'address', 'mac_address']
-        dhcp_lease_records = router_metric.dhcp_lease_records(dhcp_lease_labels, False)
+        dhcp_lease_records = DHCPMetricsDataSource.metric_records(router_entry, metric_labels = dhcp_lease_labels, add_router_id = False)   
 
         dhcp_rt_by_interface = {}
         for registration_record in sorted(registration_records, key = lambda rt_record: rt_record['rx_signal'], reverse=True):
-            BaseOutputProcessor.augment_record(router_metric, registration_record, dhcp_lease_records)
+            BaseOutputProcessor.augment_record(router_entry, registration_record, dhcp_lease_records)
 
             interface = registration_record['interface']
             if interface in dhcp_rt_by_interface.keys():

mktxp/cli/output/dhcp_out.py

@@ -0,0 +1,48 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from tabulate import tabulate   
+from mktxp.cli.output.base_out import BaseOutputProcessor
+from mktxp.datasources.dhcp_ds import DHCPMetricsDataSource
+
+
+class DHCPOutput:
+    ''' DHCP Clients CLI Output
+    '''    
+    @staticmethod
+    def clients_summary(router_entry):
+        dhcp_lease_labels = ['host_name', 'comment', 'active_address', 'address', 'mac_address', 'server', 'expires_after']
+        dhcp_lease_records = DHCPMetricsDataSource.metric_records(router_entry, metric_labels = dhcp_lease_labels, add_router_id = False)
+        if not dhcp_lease_records:
+            print('No DHCP registration records')
+            return 
+
+        dhcp_by_server = {}
+        for dhcp_lease_record in sorted(dhcp_lease_records, key = lambda dhcp_record: dhcp_record['active_address'], reverse=True):
+            server = dhcp_lease_record['server']
+            if server in dhcp_by_server.keys():
+                dhcp_by_server[server].append(dhcp_lease_record)
+            else:
+                dhcp_by_server[server] = [dhcp_lease_record]         
+
+        num_records = 0
+        output_table = []
+        for key in dhcp_by_server.keys():
+            for record in dhcp_by_server[key]:
+                output_table.append(BaseOutputProcessor.OutputDHCPEntry(**record))
+                num_records += 1
+            output_table.append({})
+        print()
+        print(tabulate(output_table, headers = "keys",  tablefmt="github"))
+        print(tabulate([{0:'DHCP Clients:', 1:num_records}], tablefmt="text"))

mktxp/cli/output/wifi_out.py

@@ -11,27 +11,31 @@
 ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 ## GNU General Public License for more details.
 
+
 from tabulate import tabulate   
 from mktxp.cli.output.base_out import BaseOutputProcessor
+from mktxp.datasources.dhcp_ds import DHCPMetricsDataSource
+from mktxp.datasources.wireless_ds import WirelessMetricsDataSource
+
 
 class WirelessOutput:
     ''' Wireless Clients CLI Output
     '''    
     @staticmethod
-    def clients_summary(router_metric):
+    def clients_summary(router_entry):
         registration_labels = ['interface', 'mac_address', 'signal_strength', 'uptime', 'tx_rate', 'rx_rate', 'signal_to_noise']
-        registration_records = router_metric.wireless_registration_table_records(registration_labels, False)
+        registration_records = WirelessMetricsDataSource.metric_records(router_entry, metric_labels = registration_labels, add_router_id = False)
         if not registration_records:
             print('No wireless registration records')
             return 
 
         # translate / trim / augment registration records
         dhcp_lease_labels = ['host_name', 'comment', 'address', 'mac_address']
-        dhcp_lease_records = router_metric.dhcp_lease_records(dhcp_lease_labels, False)
+        dhcp_lease_records = DHCPMetricsDataSource.metric_records(router_entry, metric_labels = dhcp_lease_labels, add_router_id = False)   
 
         dhcp_rt_by_interface = {}
         for registration_record in sorted(registration_records, key = lambda rt_record: rt_record['signal_strength'], reverse=True):
-            BaseOutputProcessor.augment_record(router_metric, registration_record, dhcp_lease_records)
+            BaseOutputProcessor.augment_record(router_entry, registration_record, dhcp_lease_records)
 
             interface = registration_record['interface']
             if interface in dhcp_rt_by_interface.keys():

mktxp/collectors/bandwidth_collector.py

@@ -12,16 +12,19 @@
 ## GNU General Public License for more details.
 
 
+import socket
 import speedtest
 from datetime import datetime
 from multiprocessing import Pool
 from mktxp.cli.config.config import config_handler
 from mktxp.collectors.base_collector import BaseCollector
 
+
 result_list = [{'download': 0, 'upload': 0, 'ping': 0}]
 def get_result(bandwidth_dict):
     result_list[0] = bandwidth_dict
 
+
 class BandwidthCollector(BaseCollector):
     ''' MKTXP collector
     '''    
@@ -53,8 +56,21 @@ class BandwidthCollector(BaseCollector):
 
     @staticmethod
     def bandwidth_worker():
+        if BandwidthCollector.inet_connected():
             bandwidth_test = speedtest.Speedtest()
             bandwidth_test.get_best_server()
             bandwidth_test.download()
             bandwidth_test.upload()
             return bandwidth_test.results.dict()
+        else:
+            return {'download': 0, 'upload': 0, 'ping': 0}
+
+    @staticmethod
+    def inet_connected(host="8.8.8.8", port=53, timeout=3):
+        try:
+            socket.setdefaulttimeout(timeout)
+            socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((host, port))
+            return True
+        except socket.error as exc:
+            return False
+

mktxp/collectors/base_collector.py

@@ -15,6 +15,7 @@
 from prometheus_client.core import GaugeMetricFamily, CounterMetricFamily, InfoMetricFamily
 from mktxp.cli.config.config import MKTXPConfigKeys
      
+     
 class BaseCollector:
     ''' Base Collector methods
         For use by custom collectors

mktxp/collectors/capsman_collector.py

@@ -11,43 +11,47 @@
 ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 ## GNU General Public License for more details.
 
+
 from mktxp.cli.output.base_out import BaseOutputProcessor
 from mktxp.cli.config.config import MKTXPConfigKeys
 from mktxp.collectors.base_collector import BaseCollector
+from mktxp.datasources.dhcp_ds import DHCPMetricsDataSource
+from mktxp.datasources.capsman_ds import CapsmanCapsMetricsDataSource, CapsmanRegistrationsMetricsDataSource
+
 
 class CapsmanCollector(BaseCollector):
     ''' CAPsMAN Metrics collector
     '''    
     @staticmethod
-    def collect(router_metric):
+    def collect(router_entry):
         remote_caps_labels = ['identity', 'version', 'base_mac', 'board', 'base_mac']
-        remote_caps_records = router_metric.capsman_remote_caps_records(remote_caps_labels)
+        remote_caps_records = CapsmanCapsMetricsDataSource.metric_records(router_entry, metric_labels = remote_caps_labels)
         if remote_caps_records:
             remote_caps_metrics = BaseCollector.info_collector('capsman_remote_caps', 'CAPsMAN remote caps', remote_caps_records, remote_caps_labels)
             yield remote_caps_metrics
 
         registration_labels = ['interface', 'ssid', 'mac_address', 'tx_rate', 'rx_rate', 'rx_signal', 'uptime', 'bytes']
-        registration_records = router_metric.capsman_registration_table_records(registration_labels)
+        registration_records = CapsmanRegistrationsMetricsDataSource.metric_records(router_entry, metric_labels = registration_labels)
         if registration_records:
             # calculate number of registrations per interface
             registration_per_interface = {}
             for registration_record in registration_records:
                 registration_per_interface[registration_record['interface']] = registration_per_interface.get(registration_record['interface'], 0) + 1
             # compile registrations-per-interface records
-            registration_per_interface_records = [{ MKTXPConfigKeys.ROUTERBOARD_NAME: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
+            registration_per_interface_records = [{ MKTXPConfigKeys.ROUTERBOARD_NAME: router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
-                                            MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
+                                            MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
                                             'interface': key, 'count': value} for key, value in registration_per_interface.items()]
             # yield registrations-per-interface metrics
             registration_per_interface_metrics = BaseCollector.gauge_collector('capsman_registrations_count', 'Number of active registration per CAPsMAN interface', registration_per_interface_records, 'count', ['interface'])
             yield registration_per_interface_metrics
 
             # the client info metrics
-            if router_metric.router_entry.capsman_clients:
+            if router_entry.config_entry.capsman_clients:
                 # translate / trim / augment registration records
                 dhcp_lease_labels = ['mac_address', 'address', 'host_name', 'comment']
-                dhcp_lease_records = router_metric.dhcp_lease_records(dhcp_lease_labels)
+                dhcp_lease_records = DHCPMetricsDataSource.metric_records(router_entry, metric_labels = dhcp_lease_labels)
                 for registration_record in registration_records:
-                    BaseOutputProcessor.augment_record(router_metric, registration_record, dhcp_lease_records)
+                    BaseOutputProcessor.augment_record(router_entry, registration_record, dhcp_lease_records)
                     
                 tx_byte_metrics = BaseCollector.counter_collector('capsman_clients_tx_bytes', 'Number of sent packet bytes', registration_records, 'tx_bytes', ['dhcp_name'])
                 yield tx_byte_metrics

mktxp/collectors/dhcp_collector.py

@@ -11,17 +11,19 @@
 ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 ## GNU General Public License for more details.
 
+
 from mktxp.cli.config.config import MKTXPConfigKeys
 from mktxp.collectors.base_collector import BaseCollector
+from mktxp.datasources.dhcp_ds import DHCPMetricsDataSource
 
 
 class DHCPCollector(BaseCollector):
     ''' DHCP Metrics collector
     '''    
     @staticmethod
-    def collect(router_metric):
+    def collect(router_entry):
-        dhcp_lease_labels = ['active_address', 'mac_address', 'host_name', 'comment', 'server', 'expires_after']
+        dhcp_lease_labels = ['active_address', 'address', 'mac_address', 'host_name', 'comment', 'server', 'expires_after']
-        dhcp_lease_records = router_metric.dhcp_lease_records(dhcp_lease_labels)
+        dhcp_lease_records = DHCPMetricsDataSource.metric_records(router_entry, metric_labels = dhcp_lease_labels)   
         if dhcp_lease_records:
             # calculate number of leases per DHCP server
             dhcp_lease_servers = {}
@@ -29,8 +31,8 @@ class DHCPCollector(BaseCollector):
                 dhcp_lease_servers[dhcp_lease_record['server']] = dhcp_lease_servers.get(dhcp_lease_record['server'], 0) + 1
 
             # compile leases-per-server records
-            dhcp_lease_servers_records = [{ MKTXPConfigKeys.ROUTERBOARD_NAME: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
+            dhcp_lease_servers_records = [{ MKTXPConfigKeys.ROUTERBOARD_NAME: router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
-                                            MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
+                                            MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
                                             'server': key, 'count': value} for key, value in dhcp_lease_servers.items()]
             
             # yield lease-per-server metrics
@@ -38,7 +40,7 @@ class DHCPCollector(BaseCollector):
             yield dhcp_lease_server_metrics
 
             # active lease metrics
-            if router_metric.router_entry.dhcp_lease:
+            if router_entry.config_entry.dhcp_lease:
                 dhcp_lease_metrics = BaseCollector.info_collector('dhcp_lease', 'DHCP Active Leases', dhcp_lease_records, dhcp_lease_labels)
                 yield dhcp_lease_metrics
             

mktxp/collectors/firewall_collector.py

@@ -11,34 +11,37 @@
 ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 ## GNU General Public License for more details.
 
-from mktxp.collectors.base_collector import BaseCollector
+
 from mktxp.cli.config.config import MKTXPConfigKeys
+from mktxp.collectors.base_collector import BaseCollector
+from mktxp.datasources.firewall_ds import FirewallMetricsDataSource
+
 
 class FirewallCollector(BaseCollector):
     ''' Firewall rules traffic metrics collector
     '''    
     @staticmethod
-    def collect(router_metric):
+    def collect(router_entry):
         # initialize all pool counts, including those currently not used
         firewall_labels = ['chain', 'action', 'bytes', 'comment']
         
-        firewall_filter_records = router_metric.firewall_records(firewall_labels)   
+        firewall_filter_records = FirewallMetricsDataSource.metric_records(router_entry, metric_labels = firewall_labels)   
         if firewall_filter_records:           
-            metris_records = [FirewallCollector.metric_record(router_metric, record) for record in firewall_filter_records]
+            metris_records = [FirewallCollector.metric_record(router_entry, record) for record in firewall_filter_records]
             firewall_filter_metrics = BaseCollector.counter_collector('firewall_filter', 'Total amount of bytes matched by firewall rules', metris_records, 'bytes', ['name'])
             yield firewall_filter_metrics
 
-        firewall_raw_records = router_metric.firewall_records(firewall_labels, raw = True)        
+        firewall_raw_records = FirewallMetricsDataSource.metric_records(router_entry, metric_labels = firewall_labels, raw = True)        
         if firewall_raw_records:      
-            metris_records = [FirewallCollector.metric_record(router_metric, record) for record in firewall_raw_records]     
+            metris_records = [FirewallCollector.metric_record(router_entry, record) for record in firewall_raw_records]     
             firewall_raw_metrics = BaseCollector.counter_collector('firewall_raw', 'Total amount of bytes matched by raw firewall rules', metris_records, 'bytes', ['name'])
             yield firewall_raw_metrics
 
     # Helpers
     @staticmethod
-    def metric_record(router_metric, firewall_record):
+    def metric_record(router_entry, firewall_record):
         name = f"| {firewall_record['chain']} | {firewall_record['action']} | {firewall_record['comment']}"
         bytes = firewall_record['bytes']
-        return {MKTXPConfigKeys.ROUTERBOARD_NAME: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
+        return {MKTXPConfigKeys.ROUTERBOARD_NAME: router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
-                MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
+                MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
                 'name': name, 'bytes': bytes}

mktxp/collectors/health_collector.py

@@ -11,16 +11,18 @@
 ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 ## GNU General Public License for more details.
 
+
 from mktxp.collectors.base_collector import BaseCollector
+from mktxp.datasources.health_ds import HealthMetricsDataSource
 
 
 class HealthCollector(BaseCollector):
     ''' System Health Metrics collector
     '''    
     @staticmethod
-    def collect(router_metric):
+    def collect(router_entry):
         health_labels = ['voltage', 'temperature']
-        health_records = router_metric.health_records(health_labels)        
+        health_records = HealthMetricsDataSource.metric_records(router_entry, metric_labels = health_labels)   
         if health_records:
             voltage_metrics = BaseCollector.gauge_collector('system_routerboard_voltage', 'Supplied routerboard voltage', health_records, 'voltage')
             yield voltage_metrics

mktxp/collectors/identity_collector.py

@@ -11,15 +11,18 @@
 ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 ## GNU General Public License for more details.
 
+
 from mktxp.collectors.base_collector import BaseCollector
+from mktxp.datasources.identity_ds import IdentityMetricsDataSource
+
 
 class IdentityCollector(BaseCollector):
     ''' System Identity Metrics collector
     '''     
     @staticmethod
-    def collect(router_metric):
+    def collect(router_entry):
         identity_labels = ['name']
-        identity_records = router_metric.identity_records(identity_labels)        
+        identity_records = IdentityMetricsDataSource.metric_records(router_entry, metric_labels = identity_labels)                
         if identity_records:
             identity_metrics = BaseCollector.info_collector('system_identity', 'System identity', identity_records, identity_labels)
             yield identity_metrics

mktxp/collectors/interface_collector.py

@@ -11,20 +11,23 @@
 ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 ## GNU General Public License for more details.
 
+
 from mktxp.collectors.base_collector import BaseCollector
+from mktxp.datasources.interface_ds import InterfaceTrafficMetricsDataSource
+
 
 class InterfaceCollector(BaseCollector):
     ''' Router Interface Metrics collector
     '''        
     @staticmethod
-    def collect(router_metric):
+    def collect(router_entry):
         interface_traffic_labels = ['name', 'comment', 'rx_byte', 'tx_byte', 'rx_packet', 'tx_packet', 'rx_error', 'tx_error', 'rx_drop', 'tx_drop']
-        interface_traffic_records = router_metric.interface_traffic_records(interface_traffic_labels)
+        interface_traffic_records = InterfaceTrafficMetricsDataSource.metric_records(router_entry, metric_labels = interface_traffic_labels)   
 
         if interface_traffic_records:
             for interface_traffic_record in interface_traffic_records:
                 if interface_traffic_record.get('comment'):
-                    interface_traffic_record['name'] = interface_traffic_record['comment'] if router_metric.router_entry.use_comments_over_names \
+                    interface_traffic_record['name'] = interface_traffic_record['comment'] if router_entry.config_entry.use_comments_over_names \
                                                                                 else f"{interface_traffic_record['name']} ({interface_traffic_record['comment']})"
 
             rx_byte_metric = BaseCollector.counter_collector('interface_rx_byte', 'Number of received bytes', interface_traffic_records, 'rx_byte', ['name'])

mktxp/collectors/mktxp_collector.py

@@ -11,14 +11,17 @@
 ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 ## GNU General Public License for more details.
 
+
 from mktxp.collectors.base_collector import BaseCollector
+from mktxp.datasources.mktxp_ds import MKTXPMetricsDataSource
+
 
 class MKTXPCollector(BaseCollector):
     ''' System Identity Metrics collector
     '''     
     @staticmethod
-    def collect(router_metric):
+    def collect(router_entry):
-        mktxp_records = router_metric.mktxp_records()        
+        mktxp_records = MKTXPMetricsDataSource.metric_records(router_entry)
         if mktxp_records:
             mktxp_duration_metric = BaseCollector.counter_collector('collection_time', 'Total time spent collecting metrics in milliseconds', mktxp_records, 'duration', ['name'])
             yield mktxp_duration_metric

mktxp/collectors/monitor_collector.py

@@ -11,16 +11,19 @@
 ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 ## GNU General Public License for more details.
 
+
 from mktxp.collectors.base_collector import BaseCollector
 from mktxp.cli.output.base_out import BaseOutputProcessor
+from mktxp.datasources.interface_ds import InterfaceMonitorMetricsDataSource
+
 
 class MonitorCollector(BaseCollector):
     ''' Ethernet Interface Monitor Metrics collector
     '''    
     @staticmethod
-    def collect(router_metric):
+    def collect(router_entry):
         monitor_labels = ('status', 'rate', 'full_duplex', 'name')
-        monitor_records = router_metric.interface_monitor_records(monitor_labels, include_comments = True)
+        monitor_records = InterfaceMonitorMetricsDataSource.metric_records(router_entry, metric_labels = monitor_labels, include_comments = True)   
         if monitor_records:
             # translate records to appropriate values
             for monitor_record in monitor_records:

mktxp/collectors/pool_collector.py

@@ -11,28 +11,31 @@
 ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 ## GNU General Public License for more details.
 
+
 from mktxp.cli.config.config import MKTXPConfigKeys
 from mktxp.collectors.base_collector import BaseCollector
+from mktxp.datasources.pool_ds import PoolMetricsDataSource, PoolUsedMetricsDataSource
+
 
 class PoolCollector(BaseCollector):
     ''' IP Pool Metrics collector
     '''    
     @staticmethod
-    def collect(router_metric):
+    def collect(router_entry):
         # initialize all pool counts, including those currently not used
-        pool_records = router_metric.pool_records(['name'])
+        pool_records = PoolMetricsDataSource.metric_records(router_entry, metric_labels = ['name'])   
         if pool_records:
             pool_used_labels = ['pool']
             pool_used_counts = {pool_record['name']: 0 for pool_record in pool_records}
 
             # for pools in usage, calculate the current numbers
-            pool_used_records = router_metric.pool_used_records(pool_used_labels)
+            pool_used_records = PoolUsedMetricsDataSource.metric_records(router_entry, metric_labels = pool_used_labels)   
             for pool_used_record in pool_used_records:
                 pool_used_counts[pool_used_record['pool']] = pool_used_counts.get(pool_used_record['pool'], 0) + 1
 
            # compile used-per-pool records
-            used_per_pool_records = [{ MKTXPConfigKeys.ROUTERBOARD_NAME: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
+            used_per_pool_records = [{ MKTXPConfigKeys.ROUTERBOARD_NAME: router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
-                                       MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
+                                       MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
                                        'pool': key, 'count': value} for key, value in pool_used_counts.items()]
             
             # yield used-per-pool metrics

mktxp/collectors/resource_collector.py

@@ -11,20 +11,23 @@
 ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 ## GNU General Public License for more details.
 
+
 from mktxp.collectors.base_collector import BaseCollector
 from mktxp.cli.output.base_out import BaseOutputProcessor
+from mktxp.datasources.system_resource_ds import SystemResourceMetricsDataSource
+
 
 class SystemResourceCollector(BaseCollector):
     ''' System Resource Metrics collector
     '''        
     @staticmethod
-    def collect(router_metric):
+    def collect(router_entry):
         resource_labels = ['uptime', 'version', 'free_memory', 'total_memory', 
                            'cpu', 'cpu_count', 'cpu_frequency', 'cpu_load', 
                            'free_hdd_space', 'total_hdd_space', 
                            'architecture_name', 'board_name']
                                    
-        resource_records = router_metric.system_resource_records(resource_labels)
+        resource_records = SystemResourceMetricsDataSource.metric_records(router_entry, metric_labels = resource_labels)   
         if resource_records:
             # translate records to appropriate values
             translated_fields = ['uptime']        

mktxp/collectors/route_collector.py

@@ -11,21 +11,24 @@
 ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 ## GNU General Public License for more details.
 
+
 from mktxp.cli.config.config import MKTXPConfigKeys
 from mktxp.collectors.base_collector import BaseCollector
+from mktxp.datasources.route_ds import RouteMetricsDataSource
+
 
 class RouteCollector(BaseCollector):
     ''' IP Route Metrics collector
     '''        
     @staticmethod
-    def collect(router_metric):
+    def collect(router_entry):
         route_labels = ['connect', 'dynamic', 'static', 'bgp', 'ospf']
-        route_records = router_metric.route_records(route_labels)
+        route_records = RouteMetricsDataSource.metric_records(router_entry, metric_labels = route_labels)   
         if route_records:       
             # compile total routes records
             total_routes = len(route_records)
-            total_routes_records = [{ MKTXPConfigKeys.ROUTERBOARD_NAME: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
+            total_routes_records = [{ MKTXPConfigKeys.ROUTERBOARD_NAME: router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
-                                      MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
+                                      MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
                                       'count': total_routes
                                     }]
             total_routes_metrics = BaseCollector.gauge_collector('routes_total_routes', 'Overall number of routes in RIB', total_routes_records, 'count')
@@ -40,8 +43,8 @@ class RouteCollector(BaseCollector):
                         routes_per_protocol[route_label] += 1 
 
             # compile route-per-protocol records
-            route_per_protocol_records = [{ MKTXPConfigKeys.ROUTERBOARD_NAME: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
+            route_per_protocol_records = [{ MKTXPConfigKeys.ROUTERBOARD_NAME: router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
-                                            MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
+                                            MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
                                             'protocol': key, 'count': value} for key, value in routes_per_protocol.items()]
             
             # yield route-per-protocol metrics

mktxp/collectors/wlan_collector.py

@@ -11,16 +11,21 @@
 ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 ## GNU General Public License for more details.
 
+
 from mktxp.cli.output.base_out import BaseOutputProcessor
 from mktxp.collectors.base_collector import BaseCollector
+from mktxp.datasources.dhcp_ds import DHCPMetricsDataSource
+from mktxp.datasources.wireless_ds import WirelessMetricsDataSource
+from mktxp.datasources.interface_ds import InterfaceMonitorMetricsDataSource
+
 
 class WLANCollector(BaseCollector):
     ''' Wireless Metrics collector
     '''    
     @staticmethod
-    def collect(router_metric):
+    def collect(router_entry):
         monitor_labels = ['channel', 'noise_floor', 'overall_tx_ccq', 'registered_clients']
-        monitor_records = router_metric.interface_monitor_records(monitor_labels, 'wireless')
+        monitor_records = InterfaceMonitorMetricsDataSource.metric_records(router_entry, metric_labels = monitor_labels, kind = 'wireless')   
         if monitor_records:
             # sanitize records for relevant labels
             noise_floor_records = [monitor_record for monitor_record in monitor_records if monitor_record.get('noise_floor')]
@@ -40,15 +45,15 @@ class WLANCollector(BaseCollector):
                 yield registered_clients_metrics
 
         # the client info metrics
-        if router_metric.router_entry.wireless_clients:
+        if router_entry.config_entry.wireless_clients:
             registration_labels = ['interface', 'ssid', 'mac_address', 'tx_rate', 'rx_rate', 'uptime', 'bytes', 'signal_to_noise', 'tx_ccq', 'signal_strength']
-            registration_records = router_metric.wireless_registration_table_records(registration_labels)
+            registration_records = WirelessMetricsDataSource.metric_records(router_entry, metric_labels = registration_labels)
             if registration_records:
                 dhcp_lease_labels = ['mac_address', 'address', 'host_name', 'comment']
-                dhcp_lease_records = router_metric.dhcp_lease_records(dhcp_lease_labels)
+                dhcp_lease_records = DHCPMetricsDataSource.metric_records(router_entry, metric_labels = dhcp_lease_labels)
       
                 for registration_record in registration_records:
-                    BaseOutputProcessor.augment_record(router_metric, registration_record, dhcp_lease_records)                
+                    BaseOutputProcessor.augment_record(router_entry, registration_record, dhcp_lease_records)                
 
                 tx_byte_metrics = BaseCollector.counter_collector('wlan_clients_tx_bytes', 'Number of sent packet bytes', registration_records, 'tx_bytes', ['dhcp_name'])
                 yield tx_byte_metrics

mktxp/collectors_handler.py

@@ -11,6 +11,7 @@
 ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 ## GNU General Public License for more details.
 
+
 from timeit import default_timer
 from mktxp.collectors.dhcp_collector import DHCPCollector
 from mktxp.collectors.interface_collector import InterfaceCollector
@@ -30,73 +31,73 @@ from mktxp.collectors.mktxp_collector import MKTXPCollector
 class CollectorsHandler:
     ''' MKTXP Collectors Handler
     '''
-    def __init__(self, metrics_handler):
+    def __init__(self, entries_handler):
-        self.metrics_handler = metrics_handler
+        self.entries_handler = entries_handler
         self.bandwidthCollector = BandwidthCollector()
 
     def collect(self):
         # process mktxp internal metrics
         yield from self.bandwidthCollector.collect()
 
-        for router_metric in self.metrics_handler.router_metrics:           
+        for router_entry in self.entries_handler.router_entries:
-            if not router_metric.api_connection.is_connected():
+            if not router_entry.api_connection.is_connected():
                 # let's pick up on things in the next run
-                router_metric.api_connection.connect()
+                router_entry.api_connection.connect()
                 continue
 
             start = default_timer()
-            yield from IdentityCollector.collect(router_metric)
+            yield from IdentityCollector.collect(router_entry)
-            router_metric.time_spent['IdentityCollector'] += default_timer() - start
+            router_entry.time_spent['IdentityCollector'] += default_timer() - start
 
             start = default_timer()
-            yield from SystemResourceCollector.collect(router_metric)
+            yield from SystemResourceCollector.collect(router_entry)
-            router_metric.time_spent['SystemResourceCollector'] += default_timer() - start
+            router_entry.time_spent['SystemResourceCollector'] += default_timer() - start
 
             start = default_timer()
-            yield from HealthCollector.collect(router_metric)
+            yield from HealthCollector.collect(router_entry)
-            router_metric.time_spent['HealthCollector'] += default_timer() - start
+            router_entry.time_spent['HealthCollector'] += default_timer() - start
 
-            if router_metric.router_entry.dhcp:
+            if router_entry.config_entry.dhcp:
                 start = default_timer()
-                yield from DHCPCollector.collect(router_metric)                
+                yield from DHCPCollector.collect(router_entry)                
-                router_metric.time_spent['DHCPCollector'] += default_timer() - start
+                router_entry.time_spent['DHCPCollector'] += default_timer() - start
 
-            if router_metric.router_entry.pool:
+            if router_entry.config_entry.pool:
                 start = default_timer()
-                yield from PoolCollector.collect(router_metric)
+                yield from PoolCollector.collect(router_entry)
-                router_metric.time_spent['PoolCollector'] += default_timer() - start
+                router_entry.time_spent['PoolCollector'] += default_timer() - start
             
-            if router_metric.router_entry.interface:
+            if router_entry.config_entry.interface:
                 start = default_timer()
-                yield from InterfaceCollector.collect(router_metric)
+                yield from InterfaceCollector.collect(router_entry)
-                router_metric.time_spent['InterfaceCollector'] += default_timer() - start
+                router_entry.time_spent['InterfaceCollector'] += default_timer() - start
 
-            if router_metric.router_entry.firewall:
+            if router_entry.config_entry.firewall:
                 start = default_timer()
-                yield from FirewallCollector.collect(router_metric)
+                yield from FirewallCollector.collect(router_entry)
-                router_metric.time_spent['FirewallCollector'] += default_timer() - start
+                router_entry.time_spent['FirewallCollector'] += default_timer() - start
             
-            if router_metric.router_entry.monitor:
+            if router_entry.config_entry.monitor:
                 start = default_timer()
-                yield from MonitorCollector.collect(router_metric)
+                yield from MonitorCollector.collect(router_entry)
-                router_metric.time_spent['MonitorCollector'] += default_timer() - start
+                router_entry.time_spent['MonitorCollector'] += default_timer() - start
             
-            if router_metric.router_entry.route:
+            if router_entry.config_entry.route:
                 start = default_timer()
-                yield from RouteCollector.collect(router_metric)
+                yield from RouteCollector.collect(router_entry)
-                router_metric.time_spent['RouteCollector'] += default_timer() - start
+                router_entry.time_spent['RouteCollector'] += default_timer() - start
        
-            if router_metric.router_entry.wireless:
+            if router_entry.config_entry.wireless:
                 start = default_timer()
-                yield from WLANCollector.collect(router_metric)
+                yield from WLANCollector.collect(router_entry)
-                router_metric.time_spent['WLANCollector'] += default_timer() - start
+                router_entry.time_spent['WLANCollector'] += default_timer() - start
 
-            if router_metric.router_entry.capsman:
+            if router_entry.config_entry.capsman:
                 start = default_timer()
-                yield from CapsmanCollector.collect(router_metric)
+                yield from CapsmanCollector.collect(router_entry)
-                router_metric.time_spent['CapsmanCollector'] += default_timer() - start
+                router_entry.time_spent['CapsmanCollector'] += default_timer() - start
             
-            yield from MKTXPCollector.collect(router_metric)
+            yield from MKTXPCollector.collect(router_entry)
 
 
 

mktxp/datasources/base_ds.py

@@ -0,0 +1,39 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+class BaseDSProcessor:
+    ''' Base Metrics DataSource processing
+    '''             
+
+    @staticmethod
+    def trimmed_records(router_entry, *, router_records = [], metric_labels = [], add_router_id = True, translation_table = {}):
+        if len(metric_labels) == 0 and len(router_records) > 0:
+            metric_labels = router_records[0].keys()
+        metric_labels = set(metric_labels)      
+
+        labeled_records = []
+        dash2_ = lambda x : x.replace('-', '_')
+        for router_record in router_records:
+            translated_record = {dash2_(key): value for (key, value) in router_record.items() if dash2_(key) in metric_labels}
+
+            if add_router_id:
+                for key, value in router_entry.router_id.items():
+                    translated_record[key] = value
+            
+            # translate fields if needed
+            for key, func in translation_table.items():
+                translated_record[key] = func(translated_record.get(key))
+            labeled_records.append(translated_record)
+            
+        return labeled_records

mktxp/datasources/capsman_ds.py

@@ -0,0 +1,41 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from mktxp.datasources.base_ds import BaseDSProcessor
+
+
+class CapsmanCapsMetricsDataSource:
+    ''' Caps Metrics data provider
+    '''             
+    @staticmethod
+    def metric_records(router_entry, *, metric_labels = []):
+        try:
+            remote_caps_records = router_entry.api_connection.router_api().get_resource('/caps-man/remote-cap').get()
+            return BaseDSProcessor.trimmed_records(router_entry, router_records = remote_caps_records, metric_labels = metric_labels)
+        except Exception as exc:
+            print(f'Error getting caps-man remote caps info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}')
+            return None
+
+
+class CapsmanRegistrationsMetricsDataSource:
+    ''' Capsman Registrations Metrics data provider
+    '''             
+    @staticmethod
+    def metric_records(router_entry, *, metric_labels = [],  add_router_id = True):
+        try:
+            registration_table_records = router_entry.api_connection.router_api().get_resource('/caps-man/registration-table').get()
+            return BaseDSProcessor.trimmed_records(router_entry, router_records = registration_table_records, metric_labels = metric_labels, add_router_id = add_router_id)
+        except Exception as exc:
+            print(f'Error getting caps-man registration table info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}')
+            return None

mktxp/datasources/dhcp_ds.py

@@ -0,0 +1,43 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from mktxp.datasources.base_ds import BaseDSProcessor
+
+
+class DHCPMetricsDataSource:
+    ''' DHCP Metrics data provider
+    '''             
+    @staticmethod
+    def metric_records(router_entry, *, metric_labels = [], add_router_id = True):
+        try:
+            #dhcp_lease_records = router_entry.api_connection.router_api().get_resource('/ip/dhcp-server/lease').get(status='bound')
+            dhcp_lease_records = router_entry.api_connection.router_api().get_resource('/ip/dhcp-server/lease').call('print', {'active':''})
+
+            # translation rules
+            translation_table = {}
+            if 'comment' in metric_labels:
+                translation_table['comment'] = lambda c: c if c else ''           
+            if 'host_name' in metric_labels:
+                translation_table['host_name'] = lambda c: c if c else ''           
+            if 'expires_after' in metric_labels:
+                translation_table['expires_after'] = lambda c: c if c else ''        
+            if 'active_address' in metric_labels:
+                translation_table['active_address'] = lambda c: c if c else ''        
+
+            return BaseDSProcessor.trimmed_records(router_entry, router_records = dhcp_lease_records, metric_labels = metric_labels, add_router_id = add_router_id, translation_table = translation_table)
+        except Exception as exc:
+            print(f'Error getting dhcp info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}')
+            return None
+
+

mktxp/datasources/firewall_ds.py

@@ -0,0 +1,39 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from mktxp.datasources.base_ds import BaseDSProcessor
+
+
+class FirewallMetricsDataSource:
+    ''' Firewall Metrics data provider
+    '''             
+    @staticmethod
+    def metric_records(router_entry, *, metric_labels = [], raw = False, matching_only = True):
+        try:
+            filter_path = '/ip/firewall/filter' if not raw else '/ip/firewall/raw'
+            firewall_records = router_entry.api_connection.router_api().get_resource(filter_path).call('print', {'stats':'', 'all':''})
+            if matching_only:
+                firewall_records = [record for record in firewall_records if int(record.get('bytes', '0')) > 0]
+
+            # translation rules
+            translation_table = {}
+            if 'comment' in metric_labels:
+                translation_table['comment'] = lambda c: c if c else ''           
+
+            return BaseDSProcessor.trimmed_records(router_entry, router_records = firewall_records, metric_labels = metric_labels, translation_table = translation_table)
+        except Exception as exc:
+            print(f'Error getting firewall filters info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}')
+            return None
+
+

mktxp/datasources/health_ds.py

@@ -0,0 +1,28 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from mktxp.datasources.base_ds import BaseDSProcessor
+
+
+class HealthMetricsDataSource:
+    ''' Health Metrics data provider
+    '''             
+    @staticmethod
+    def metric_records(router_entry, *, metric_labels = []):
+        try:
+            health_records = router_entry.api_connection.router_api().get_resource('/system/health').get()
+            return BaseDSProcessor.trimmed_records(router_entry, router_records = health_records, metric_labels = metric_labels)
+        except Exception as exc:
+            print(f'Error getting system health info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}')
+            return None

mktxp/datasources/identity_ds.py

@@ -0,0 +1,28 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from mktxp.datasources.base_ds import BaseDSProcessor
+
+
+class IdentityMetricsDataSource:
+    ''' Identity Metrics data provider
+    '''             
+    @staticmethod
+    def metric_records(router_entry, *, metric_labels = []):
+        try:
+            identity_records = router_entry.api_connection.router_api().get_resource('/system/identity').get()
+            return BaseDSProcessor.trimmed_records(router_entry, router_records = identity_records, metric_labels = metric_labels)
+        except Exception as exc:
+            print(f'Error getting system identity info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}')
+            return None

mktxp/datasources/interface_ds.py

@@ -0,0 +1,59 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from mktxp.datasources.base_ds import BaseDSProcessor
+
+
+class InterfaceTrafficMetricsDataSource:
+    ''' Interface Traffic Metrics data provider
+    '''             
+    @staticmethod
+    def metric_records(router_entry, *, metric_labels = []):
+        try:
+            traffic_records = router_entry.api_connection.router_api().get_resource('/interface').get(running='yes', disabled='no')
+            return BaseDSProcessor.trimmed_records(router_entry, router_records = traffic_records, metric_labels = metric_labels)
+        except Exception as exc:
+            print(f'Error getting interface traffic info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}')
+            return None
+
+
+class InterfaceMonitorMetricsDataSource:
+    ''' Interface Monitor Metrics data provider
+    '''             
+    @staticmethod
+    def metric_records(router_entry, *, metric_labels = [], kind = 'ethernet', include_comments = False, running_only = True):
+        try:
+            interfaces = router_entry.api_connection.router_api().get_resource(f'/interface/{kind}').get()
+            interface_names = [(interface['name'], interface.get('comment'), interface.get('running')) for interface in interfaces]
+
+            interface_monitor_records = []
+            for int_num, interface_name in enumerate(interface_names):
+                interface_monitor_record = {}
+                if not running_only or interface_name[2] == 'true':
+                    interface_monitor_record = router_entry.api_connection.router_api().get_resource(f'/interface/{kind}').call('monitor', {'once':'', 'numbers':f'{int_num}'})[0]
+                else:
+                    # unless explicitly requested, no need to do a monitor call for not running interfaces                    
+                    interface_monitor_record = {'name': interface_name[0], 'status': 'no-link'}
+
+                if include_comments and interface_name[1]:
+                    # combines names with comments
+                    interface_monitor_record['name'] = interface_name[1] if router_entry.config_entry.use_comments_over_names else \
+                                                                                                        f"{interface_name[0]} ({interface_name[1]})"                                
+                interface_monitor_records.append(interface_monitor_record)
+                
+            return BaseDSProcessor.trimmed_records(router_entry, router_records = interface_monitor_records, metric_labels = metric_labels)
+        except Exception as exc:
+            print(f'Error getting {kind} interface monitor info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}')
+            return None
+

mktxp/datasources/mktxp_ds.py

@@ -0,0 +1,29 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from mktxp.datasources.base_ds import BaseDSProcessor
+
+
+class MKTXPMetricsDataSource:
+    ''' MKTXP Metrics data provider
+    '''             
+    @staticmethod
+    def metric_records(router_entry):
+        mktxp_records = []
+        for key in router_entry.time_spent.keys():
+            mktxp_records.append({'name': key, 'duration': router_entry.time_spent[key]})           
+
+        # translation rules            
+        translation_table = {'duration': lambda d: d*1000}
+        return BaseDSProcessor.trimmed_records(router_entry, router_records = mktxp_records, translation_table = translation_table)

mktxp/datasources/pool_ds.py

@@ -0,0 +1,41 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from mktxp.datasources.base_ds import BaseDSProcessor
+
+
+class PoolMetricsDataSource:
+    ''' Pool Metrics data provider
+    '''             
+    @staticmethod
+    def metric_records(router_entry, *, metric_labels = []):
+        try:
+            pool_records = router_entry.api_connection.router_api().get_resource('/ip/pool').get()
+            return BaseDSProcessor.trimmed_records(router_entry, router_records = pool_records, metric_labels = metric_labels)
+        except Exception as exc:
+            print(f'Error getting pool info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}')
+            return None
+
+
+class PoolUsedMetricsDataSource:
+    ''' Pool/Used Metrics data provider
+    '''             
+    @staticmethod
+    def metric_records(router_entry, *, metric_labels = []):
+        try:
+            pool_used_records = router_entry.api_connection.router_api().get_resource('/ip/pool/used').get()
+            return BaseDSProcessor.trimmed_records(router_entry, router_records = pool_used_records, metric_labels = metric_labels)
+        except Exception as exc:
+            print(f'Error getting pool used info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}')
+            return None

mktxp/datasources/route_ds.py

@@ -0,0 +1,28 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from mktxp.datasources.base_ds import BaseDSProcessor
+
+
+class RouteMetricsDataSource:
+    ''' Routes Metrics data provider
+    '''             
+    @staticmethod
+    def metric_records(router_entry, *, metric_labels = []):
+        try:
+            route_records = router_entry.api_connection.router_api().get_resource('/ip/route').get(active='yes')
+            return BaseDSProcessor.trimmed_records(router_entry, router_records = route_records, metric_labels = metric_labels)
+        except Exception as exc:
+            print(f'Error getting routes info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}')
+            return None

mktxp/datasources/routerboard_ds.py

@@ -0,0 +1,29 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from mktxp.datasources.base_ds import BaseDSProcessor
+
+
+class RouterboardMetricsDataSource:
+    ''' Routerboard Metrics data provider
+    '''             
+    @staticmethod
+    def metric_records(router_entry, *, metric_labels = []):
+        try:
+            routerboard_records = router_entry.api_connection.router_api().get_resource('/system/routerboard').get()
+            return BaseDSProcessor.trimmed_records(router_entry, router_records = routerboard_records, metric_labels = metric_labels)
+        except Exception as exc:
+            print(f'Error getting system routerboard info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}')
+            return None
+

mktxp/datasources/system_resource_ds.py

@@ -0,0 +1,28 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from mktxp.datasources.base_ds import BaseDSProcessor
+
+
+class SystemResourceMetricsDataSource:
+    ''' System Resource Metrics data provider
+    '''             
+    @staticmethod    
+    def metric_records(router_entry, *, metric_labels = []):
+        try:
+            system_resource_records = router_entry.api_connection.router_api().get_resource('/system/resource').get()
+            return BaseDSProcessor.trimmed_records(router_entry, router_records = system_resource_records, metric_labels = metric_labels)
+        except Exception as exc:
+            print(f'Error getting system resource info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}')
+            return None

mktxp/datasources/wireless_ds.py

@@ -0,0 +1,30 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from mktxp.datasources.base_ds import BaseDSProcessor
+
+
+class WirelessMetricsDataSource:
+    ''' Wireless Metrics data provider
+    '''             
+    @staticmethod
+    def metric_records(router_entry, *, metric_labels = [], add_router_id = True):
+        try:
+            registration_table_records = router_entry.api_connection.router_api().get_resource('/interface/wireless/registration-table').get()
+            return BaseDSProcessor.trimmed_records(router_entry, router_records = registration_table_records, metric_labels = metric_labels, add_router_id = add_router_id)
+        except Exception as exc:
+            print(f'Error getting wireless registration table info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}')
+            return None
+
+

mktxp/router_connection.py

@@ -11,12 +11,14 @@
 ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 ## GNU General Public License for more details.
 
+
 import ssl
 import socket
 from datetime import datetime
 from routeros_api import RouterOsApiPool
 from mktxp.cli.config.config import config_handler
 
+
 class RouterAPIConnectionError(Exception):
     pass
 
@@ -24,24 +26,24 @@ class RouterAPIConnectionError(Exception):
 class RouterAPIConnection:
     ''' Base wrapper interface for the routeros_api library
     '''            
-    def __init__(self, router_name, router_entry):
+    def __init__(self, router_name, config_entry):
         self.router_name = router_name
-        self.router_entry = router_entry
+        self.config_entry = config_entry
         self.last_failure_timestamp = self.successive_failure_count = 0
         
         ctx = None
-        if self.router_entry.use_ssl and self.router_entry.no_ssl_certificate:
+        if self.config_entry.use_ssl and self.config_entry.no_ssl_certificate:
             ctx = ssl.create_default_context()
             ctx.set_ciphers('ADH:@SECLEVEL=0')       
 
         self.connection = RouterOsApiPool(
-                host = self.router_entry.hostname,
+                host = self.config_entry.hostname,
-                username = self.router_entry.username,
+                username = self.config_entry.username,
-                password = self.router_entry.password,
+                password = self.config_entry.password,
-                port = self.router_entry.port,
+                port = self.config_entry.port,
                 plaintext_login = True,
-                use_ssl = self.router_entry.use_ssl,
+                use_ssl = self.config_entry.use_ssl,
-                ssl_verify = self.router_entry.ssl_certificate_verify,
+                ssl_verify = self.config_entry.ssl_certificate_verify,
                 ssl_context = ctx)
         
         self.connection.socket_timeout = config_handler._entry().socket_timeout
@@ -62,7 +64,7 @@ class RouterAPIConnection:
         if self.is_connected() or self._in_connect_timeout(connect_time.timestamp()):
             return
         try:
-            print(f'Connecting to router {self.router_name}@{self.router_entry.hostname}')
+            print(f'Connecting to router {self.router_name}@{self.config_entry.hostname}')
             self.api = self.connection.get_api()
             self._set_connect_state(success = True, connect_time = connect_time)
         except (socket.error, socket.timeout, Exception) as exc:
@@ -78,11 +80,11 @@ class RouterAPIConnection:
         connect_delay = self._connect_delay()
         if (connect_timestamp - self.last_failure_timestamp) < connect_delay:
             if not quiet: 
-                print(f'{self.router_name}@{self.router_entry.hostname}: in connect timeout, {int(connect_delay - (connect_timestamp - self.last_failure_timestamp))}secs remaining')
+                print(f'{self.router_name}@{self.config_entry.hostname}: in connect timeout, {int(connect_delay - (connect_timestamp - self.last_failure_timestamp))}secs remaining')
                 print(f'Successive failure count: {self.successive_failure_count}')
             return True
         if not quiet: 
-            print(f'{self.router_name}@{self.router_entry.hostname}: OK to connect')
+            print(f'{self.router_name}@{self.config_entry.hostname}: OK to connect')
             if self.last_failure_timestamp > 0:
                 print(f'Seconds since last failure: {connect_timestamp - self.last_failure_timestamp}')
                 print(f'Prior successive failure count: {self.successive_failure_count}')
@@ -98,12 +100,12 @@ class RouterAPIConnection:
         if success:
             self.last_failure_timestamp = 0
             self.successive_failure_count = 0
-            print(f'{connect_time.strftime("%Y-%m-%d %H:%M:%S")} Connection to router {self.router_name}@{self.router_entry.hostname} has been established')
+            print(f'{connect_time.strftime("%Y-%m-%d %H:%M:%S")} Connection to router {self.router_name}@{self.config_entry.hostname} has been established')
         else:
             self.api = None
             self.successive_failure_count += 1
             self.last_failure_timestamp = connect_time.timestamp() 
-            print(f'{connect_time.strftime("%Y-%m-%d %H:%M:%S")} Connection to router {self.router_name}@{self.router_entry.hostname} has failed: {exc}')
+            print(f'{connect_time.strftime("%Y-%m-%d %H:%M:%S")} Connection to router {self.router_name}@{self.config_entry.hostname} has failed: {exc}')
 
 
 

mktxp/router_entries_handler.py

@@ -13,28 +13,28 @@
 
 
 from mktxp.cli.config.config import config_handler
-from mktxp.router_metric import RouterMetric
+from mktxp.router_entry import RouterEntry
 
 
-class RouterMetricsHandler:
+class RouterEntriesHandler:
     ''' Handles RouterOS entries defined in MKTXP config 
     '''         
     def __init__(self):
-        self.router_metrics = []
+        self.router_entries = []
         for router_name in config_handler.registered_entries():
             entry = config_handler.entry(router_name)
             if entry.enabled:
-                self.router_metrics.append(RouterMetric(router_name))
+                self.router_entries.append(RouterEntry(router_name))
 
     @staticmethod
-    def router_metric(entry_name, enabled_only = False):
+    def router_entry(entry_name, enabled_only = False):
-        router_metric = None
+        router_entry = None
         for router_name in config_handler.registered_entries():
             if router_name == entry_name:
                 if enabled_only:
                     entry = config_handler.entry(router_name)
                     if not entry.enabled:
                         break
-                router_metric = RouterMetric(router_name)
+                router_entry = RouterEntry(router_name)
                 break
-        return router_metric
+        return router_entry

mktxp/router_entry.py

@@ -0,0 +1,41 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from mktxp.cli.config.config import config_handler, MKTXPConfigKeys
+from mktxp.router_connection import RouterAPIConnection
+
+
+class RouterEntry:
+    ''' RouterOS Entry
+    '''                 
+    def __init__(self, router_name):
+        self.router_name = router_name
+        self.config_entry  = config_handler.entry(router_name)
+        self.api_connection = RouterAPIConnection(router_name, self.config_entry)
+        self.router_id = {
+            MKTXPConfigKeys.ROUTERBOARD_NAME: self.router_name,
+            MKTXPConfigKeys.ROUTERBOARD_ADDRESS: self.config_entry.hostname
+            }
+        self.time_spent =  { 'IdentityCollector': 0,
+                            'SystemResourceCollector': 0,
+                            'HealthCollector': 0,
+                            'DHCPCollector': 0,
+                            'PoolCollector': 0,
+                            'InterfaceCollector': 0,
+                            'FirewallCollector': 0,
+                            'MonitorCollector': 0,
+                            'RouteCollector': 0,
+                            'WLANCollector': 0,
+                            'CapsmanCollector': 0
+                            }            

mktxp/router_metric.py

@@ -1,201 +0,0 @@
-# coding=utf8
-## Copyright (c) 2020 Arseniy Kuznetsov
-##
-## This program is free software; you can redistribute it and/or
-## modify it under the terms of the GNU General Public License
-## as published by the Free Software Foundation; either version 2
-## of the License, or (at your option) any later version.
-##
-## This program is distributed in the hope that it will be useful,
-## but WITHOUT ANY WARRANTY; without even the implied warranty of
-## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-## GNU General Public License for more details.
-
-
-from mktxp.cli.config.config import config_handler, MKTXPConfigKeys
-from mktxp.router_connection import RouterAPIConnection
-
-class RouterMetric:
-    ''' RouterOS Metrics data provider
-    '''             
-    def __init__(self, router_name):
-        self.router_name = router_name
-        self.router_entry  = config_handler.entry(router_name)
-        self.api_connection = RouterAPIConnection(router_name, self.router_entry)
-        self.router_id = {
-            MKTXPConfigKeys.ROUTERBOARD_NAME: self.router_name,
-            MKTXPConfigKeys.ROUTERBOARD_ADDRESS: self.router_entry.hostname
-            }
-        self.time_spent =  { 'IdentityCollector': 0,
-                            'SystemResourceCollector': 0,
-                            'HealthCollector': 0,
-                            'DHCPCollector': 0,
-                            'PoolCollector': 0,
-                            'InterfaceCollector': 0,
-                            'FirewallCollector': 0,
-                            'MonitorCollector': 0,
-                            'RouteCollector': 0,
-                            'WLANCollector': 0,
-                            'CapsmanCollector': 0
-                            }            
-
-    def identity_records(self, identity_labels = []):
-        try:
-            identity_records = self.api_connection.router_api().get_resource('/system/identity').get()
-            return self._trimmed_records(identity_records, identity_labels)
-        except Exception as exc:
-            print(f'Error getting system identity info from router{self.router_name}@{self.router_entry.hostname}: {exc}')
-            return None
-
-    def routerboard_records(self, routerboard_labels = []):
-        try:
-            routerboard_records = self.api_connection.router_api().get_resource('/system/routerboard').get()
-            return self._trimmed_records(routerboard_records, routerboard_labels)
-        except Exception as exc:
-            print(f'Error getting system routerboard info from router{self.router_name}@{self.router_entry.hostname}: {exc}')
-            return None
-
-    def health_records(self, health_labels = []):
-        try:
-            health_records = self.api_connection.router_api().get_resource('/system/health').get()
-            return self._trimmed_records(health_records, health_labels)
-        except Exception as exc:
-            print(f'Error getting system health info from router{self.router_name}@{self.router_entry.hostname}: {exc}')
-            return None
-
-    def system_resource_records(self, resource_labels = []):
-        try:
-            system_resource_records = self.api_connection.router_api().get_resource('/system/resource').get()
-            return self._trimmed_records(system_resource_records, resource_labels)
-        except Exception as exc:
-            print(f'Error getting system resource info from router{self.router_name}@{self.router_entry.hostname}: {exc}')
-            return None
-
-    def dhcp_lease_records(self, dhcp_lease_labels = [], add_router_id = True):
-        try:
-            #dhcp_lease_records = self.api_connection.router_api().get_resource('/ip/dhcp-server/lease').get(status='bound')
-            dhcp_lease_records = self.api_connection.router_api().get_resource('/ip/dhcp-server/lease').call('print', {'active':''})
-            return self._trimmed_records(dhcp_lease_records, dhcp_lease_labels, add_router_id)
-        except Exception as exc:
-            print(f'Error getting dhcp info from router{self.router_name}@{self.router_entry.hostname}: {exc}')
-            return None
-
-    def interface_traffic_records(self, interface_traffic_labels = []):
-        try:
-            traffic_records = self.api_connection.router_api().get_resource('/interface').get(running='yes', disabled='no')
-            return self._trimmed_records(traffic_records, interface_traffic_labels)
-        except Exception as exc:
-            print(f'Error getting interface traffic info from router{self.router_name}@{self.router_entry.hostname}: {exc}')
-            return None
-
-    def interface_monitor_records(self, interface_monitor_labels = [], kind = 'ethernet', include_comments = False):
-        try:
-            interfaces = self.api_connection.router_api().get_resource(f'/interface/{kind}').get()
-            interface_names = [(interface['name'], interface.get('comment'), interface.get('running')) for interface in interfaces]
-
-            interface_monitor = lambda int_num : self.api_connection.router_api().get_resource(f'/interface/{kind}').call('monitor', {'once':'', 'numbers':f'{int_num}'})
-            interface_monitor_records = [interface_monitor(int_num)[0] for int_num in range(len(interface_names))]
-
-            if include_comments:
-                # combine interfaces names with comments
-                for interface_monitor_record in interface_monitor_records:
-                    for interface_name in interface_names:
-                        if interface_name[1] and interface_name[0] == interface_monitor_record['name']:
-                            interface_monitor_record['name'] = interface_name[1] if self.router_entry.use_comments_over_names else \
-                                                                                 f"{interface_name[0]} ({interface_name[1]})"
-            return self._trimmed_records(interface_monitor_records, interface_monitor_labels)
-        except Exception as exc:
-            print(f'Error getting {kind} interface monitor info from router{self.router_name}@{self.router_entry.hostname}: {exc}')
-            return None
-
-    def pool_records(self, pool_labels = []):
-        try:
-            pool_records = self.api_connection.router_api().get_resource('/ip/pool').get()
-            return self._trimmed_records(pool_records, pool_labels)
-        except Exception as exc:
-            print(f'Error getting pool info from router{self.router_name}@{self.router_entry.hostname}: {exc}')
-            return None
-
-    def pool_used_records(self, pool_used_labels = []):
-        try:
-            pool_used_records = self.api_connection.router_api().get_resource('/ip/pool/used').get()
-            return self._trimmed_records(pool_used_records, pool_used_labels)
-        except Exception as exc:
-            print(f'Error getting pool used info from router{self.router_name}@{self.router_entry.hostname}: {exc}')
-            return None
-
-    def route_records(self, route_labels = []):
-        try:
-            route_records = self.api_connection.router_api().get_resource('/ip/route').get(active='yes')
-            return self._trimmed_records(route_records, route_labels)
-        except Exception as exc:
-            print(f'Error getting routes info from router{self.router_name}@{self.router_entry.hostname}: {exc}')
-            return None
-            
-    def wireless_registration_table_records(self, registration_table_labels = [], add_router_id = True):
-        try:
-            registration_table_records = self.api_connection.router_api().get_resource('/interface/wireless/registration-table').get()
-            return self._trimmed_records(registration_table_records, registration_table_labels, add_router_id)
-        except Exception as exc:
-            print(f'Error getting wireless registration table info from router{self.router_name}@{self.router_entry.hostname}: {exc}')
-            return None
-
-    def capsman_remote_caps_records(self, remote_caps_labels = []):
-        try:
-            remote_caps_records = self.api_connection.router_api().get_resource('/caps-man/remote-cap').get()
-            return self._trimmed_records(remote_caps_records, remote_caps_labels)
-        except Exception as exc:
-            print(f'Error getting caps-man remote caps info from router{self.router_name}@{self.router_entry.hostname}: {exc}')
-            return None
-
-    def capsman_registration_table_records(self, registration_table_labels = [], add_router_id = True):
-        try:
-            registration_table_records = self.api_connection.router_api().get_resource('/caps-man/registration-table').get()
-            return self._trimmed_records(registration_table_records, registration_table_labels, add_router_id)
-        except Exception as exc:
-            print(f'Error getting caps-man registration table info from router{self.router_name}@{self.router_entry.hostname}: {exc}')
-            return None
-
-    def firewall_records(self, firewall_labels = [], raw = False, matching_only = True):
-        try:
-            filter_path = '/ip/firewall/filter' if not raw else '/ip/firewall/raw'
-            firewall_records = self.api_connection.router_api().get_resource(filter_path).call('print', {'stats':'', 'all':''})
-            if matching_only:
-                firewall_records = [record for record in firewall_records if int(record.get('bytes', '0')) > 0]
-            # translation rules
-            translation_table = {}
-#            if 'id' in firewall_labels:
-#                translation_table['id'] = lambda id: str(int(id[1:], 16) - 1)
-            if 'comment' in firewall_labels:
-                translation_table['comment'] = lambda c: c if c else ''
-            return self._trimmed_records(firewall_records, firewall_labels, translation_table = translation_table)
-        except Exception as exc:
-            print(f'Error getting firewall filters info from router{self.router_name}@{self.router_entry.hostname}: {exc}')
-            return None
-
-    def mktxp_records(self):
-        mktxp_records = []
-        for key in self.time_spent.keys():
-            mktxp_records.append({'name': key, 'duration': self.time_spent[key]})
-        # translation rules            
-        translation_table = {'duration': lambda d: d*1000}
-        return self._trimmed_records(mktxp_records, translation_table = translation_table)
-
-    # Helpers
-    def _trimmed_records(self, router_records, metric_labels = [], add_router_id = True, translation_table = {}):
-        if len(metric_labels) == 0 and len(router_records) > 0:
-            metric_labels = router_records[0].keys()
-        metric_labels = set(metric_labels)      
-
-        labeled_records = []
-        dash2_ = lambda x : x.replace('-', '_')
-        for router_record in router_records:
-            translated_record = {dash2_(key): value for (key, value) in router_record.items() if dash2_(key) in metric_labels}
-            if add_router_id:
-                for key, value in self.router_id.items():
-                    translated_record[key] = value
-            # translate fields if needed
-            for key, func in translation_table.items():
-                translated_record[key] = func(translated_record.get(key))
-            labeled_records.append(translated_record)
-        return labeled_records

mktxp/utils/utils.py

@@ -11,6 +11,7 @@
 ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 ## GNU General Public License for more details.
 
+
 import os, sys, shlex, tempfile, shutil, re
 import subprocess, hashlib
 from timeit import default_timer

setup.py

@@ -20,7 +20,7 @@ with open(path.join(pkg_dir, 'README.md'), encoding='utf-8') as f:
 
 setup(
     name='mktxp',
-    version='0.23',
+    version='0.24',
 
     url='https://github.com/akpw/mktxp',