DS refactor, fixes/optimizations

mktxp/collector/bandwidth_collector.py

@@ -0,0 +1,76 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+import socket
+import speedtest
+from datetime import datetime
+from multiprocessing import Pool
+from mktxp.cli.config.config import config_handler
+from mktxp.collector.base_collector import BaseCollector
+
+
+result_list = [{'download': 0, 'upload': 0, 'ping': 0}]
+def get_result(bandwidth_dict):
+    result_list[0] = bandwidth_dict
+
+
+class BandwidthCollector(BaseCollector):
+    ''' MKTXP collector
+    '''    
+    def __init__(self):
+        self.pool = Pool()
+        self.last_call_timestamp = 0        
+    
+    def collect(self):
+        if result_list:      
+            result_dict = result_list[0]
+            bandwidth_records = [{'direction': key, 'bandwidth': str(result_dict[key])} for key in ('download', 'upload')]     
+            bandwidth_metrics = BaseCollector.gauge_collector('internet_bandwidth', 'Internet bandwidth in bits per second', 
+                                                                            bandwidth_records, 'bandwidth', ['direction'], add_id_labels = False)
+            yield bandwidth_metrics
+
+            latency_records = [{'latency': str(result_dict['ping'])}]
+            latency_metrics = BaseCollector.gauge_collector('internet_latency', 'Internet latency in milliseconds', 
+                                                                            latency_records, 'latency', [], add_id_labels = False)
+            yield latency_metrics
+
+        ts =  datetime.now().timestamp()       
+        if (ts - self.last_call_timestamp) > config_handler._entry().bandwidth_test_interval:            
+            self.pool.apply_async(BandwidthCollector.bandwidth_worker, callback=get_result)            
+            self.last_call_timestamp = ts
+
+    def __del__(self):
+        self.pool.close()
+        self.pool.join()        
+
+    @staticmethod
+    def bandwidth_worker():
+        if BandwidthCollector.inet_connected():
+            bandwidth_test = speedtest.Speedtest()
+            bandwidth_test.get_best_server()
+            bandwidth_test.download()
+            bandwidth_test.upload()
+            return bandwidth_test.results.dict()
+        else:
+            return {'download': 0, 'upload': 0, 'ping': 0}
+
+    @staticmethod
+    def inet_connected(host="8.8.8.8", port=53, timeout=3):
+        try:
+            socket.setdefaulttimeout(timeout)
+            socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((host, port))
+            return True
+        except socket.error as exc:
+            return False
+

mktxp/collector/base_collector.py

@@ -0,0 +1,59 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from prometheus_client.core import GaugeMetricFamily, CounterMetricFamily, InfoMetricFamily
+from mktxp.cli.config.config import MKTXPConfigKeys
+     
+     
+class BaseCollector:
+    ''' Base Collector methods
+        For use by custom collector
+    '''
+    @staticmethod
+    def info_collector(name, decription, router_records, metric_labels=[]):
+        BaseCollector._add_id_labels(metric_labels)
+        collector = InfoMetricFamily(f'mktxp_{name}', decription)
+
+        for router_record in router_records:
+            label_values = {label: router_record.get(label) if router_record.get(label) else '' for label in metric_labels}
+            collector.add_metric(metric_labels, label_values)
+        return collector
+
+    @staticmethod
+    def counter_collector(name, decription, router_records, metric_key, metric_labels=[]):
+        BaseCollector._add_id_labels(metric_labels)
+        collector = CounterMetricFamily(f'mktxp_{name}', decription, labels=metric_labels)
+
+        for router_record in router_records:           
+            label_values = [router_record.get(label) for label in metric_labels]
+            collector.add_metric(label_values, router_record.get(metric_key, 0))
+        return collector
+
+    @staticmethod
+    def gauge_collector(name, decription, router_records, metric_key, metric_labels=[], add_id_labels = True):
+        if add_id_labels:
+            BaseCollector._add_id_labels(metric_labels)
+        collector = GaugeMetricFamily(f'mktxp_{name}', decription, labels=metric_labels)
+
+        for router_record in router_records:       
+            label_values = [router_record.get(label) for label in metric_labels]
+            collector.add_metric(label_values, router_record.get(metric_key, 0))
+        return collector
+
+
+    # Helpers
+    @staticmethod
+    def _add_id_labels(metric_labels):
+        metric_labels.append(MKTXPConfigKeys.ROUTERBOARD_NAME)
+        metric_labels.append(MKTXPConfigKeys.ROUTERBOARD_ADDRESS)

mktxp/collector/capsman_collector.py

@@ -0,0 +1,69 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from mktxp.cli.output.base_out import BaseOutputProcessor
+from mktxp.cli.config.config import MKTXPConfigKeys
+from mktxp.collector.base_collector import BaseCollector
+from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource
+from mktxp.datasource.capsman_ds import CapsmanCapsMetricsDataSource, CapsmanRegistrationsMetricsDataSource
+
+
+class CapsmanCollector(BaseCollector):
+    ''' CAPsMAN Metrics collector
+    '''    
+    @staticmethod
+    def collect(router_entry):
+        remote_caps_labels = ['identity', 'version', 'base_mac', 'board', 'base_mac']
+        remote_caps_records = CapsmanCapsMetricsDataSource.metric_records(router_entry, metric_labels = remote_caps_labels)
+        if remote_caps_records:
+            remote_caps_metrics = BaseCollector.info_collector('capsman_remote_caps', 'CAPsMAN remote caps', remote_caps_records, remote_caps_labels)
+            yield remote_caps_metrics
+
+        registration_labels = ['interface', 'ssid', 'mac_address', 'tx_rate', 'rx_rate', 'rx_signal', 'uptime', 'bytes']
+        registration_records = CapsmanRegistrationsMetricsDataSource.metric_records(router_entry, metric_labels = registration_labels)
+        if registration_records:
+            # calculate number of registrations per interface
+            registration_per_interface = {}
+            for registration_record in registration_records:
+                registration_per_interface[registration_record['interface']] = registration_per_interface.get(registration_record['interface'], 0) + 1
+            # compile registrations-per-interface records
+            registration_per_interface_records = [{ MKTXPConfigKeys.ROUTERBOARD_NAME: router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
+                                            MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
+                                            'interface': key, 'count': value} for key, value in registration_per_interface.items()]
+            # yield registrations-per-interface metrics
+            registration_per_interface_metrics = BaseCollector.gauge_collector('capsman_registrations_count', 'Number of active registration per CAPsMAN interface', registration_per_interface_records, 'count', ['interface'])
+            yield registration_per_interface_metrics
+
+            # the client info metrics
+            if router_entry.config_entry.capsman_clients:
+                # translate / trim / augment registration records
+                dhcp_lease_labels = ['mac_address', 'address', 'host_name', 'comment']
+                dhcp_lease_records = DHCPMetricsDataSource.metric_records(router_entry, metric_labels = dhcp_lease_labels)
+                for registration_record in registration_records:
+                    BaseOutputProcessor.augment_record(router_entry, registration_record, dhcp_lease_records)
+                    
+                tx_byte_metrics = BaseCollector.counter_collector('capsman_clients_tx_bytes', 'Number of sent packet bytes', registration_records, 'tx_bytes', ['dhcp_name'])
+                yield tx_byte_metrics
+
+                rx_byte_metrics = BaseCollector.counter_collector('capsman_clients_rx_bytes', 'Number of received packet bytes', registration_records, 'rx_bytes', ['dhcp_name'])
+                yield rx_byte_metrics
+
+                signal_strength_metrics = BaseCollector.gauge_collector('capsman_clients_signal_strength', 'Client devices signal strength', registration_records, 'rx_signal', ['dhcp_name'])
+                yield signal_strength_metrics
+
+                registration_metrics = BaseCollector.info_collector('capsman_clients_devices', 'Registered client devices info', 
+                                        registration_records, ['dhcp_name', 'dhcp_address', 'rx_signal', 'ssid', 'tx_rate', 'rx_rate', 'interface', 'mac_address', 'uptime'])
+                yield registration_metrics
+    
+

mktxp/collector/dhcp_collector.py

@@ -0,0 +1,46 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from mktxp.cli.config.config import MKTXPConfigKeys
+from mktxp.collector.base_collector import BaseCollector
+from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource
+
+
+class DHCPCollector(BaseCollector):
+    ''' DHCP Metrics collector
+    '''    
+    @staticmethod
+    def collect(router_entry):
+        dhcp_lease_labels = ['active_address', 'address', 'mac_address', 'host_name', 'comment', 'server', 'expires_after']
+        dhcp_lease_records = DHCPMetricsDataSource.metric_records(router_entry, metric_labels = dhcp_lease_labels)   
+        if dhcp_lease_records:
+            # calculate number of leases per DHCP server
+            dhcp_lease_servers = {}
+            for dhcp_lease_record in dhcp_lease_records:
+                dhcp_lease_servers[dhcp_lease_record['server']] = dhcp_lease_servers.get(dhcp_lease_record['server'], 0) + 1
+
+            # compile leases-per-server records
+            dhcp_lease_servers_records = [{ MKTXPConfigKeys.ROUTERBOARD_NAME: router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
+                                            MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
+                                            'server': key, 'count': value} for key, value in dhcp_lease_servers.items()]
+            
+            # yield lease-per-server metrics
+            dhcp_lease_server_metrics = BaseCollector.gauge_collector('dhcp_lease_active_count', 'Number of active leases per DHCP server', dhcp_lease_servers_records, 'count', ['server'])
+            yield dhcp_lease_server_metrics
+
+            # active lease metrics
+            if router_entry.config_entry.dhcp_lease:
+                dhcp_lease_metrics = BaseCollector.info_collector('dhcp_lease', 'DHCP Active Leases', dhcp_lease_records, dhcp_lease_labels)
+                yield dhcp_lease_metrics
+            

mktxp/collector/firewall_collector.py

@@ -0,0 +1,47 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from mktxp.cli.config.config import MKTXPConfigKeys
+from mktxp.collector.base_collector import BaseCollector
+from mktxp.datasource.firewall_ds import FirewallMetricsDataSource
+
+
+class FirewallCollector(BaseCollector):
+    ''' Firewall rules traffic metrics collector
+    '''    
+    @staticmethod
+    def collect(router_entry):
+        # initialize all pool counts, including those currently not used
+        firewall_labels = ['chain', 'action', 'bytes', 'comment']
+        
+        firewall_filter_records = FirewallMetricsDataSource.metric_records(router_entry, metric_labels = firewall_labels)   
+        if firewall_filter_records:           
+            metris_records = [FirewallCollector.metric_record(router_entry, record) for record in firewall_filter_records]
+            firewall_filter_metrics = BaseCollector.counter_collector('firewall_filter', 'Total amount of bytes matched by firewall rules', metris_records, 'bytes', ['name'])
+            yield firewall_filter_metrics
+
+        firewall_raw_records = FirewallMetricsDataSource.metric_records(router_entry, metric_labels = firewall_labels, raw = True)        
+        if firewall_raw_records:      
+            metris_records = [FirewallCollector.metric_record(router_entry, record) for record in firewall_raw_records]     
+            firewall_raw_metrics = BaseCollector.counter_collector('firewall_raw', 'Total amount of bytes matched by raw firewall rules', metris_records, 'bytes', ['name'])
+            yield firewall_raw_metrics
+
+    # Helpers
+    @staticmethod
+    def metric_record(router_entry, firewall_record):
+        name = f"| {firewall_record['chain']} | {firewall_record['action']} | {firewall_record['comment']}"
+        bytes = firewall_record['bytes']
+        return {MKTXPConfigKeys.ROUTERBOARD_NAME: router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
+                MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
+                'name': name, 'bytes': bytes}

mktxp/collector/health_collector.py

@@ -0,0 +1,31 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from mktxp.collector.base_collector import BaseCollector
+from mktxp.datasource.health_ds import HealthMetricsDataSource
+
+
+class HealthCollector(BaseCollector):
+    ''' System Health Metrics collector
+    '''    
+    @staticmethod
+    def collect(router_entry):
+        health_labels = ['voltage', 'temperature']
+        health_records = HealthMetricsDataSource.metric_records(router_entry, metric_labels = health_labels)   
+        if health_records:
+            voltage_metrics = BaseCollector.gauge_collector('system_routerboard_voltage', 'Supplied routerboard voltage', health_records, 'voltage')
+            yield voltage_metrics
+
+            temperature_metrics = BaseCollector.gauge_collector('system_routerboard_temperature', ' Routerboard current temperature', health_records, 'temperature')
+            yield temperature_metrics

mktxp/collector/identity_collector.py

@@ -0,0 +1,29 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from mktxp.collector.base_collector import BaseCollector
+from mktxp.datasource.identity_ds import IdentityMetricsDataSource
+
+
+class IdentityCollector(BaseCollector):
+    ''' System Identity Metrics collector
+    '''     
+    @staticmethod
+    def collect(router_entry):
+        identity_labels = ['name']
+        identity_records = IdentityMetricsDataSource.metric_records(router_entry, metric_labels = identity_labels)                
+        if identity_records:
+            identity_metrics = BaseCollector.info_collector('system_identity', 'System identity', identity_records, identity_labels)
+            yield identity_metrics
+

mktxp/collector/interface_collector.py

@@ -0,0 +1,55 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from mktxp.collector.base_collector import BaseCollector
+from mktxp.datasource.interface_ds import InterfaceTrafficMetricsDataSource
+
+
+class InterfaceCollector(BaseCollector):
+    ''' Router Interface Metrics collector
+    '''        
+    @staticmethod
+    def collect(router_entry):
+        interface_traffic_labels = ['name', 'comment', 'rx_byte', 'tx_byte', 'rx_packet', 'tx_packet', 'rx_error', 'tx_error', 'rx_drop', 'tx_drop']
+        interface_traffic_records = InterfaceTrafficMetricsDataSource.metric_records(router_entry, metric_labels = interface_traffic_labels)   
+
+        if interface_traffic_records:
+            for interface_traffic_record in interface_traffic_records:
+                if interface_traffic_record.get('comment'):
+                    interface_traffic_record['name'] = interface_traffic_record['comment'] if router_entry.config_entry.use_comments_over_names \
+                                                                                else f"{interface_traffic_record['name']} ({interface_traffic_record['comment']})"
+
+            rx_byte_metric = BaseCollector.counter_collector('interface_rx_byte', 'Number of received bytes', interface_traffic_records, 'rx_byte', ['name'])
+            yield rx_byte_metric
+
+            tx_byte_metric = BaseCollector.counter_collector('interface_tx_byte', 'Number of transmitted bytes', interface_traffic_records, 'tx_byte', ['name'])
+            yield tx_byte_metric
+
+            rx_packet_metric = BaseCollector.counter_collector('interface_rx_packet', 'Number of packets received', interface_traffic_records, 'rx_packet', ['name'])
+            yield rx_packet_metric
+
+            tx_packet_metric = BaseCollector.counter_collector('interface_tx_packet', 'Number of transmitted packets', interface_traffic_records, 'tx_packet', ['name'])
+            yield tx_packet_metric
+
+            rx_error_metric = BaseCollector.counter_collector('interface_rx_error', 'Number of packets received with an error', interface_traffic_records, 'rx_error', ['name'])
+            yield rx_error_metric
+
+            tx_error_metric = BaseCollector.counter_collector('interface_tx_error', 'Number of packets transmitted with an error', interface_traffic_records, 'tx_error', ['name'])
+            yield tx_error_metric
+
+            rx_drop_metric = BaseCollector.counter_collector('interface_rx_drop', 'Number of received packets being dropped', interface_traffic_records, 'rx_drop', ['name'])
+            yield rx_drop_metric
+
+            tx_drop_metric = BaseCollector.counter_collector('interface_tx_drop', 'Number of transmitted packets being dropped', interface_traffic_records, 'tx_drop', ['name'])
+            yield tx_drop_metric

mktxp/collector/mktxp_collector.py

@@ -0,0 +1,29 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from mktxp.collector.base_collector import BaseCollector
+from mktxp.datasource.mktxp_ds import MKTXPMetricsDataSource
+
+
+class MKTXPCollector(BaseCollector):
+    ''' System Identity Metrics collector
+    '''     
+    @staticmethod
+    def collect(router_entry):
+        mktxp_records = MKTXPMetricsDataSource.metric_records(router_entry)
+        if mktxp_records:
+            mktxp_duration_metric = BaseCollector.counter_collector('collection_time', 'Total time spent collecting metrics in milliseconds', mktxp_records, 'duration', ['name'])
+            yield mktxp_duration_metric
+
+

mktxp/collector/monitor_collector.py

@@ -0,0 +1,77 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from mktxp.collector.base_collector import BaseCollector
+from mktxp.cli.output.base_out import BaseOutputProcessor
+from mktxp.datasource.interface_ds import InterfaceMonitorMetricsDataSource
+
+
+class MonitorCollector(BaseCollector):
+    ''' Ethernet Interface Monitor Metrics collector
+    '''    
+    @staticmethod
+    def collect(router_entry):
+        monitor_labels = ('status', 'rate', 'full_duplex', 'name')
+        monitor_records = InterfaceMonitorMetricsDataSource.metric_records(router_entry, metric_labels = monitor_labels, include_comments = True)   
+        if monitor_records:
+            # translate records to appropriate values
+            for monitor_record in monitor_records:
+                for monitor_label in monitor_labels:
+                    value = monitor_record.get(monitor_label, None)    
+                    if value:            
+                        monitor_record[monitor_label] = MonitorCollector._translated_values(monitor_label, value)
+
+            monitor_status_metrics = BaseCollector.gauge_collector('interface_status', 'Current interface link status', monitor_records, 'status', ['name'])
+            yield monitor_status_metrics
+
+            # limit records according to the relevant metrics
+            rate_records = [monitor_record for monitor_record in monitor_records if monitor_record.get('rate', None)]
+            monitor_rates_metrics = BaseCollector.gauge_collector('interface_rate', 'Actual interface connection data rate', rate_records, 'rate', ['name'])
+            yield monitor_rates_metrics
+
+            full_duplex_records = [monitor_record for monitor_record in monitor_records if monitor_record.get('full_duplex', None)]
+            monitor_rates_metrics = BaseCollector.gauge_collector('interface_full_duplex', 'Full duplex data transmission', full_duplex_records, 'full_duplex', ['name'])
+            yield monitor_rates_metrics
+
+    # Helpers
+    @staticmethod
+    def _translated_values(monitor_label, value):
+        return {
+                'status': lambda value: '1' if value=='link-ok' else '0',
+                'rate': lambda value: MonitorCollector._rates(value),
+                'full_duplex': lambda value: '1' if value=='true' else '0',
+                'name': lambda value: value
+                }[monitor_label](value)
+
+    @staticmethod
+    def _rates(rate_option):
+        # according mikrotik docs, an interface rate should be one of these
+        rate_value =  {
+                '10Mbps': '10',
+                '100Mbps': '100',
+                '1Gbps': '1000',
+                '2.5Gbps': '2500',
+                '5Gbps': '5000',
+                '10Gbps': '10000',
+                '40Gbps': '40000'
+                }.get(rate_option, None)
+        if rate_value:
+            return rate_value
+        
+        # ...or just calculate in case it's not
+        return BaseOutputProcessor.parse_interface_rate(rate_option)
+
+
+
+

mktxp/collector/pool_collector.py

@@ -0,0 +1,43 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from mktxp.cli.config.config import MKTXPConfigKeys
+from mktxp.collector.base_collector import BaseCollector
+from mktxp.datasource.pool_ds import PoolMetricsDataSource, PoolUsedMetricsDataSource
+
+
+class PoolCollector(BaseCollector):
+    ''' IP Pool Metrics collector
+    '''    
+    @staticmethod
+    def collect(router_entry):
+        # initialize all pool counts, including those currently not used
+        pool_records = PoolMetricsDataSource.metric_records(router_entry, metric_labels = ['name'])   
+        if pool_records:
+            pool_used_labels = ['pool']
+            pool_used_counts = {pool_record['name']: 0 for pool_record in pool_records}
+
+            # for pools in usage, calculate the current numbers
+            pool_used_records = PoolUsedMetricsDataSource.metric_records(router_entry, metric_labels = pool_used_labels)   
+            for pool_used_record in pool_used_records:
+                pool_used_counts[pool_used_record['pool']] = pool_used_counts.get(pool_used_record['pool'], 0) + 1
+
+           # compile used-per-pool records
+            used_per_pool_records = [{ MKTXPConfigKeys.ROUTERBOARD_NAME: router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
+                                       MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
+                                       'pool': key, 'count': value} for key, value in pool_used_counts.items()]
+            
+            # yield used-per-pool metrics
+            used_per_pool_metrics = BaseCollector.gauge_collector('ip_pool_used', 'Number of used addresses per IP pool', used_per_pool_records, 'count', ['pool'])
+            yield used_per_pool_metrics

mktxp/collector/resource_collector.py

@@ -0,0 +1,71 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from mktxp.collector.base_collector import BaseCollector
+from mktxp.cli.output.base_out import BaseOutputProcessor
+from mktxp.datasource.system_resource_ds import SystemResourceMetricsDataSource
+
+
+class SystemResourceCollector(BaseCollector):
+    ''' System Resource Metrics collector
+    '''        
+    @staticmethod
+    def collect(router_entry):
+        resource_labels = ['uptime', 'version', 'free_memory', 'total_memory', 
+                           'cpu', 'cpu_count', 'cpu_frequency', 'cpu_load', 
+                           'free_hdd_space', 'total_hdd_space', 
+                           'architecture_name', 'board_name']
+                                   
+        resource_records = SystemResourceMetricsDataSource.metric_records(router_entry, metric_labels = resource_labels)   
+        if resource_records:
+            # translate records to appropriate values
+            translated_fields = ['uptime']        
+            for resource_record in resource_records:
+                for translated_field in translated_fields:
+                    value = resource_record.get(translated_field, None)    
+                    if value:            
+                        resource_record[translated_field] = SystemResourceCollector._translated_values(translated_field, value)
+
+            uptime_metrics = BaseCollector.gauge_collector('system_uptime', 'Time interval since boot-up', resource_records, 'uptime', ['version', 'board_name', 'cpu', 'architecture_name'])
+            yield uptime_metrics
+
+            free_memory_metrics = BaseCollector.gauge_collector('system_free_memory', 'Unused amount of RAM', resource_records, 'free_memory', ['version', 'board_name', 'cpu', 'architecture_name'])
+            yield free_memory_metrics
+
+            total_memory_metrics = BaseCollector.gauge_collector('system_total_memory', 'Amount of installed RAM', resource_records, 'total_memory', ['version', 'board_name', 'cpu', 'architecture_name'])
+            yield total_memory_metrics
+
+            free_hdd_metrics = BaseCollector.gauge_collector('system_free_hdd_space', 'Free space on hard drive or NAND', resource_records, 'free_hdd_space', ['version', 'board_name', 'cpu', 'architecture_name'])
+            yield free_hdd_metrics
+
+            total_hdd_metrics = BaseCollector.gauge_collector('system_total_hdd_space', 'Size of the hard drive or NAND', resource_records, 'total_hdd_space', ['version', 'board_name', 'cpu', 'architecture_name'])
+            yield total_hdd_metrics
+
+            cpu_load_metrics = BaseCollector.gauge_collector('system_cpu_load', 'Percentage of used CPU resources', resource_records, 'cpu_load', ['version', 'board_name', 'cpu', 'architecture_name'])
+            yield cpu_load_metrics
+
+            cpu_count_metrics = BaseCollector.gauge_collector('system_cpu_count', 'Number of CPUs present on the system', resource_records, 'cpu_count', ['version', 'board_name', 'cpu', 'architecture_name'])
+            yield cpu_count_metrics
+
+            cpu_frequency_metrics = BaseCollector.gauge_collector('system_cpu_frequency', 'Current CPU frequency', resource_records, 'cpu_frequency', ['version', 'board_name', 'cpu', 'architecture_name'])
+            yield cpu_frequency_metrics
+
+
+    # Helpers
+    @staticmethod
+    def _translated_values(translated_field, value):
+        return {
+                'uptime': lambda value: BaseOutputProcessor.parse_timedelta_seconds(value)
+                }[translated_field](value)
+

mktxp/collector/route_collector.py

@@ -0,0 +1,53 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from mktxp.cli.config.config import MKTXPConfigKeys
+from mktxp.collector.base_collector import BaseCollector
+from mktxp.datasource.route_ds import RouteMetricsDataSource
+
+
+class RouteCollector(BaseCollector):
+    ''' IP Route Metrics collector
+    '''        
+    @staticmethod
+    def collect(router_entry):
+        route_labels = ['connect', 'dynamic', 'static', 'bgp', 'ospf']
+        route_records = RouteMetricsDataSource.metric_records(router_entry, metric_labels = route_labels)   
+        if route_records:       
+            # compile total routes records
+            total_routes = len(route_records)
+            total_routes_records = [{ MKTXPConfigKeys.ROUTERBOARD_NAME: router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
+                                      MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
+                                      'count': total_routes
+                                    }]
+            total_routes_metrics = BaseCollector.gauge_collector('routes_total_routes', 'Overall number of routes in RIB', total_routes_records, 'count')
+            yield total_routes_metrics
+
+
+            # init routes per protocol (with 0)
+            routes_per_protocol = {route_label: 0 for route_label in route_labels}
+            for route_record in route_records:
+                for route_label in route_labels:
+                    if route_record.get(route_label):
+                        routes_per_protocol[route_label] += 1 
+
+            # compile route-per-protocol records
+            route_per_protocol_records = [{ MKTXPConfigKeys.ROUTERBOARD_NAME: router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
+                                            MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
+                                            'protocol': key, 'count': value} for key, value in routes_per_protocol.items()]
+            
+            # yield route-per-protocol metrics
+            route_per_protocol_metrics = BaseCollector.gauge_collector('routes_protocol_count', 'Number of routes per protocol in RIB', route_per_protocol_records, 'count', ['protocol'])
+            yield route_per_protocol_metrics
+

mktxp/collector/wlan_collector.py

@@ -0,0 +1,78 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from mktxp.cli.output.base_out import BaseOutputProcessor
+from mktxp.collector.base_collector import BaseCollector
+from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource
+from mktxp.datasource.wireless_ds import WirelessMetricsDataSource
+from mktxp.datasource.interface_ds import InterfaceMonitorMetricsDataSource
+
+
+class WLANCollector(BaseCollector):
+    ''' Wireless Metrics collector
+    '''    
+    @staticmethod
+    def collect(router_entry):
+        monitor_labels = ['channel', 'noise_floor', 'overall_tx_ccq', 'registered_clients']
+        monitor_records = InterfaceMonitorMetricsDataSource.metric_records(router_entry, metric_labels = monitor_labels, kind = 'wireless')   
+        if monitor_records:
+            # sanitize records for relevant labels
+            noise_floor_records = [monitor_record for monitor_record in monitor_records if monitor_record.get('noise_floor')]
+            tx_ccq_records = [monitor_record for monitor_record in monitor_records if monitor_record.get('overall_tx_ccq')]
+            registered_clients_records  = [monitor_record for monitor_record in monitor_records if monitor_record.get('registered_clients')]
+
+            if noise_floor_records:
+                noise_floor_metrics = BaseCollector.gauge_collector('wlan_noise_floor', 'Noise floor threshold', noise_floor_records, 'noise_floor', ['channel'])
+                yield noise_floor_metrics
+
+            if tx_ccq_records:
+                overall_tx_ccq_metrics = BaseCollector.gauge_collector('wlan_overall_tx_ccq', 'Client Connection Quality for transmitting', tx_ccq_records, 'overall_tx_ccq', ['channel'])
+                yield overall_tx_ccq_metrics
+
+            if registered_clients_records:
+                registered_clients_metrics = BaseCollector.gauge_collector('wlan_registered_clients', 'Number of registered clients', registered_clients_records, 'registered_clients', ['channel'])
+                yield registered_clients_metrics
+
+        # the client info metrics
+        if router_entry.config_entry.wireless_clients:
+            registration_labels = ['interface', 'ssid', 'mac_address', 'tx_rate', 'rx_rate', 'uptime', 'bytes', 'signal_to_noise', 'tx_ccq', 'signal_strength']
+            registration_records = WirelessMetricsDataSource.metric_records(router_entry, metric_labels = registration_labels)
+            if registration_records:
+                dhcp_lease_labels = ['mac_address', 'address', 'host_name', 'comment']
+                dhcp_lease_records = DHCPMetricsDataSource.metric_records(router_entry, metric_labels = dhcp_lease_labels)
+      
+                for registration_record in registration_records:
+                    BaseOutputProcessor.augment_record(router_entry, registration_record, dhcp_lease_records)                
+
+                tx_byte_metrics = BaseCollector.counter_collector('wlan_clients_tx_bytes', 'Number of sent packet bytes', registration_records, 'tx_bytes', ['dhcp_name'])
+                yield tx_byte_metrics
+
+                rx_byte_metrics = BaseCollector.counter_collector('wlan_clients_rx_bytes', 'Number of received packet bytes', registration_records, 'rx_bytes', ['dhcp_name'])
+                yield rx_byte_metrics
+
+                signal_strength_metrics = BaseCollector.gauge_collector('wlan_clients_signal_strength', 'Average strength of the client signal recevied by AP', registration_records, 'signal_strength', ['dhcp_name'])
+                yield signal_strength_metrics
+
+                signal_to_noise_metrics = BaseCollector.gauge_collector('wlan_clients_signal_to_noise', 'Client devices signal to noise ratio', registration_records, 'signal_to_noise', ['dhcp_name'])
+                yield signal_to_noise_metrics
+
+                tx_ccq_metrics = BaseCollector.gauge_collector('wlan_clients_tx_ccq', 'Client Connection Quality (CCQ) for transmit', registration_records, 'tx_ccq', ['dhcp_name'])
+                yield tx_ccq_metrics
+
+                registration_metrics = BaseCollector.info_collector('wlan_clients_devices', 'Client devices info', 
+                                        registration_records, ['dhcp_name', 'dhcp_address', 'rx_signal', 'ssid', 'tx_rate', 'rx_rate', 'interface', 'mac_address', 'uptime'])
+                yield registration_metrics
+
+
+