From df84ada9c7e45d26c6f98809a1656a646096ef21 Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Sun, 14 Feb 2021 09:03:08 +0100 Subject: [PATCH] Collector registry, fixes/optimizations --- mktxp/cli/config/_mktxp.conf | 15 +- mktxp/cli/config/config.py | 155 ++++++++---------- mktxp/cli/dispatch.py | 15 +- mktxp/cli/options.py | 2 +- mktxp/cli/output/capsman_out.py | 2 +- mktxp/cli/output/dhcp_out.py | 2 +- mktxp/cli/output/wifi_out.py | 2 +- mktxp/collector/bandwidth_collector.py | 2 +- mktxp/collector/capsman_collector.py | 5 +- mktxp/collector/dhcp_collector.py | 3 + mktxp/collector/firewall_collector.py | 11 +- mktxp/collector/interface_collector.py | 3 + mktxp/collector/monitor_collector.py | 5 +- mktxp/collector/pool_collector.py | 3 + mktxp/collector/resource_collector.py | 2 +- mktxp/collector/route_collector.py | 3 + mktxp/collector/wlan_collector.py | 5 +- mktxp/datasource/firewall_ds.py | 4 +- mktxp/flow/collector_handler.py | 38 +++++ mktxp/flow/collector_registry.py | 60 +++++++ mktxp/flow/collectors_handler.py | 104 ------------ mktxp/{ => flow}/processor/__init__.py | 0 .../mktxp.py => flow/processor/base_proc.py} | 15 +- mktxp/{ => flow}/processor/output.py | 0 mktxp/flow/router_connection.py | 4 +- mktxp/flow/router_entries_handler.py | 4 +- mktxp/flow/router_entry.py | 5 +- setup.py | 2 +- 28 files changed, 237 insertions(+), 234 deletions(-) create mode 100644 mktxp/flow/collector_handler.py create mode 100644 mktxp/flow/collector_registry.py delete mode 100644 mktxp/flow/collectors_handler.py rename mktxp/{ => flow}/processor/__init__.py (100%) rename mktxp/{processor/mktxp.py => flow/processor/base_proc.py} (86%) rename mktxp/{ => flow}/processor/output.py (100%) diff --git a/mktxp/cli/config/_mktxp.conf b/mktxp/cli/config/_mktxp.conf index 1f987ae..fa7e1bc 100644 --- a/mktxp/cli/config/_mktxp.conf +++ b/mktxp/cli/config/_mktxp.conf @@ -12,10 +12,11 @@ [MKTXP] - port = 49090 # default metrics HTTP server port - bandwidth_test_interval = 420 # Interval between periodic bandwidth tests - - socket_timeout = 2 # Socket connection timeout - initial_delay_on_failure = 120 # Delay untill next connection attempt to a RouterOS Device - max_delay_on_failure = 900 # Max delay untill next connection attempt to a RouterOS Device - delay_inc_div = 5 # Delay increment factor + port = 49090 + socket_timeout = 2 + initial_delay_on_failure = 120 + max_delay_on_failure = 900 + delay_inc_div = 5 + bandwidth_test_interval = 420 + + collectors = IdentityCollector, SystemResourceCollector, HealthCollector, DHCPCollector, PoolCollector, InterfaceCollector, FirewallCollector, MonitorCollector, RouteCollector, WLANCollector, CapsmanCollector, MKTXPCollector \ No newline at end of file diff --git a/mktxp/cli/config/config.py b/mktxp/cli/config/config.py index 6c92341..972ae21 100755 --- a/mktxp/cli/config/config.py +++ b/mktxp/cli/config/config.py @@ -54,6 +54,7 @@ class MKTXPConfigKeys: MKTXP_MAX_DELAY = 'max_delay_on_failure' MKTXP_INC_DIV = 'delay_inc_div' MKTXP_BANDWIDTH_TEST_INTERVAL = 'bandwidth_test_interval' + MKTXP_COLLECTORS = 'collectors' # UnRegistered entries placeholder NO_ENTRIES_REGISTERED = 'NoEntriesRegistered' @@ -74,6 +75,8 @@ class MKTXPConfigKeys: DEFAULT_MKTXP_MAX_DELAY = 900 DEFAULT_MKTXP_INC_DIV = 5 DEFAULT_MKTXP_BANDWIDTH_TEST_INTERVAL = 420 + DEFAULT_MKTXP_COLLECTORS = ['IdentityCollector', 'SystemResourceCollector', 'HealthCollector', 'DHCPCollector', 'PoolCollector', + 'InterfaceCollector', 'FirewallCollector', 'MonitorCollector', 'RouteCollector', 'WLANCollector', 'CapsmanCollector', 'MKTXPCollector'] BOOLEAN_KEYS = (ENABLED_KEY, SSL_KEY, NO_SSL_CERTIFICATE, SSL_CERTIFICATE_VERIFY, FE_DHCP_KEY, FE_DHCP_LEASE_KEY, FE_DHCP_POOL_KEY, FE_INTERFACE_KEY, FE_FIREWALL_KEY, @@ -88,17 +91,17 @@ class MKTXPConfigKeys: class ConfigEntry: - MKTXPEntry = namedtuple('MKTXPEntry', [MKTXPConfigKeys.ENABLED_KEY, MKTXPConfigKeys.HOST_KEY, MKTXPConfigKeys.PORT_KEY, - MKTXPConfigKeys.USER_KEY, MKTXPConfigKeys.PASSWD_KEY, - MKTXPConfigKeys.SSL_KEY, MKTXPConfigKeys.NO_SSL_CERTIFICATE, MKTXPConfigKeys.SSL_CERTIFICATE_VERIFY, - - MKTXPConfigKeys.FE_DHCP_KEY, MKTXPConfigKeys.FE_DHCP_LEASE_KEY, MKTXPConfigKeys.FE_DHCP_POOL_KEY, MKTXPConfigKeys.FE_INTERFACE_KEY, MKTXPConfigKeys.FE_FIREWALL_KEY, - MKTXPConfigKeys.FE_MONITOR_KEY, MKTXPConfigKeys.FE_ROUTE_KEY, MKTXPConfigKeys.FE_WIRELESS_KEY, MKTXPConfigKeys.FE_WIRELESS_CLIENTS_KEY, - MKTXPConfigKeys.FE_CAPSMAN_KEY, MKTXPConfigKeys.FE_CAPSMAN_CLIENTS_KEY, MKTXPConfigKeys.MKTXP_USE_COMMENTS_OVER_NAMES - ]) - _MKTXPEntry = namedtuple('_MKTXPEntry', [MKTXPConfigKeys.PORT_KEY, MKTXPConfigKeys.MKTXP_SOCKET_TIMEOUT, - MKTXPConfigKeys.MKTXP_INITIAL_DELAY, MKTXPConfigKeys.MKTXP_MAX_DELAY, - MKTXPConfigKeys.MKTXP_INC_DIV, MKTXPConfigKeys.MKTXP_BANDWIDTH_TEST_INTERVAL]) + MKTXPConfigEntry = namedtuple('MKTXPConfigEntry', [MKTXPConfigKeys.ENABLED_KEY, MKTXPConfigKeys.HOST_KEY, MKTXPConfigKeys.PORT_KEY, + MKTXPConfigKeys.USER_KEY, MKTXPConfigKeys.PASSWD_KEY, + MKTXPConfigKeys.SSL_KEY, MKTXPConfigKeys.NO_SSL_CERTIFICATE, MKTXPConfigKeys.SSL_CERTIFICATE_VERIFY, + + MKTXPConfigKeys.FE_DHCP_KEY, MKTXPConfigKeys.FE_DHCP_LEASE_KEY, MKTXPConfigKeys.FE_DHCP_POOL_KEY, MKTXPConfigKeys.FE_INTERFACE_KEY, MKTXPConfigKeys.FE_FIREWALL_KEY, + MKTXPConfigKeys.FE_MONITOR_KEY, MKTXPConfigKeys.FE_ROUTE_KEY, MKTXPConfigKeys.FE_WIRELESS_KEY, MKTXPConfigKeys.FE_WIRELESS_CLIENTS_KEY, + MKTXPConfigKeys.FE_CAPSMAN_KEY, MKTXPConfigKeys.FE_CAPSMAN_CLIENTS_KEY, MKTXPConfigKeys.MKTXP_USE_COMMENTS_OVER_NAMES + ]) + MKTXPSystemEntry = namedtuple('MKTXPSystemEntry', [MKTXPConfigKeys.PORT_KEY, MKTXPConfigKeys.MKTXP_SOCKET_TIMEOUT, + MKTXPConfigKeys.MKTXP_INITIAL_DELAY, MKTXPConfigKeys.MKTXP_MAX_DELAY, + MKTXPConfigKeys.MKTXP_INC_DIV, MKTXPConfigKeys.MKTXP_BANDWIDTH_TEST_INTERVAL, MKTXPConfigKeys.MKTXP_COLLECTORS]) class OSConfig(metaclass = ABCMeta): @@ -158,9 +161,32 @@ class MKTXPConfigHandler: self._create_os_path(self.mktxp_conf_path, 'mktxp/cli/config/_mktxp.conf') self.re_compiled = {} - + self._read_from_disk() + # MKTXP entries + def registered_entries(self): + ''' All MKTXP registered entries + ''' + registered_entries = [entry_name for entry_name in self.config.keys()] + if not registered_entries: + registered_entries = [MKTXPConfigKeys.NO_ENTRIES_REGISTERED] + + return registered_entries + + def config_entry(self, entry_name): + ''' Given an entry name, reads and returns the entry info + ''' + entry_reader = self._config_entry_reader(entry_name) + return ConfigEntry.MKTXPConfigEntry(**entry_reader) + + def system_entry(self): + ''' MKTXP internal config entry + ''' + _entry_reader = self._system_entry_reader() + return ConfigEntry.MKTXPSystemEntry(**_entry_reader) + + # Helpers def _read_from_disk(self): ''' (Force-)Read conf data from disk ''' @@ -173,112 +199,67 @@ class MKTXPConfigHandler: lookup_path = resource_filename(Requirement.parse("mktxp"), resource_path) shutil.copy(lookup_path, os_path) - - # MKTXP entries - ############## - def register_entry(self, entry_name, entry_args, quiet = False): - ''' Registers MKTXP conf entry - ''' - if entry_name in self.registered_entries(): - if not quiet: - print('"{0}": entry name already registered'.format(entry_name)) - return False - else: - self.config[entry_name] = entry_args - self.config.write() - if not quiet: - print('Entry registered: {0}'.format(entry_name)) - return True - - def unregister_entry(self, entry_name, quiet = False): - ''' Un-registers MKTXP conf entry - ''' - if self.config[entry_name]: - del(self.config[entry_name]) - self.config.write() - if not quiet: - print('Unregistered entry: {}'.format(entry_name)) - return True - else: - if not quiet: - print('Entry is not registered: {}'.format(entry_name)) - return False - - def registered_entries(self): - ''' All MKTXP registered entries - ''' - registered_entries = [entry_name for entry_name in self.config.keys()] - if not registered_entries: - registered_entries = [MKTXPConfigKeys.NO_ENTRIES_REGISTERED] - - return registered_entries - - def entry(self, entry_name): - ''' Given an entry name, reads and returns the entry info - ''' - entry_reader = self.entry_reader(entry_name) - return ConfigEntry.MKTXPEntry(**entry_reader) - - def _entry(self): - ''' MKTXP internal config entry - ''' - _entry_reader = self._entry_reader() - return ConfigEntry._MKTXPEntry(**_entry_reader) - - - # Helpers - def entry_reader(self, entry_name): - entry_reader = {} + def _config_entry_reader(self, entry_name): + config_entry_reader = {} write_needed = False for key in MKTXPConfigKeys.BOOLEAN_KEYS: if self.config[entry_name].get(key): - entry_reader[key] = self.config[entry_name].as_bool(key) + config_entry_reader[key] = self.config[entry_name].as_bool(key) else: - entry_reader[key] = False + config_entry_reader[key] = False write_needed = True # read from disk next time for key in MKTXPConfigKeys.STR_KEYS: - entry_reader[key] = self.config[entry_name][key] + config_entry_reader[key] = self.config[entry_name][key] # port if self.config[entry_name].get(MKTXPConfigKeys.PORT_KEY): - entry_reader[MKTXPConfigKeys.PORT_KEY] = self.config[entry_name].as_int(MKTXPConfigKeys.PORT_KEY) + config_entry_reader[MKTXPConfigKeys.PORT_KEY] = self.config[entry_name].as_int(MKTXPConfigKeys.PORT_KEY) else: - entry_reader[MKTXPConfigKeys.PORT_KEY] = self._default_value_for_key(MKTXPConfigKeys.SSL_KEY, entry_reader[MKTXPConfigKeys.SSL_KEY]) + config_entry_reader[MKTXPConfigKeys.PORT_KEY] = self._default_value_for_key(MKTXPConfigKeys.SSL_KEY, config_entry_reader[MKTXPConfigKeys.SSL_KEY]) write_needed = True # read from disk next time if write_needed: - self.config[entry_name] = entry_reader + self.config[entry_name] = config_entry_reader self.config.write() - return entry_reader + return config_entry_reader - def _entry_reader(self): - _entry_reader = {} + def _system_entry_reader(self): + system_entry_reader = {} entry_name = MKTXPConfigKeys.MKTXP_CONFIG_ENTRY_NAME write_needed = False + for key in MKTXPConfigKeys.MKTXP_INT_KEYS: if self._config[entry_name].get(key): - _entry_reader[key] = self._config[entry_name].as_int(key) + system_entry_reader[key] = self._config[entry_name].as_int(key) else: - _entry_reader[key] = self._default_value_for_key(key) + system_entry_reader[key] = self._default_value_for_key(key) write_needed = True # read from disk next time + # Collectors + if self._config[entry_name].get(MKTXPConfigKeys.MKTXP_COLLECTORS): + system_entry_reader[MKTXPConfigKeys.MKTXP_COLLECTORS] = self._config[entry_name][MKTXPConfigKeys.MKTXP_COLLECTORS] + else: + system_entry_reader[MKTXPConfigKeys.MKTXP_COLLECTORS] = self._default_value_for_key(MKTXPConfigKeys.MKTXP_COLLECTORS) + write_needed = True # read from disk next time + if write_needed: - self._config[entry_name] = _entry_reader + self._config[entry_name] = system_entry_reader self._config.write() - return _entry_reader + return system_entry_reader def _default_value_for_key(self, key, value = None): return { MKTXPConfigKeys.SSL_KEY: lambda value: MKTXPConfigKeys.DEFAULT_API_SSL_PORT if value else MKTXPConfigKeys.DEFAULT_API_PORT, - MKTXPConfigKeys.PORT_KEY: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_PORT, - MKTXPConfigKeys.MKTXP_SOCKET_TIMEOUT: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_SOCKET_TIMEOUT, - MKTXPConfigKeys.MKTXP_INITIAL_DELAY: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_INITIAL_DELAY, - MKTXPConfigKeys.MKTXP_MAX_DELAY: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_MAX_DELAY, - MKTXPConfigKeys.MKTXP_INC_DIV: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_INC_DIV, - MKTXPConfigKeys.MKTXP_BANDWIDTH_TEST_INTERVAL: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_BANDWIDTH_TEST_INTERVAL + MKTXPConfigKeys.PORT_KEY: MKTXPConfigKeys.DEFAULT_MKTXP_PORT, + MKTXPConfigKeys.MKTXP_SOCKET_TIMEOUT: MKTXPConfigKeys.DEFAULT_MKTXP_SOCKET_TIMEOUT, + MKTXPConfigKeys.MKTXP_INITIAL_DELAY: MKTXPConfigKeys.DEFAULT_MKTXP_INITIAL_DELAY, + MKTXPConfigKeys.MKTXP_MAX_DELAY: MKTXPConfigKeys.DEFAULT_MKTXP_MAX_DELAY, + MKTXPConfigKeys.MKTXP_INC_DIV: MKTXPConfigKeys.DEFAULT_MKTXP_INC_DIV, + MKTXPConfigKeys.MKTXP_BANDWIDTH_TEST_INTERVAL: MKTXPConfigKeys.DEFAULT_MKTXP_BANDWIDTH_TEST_INTERVAL, + MKTXPConfigKeys.MKTXP_COLLECTORS: MKTXPConfigKeys.DEFAULT_MKTXP_COLLECTORS }[key](value) diff --git a/mktxp/cli/dispatch.py b/mktxp/cli/dispatch.py index f6e84cc..43208d0 100755 --- a/mktxp/cli/dispatch.py +++ b/mktxp/cli/dispatch.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!.usr/bin/env python # coding=utf8 ## Copyright (c) 2020 Arseniy Kuznetsov ## @@ -16,7 +16,7 @@ import subprocess from mktxp.cli.config.config import config_handler from mktxp.cli.options import MKTXPOptionsParser, MKTXPCommands -from mktxp.processor.mktxp import MKTXPProcessor, MKTXPCLIProcessor +from mktxp.flow.processor.base_proc import ExportProcessor, OutputProcessor class MKTXPDispatcher: @@ -60,12 +60,11 @@ class MKTXPDispatcher: if args['config']: print(f'MKTXP data config: {config_handler.usr_conf_data_path}') print(f'MKTXP internal config: {config_handler.mktxp_conf_path}') - else: for entryname in config_handler.registered_entries(): if args['entry_name'] and entryname != args['entry_name']: continue - entry = config_handler.entry(entryname) + entry = config_handler.config_entry(entryname) print(f'[{entryname}]') divider_fields = set(['username', 'use_ssl', 'dhcp']) for field in entry._fields: @@ -87,20 +86,20 @@ class MKTXPDispatcher: subprocess.check_call([editor, config_handler.usr_conf_data_path]) def start_export(self, args): - MKTXPProcessor.start() + ExportProcessor.start() def print(self, args): if not (args['wifi_clients'] or args['capsman_clients']): print("Select metric option(s) to print out, or run 'mktxp print -h' to find out more") if args['wifi_clients']: - MKTXPCLIProcessor.wifi_clients(args['entry_name']) + OutputProcessor.wifi_clients(args['entry_name']) if args['capsman_clients']: - MKTXPCLIProcessor.capsman_clients(args['entry_name']) + OutputProcessor.capsman_clients(args['entry_name']) if args['dhcp_clients']: - MKTXPCLIProcessor.dhcp_clients(args['entry_name']) + OutputProcessor.dhcp_clients(args['entry_name']) def main(): diff --git a/mktxp/cli/options.py b/mktxp/cli/options.py index 373ca3d..899aff3 100755 --- a/mktxp/cli/options.py +++ b/mktxp/cli/options.py @@ -150,7 +150,7 @@ Selected metrics info can be printed on the command line. For more information, args['entry_name'] = UniquePartialMatchList(config_handler.registered_entries()).find(args['entry_name']) if args['sub_cmd'] == MKTXPCommands.PRINT: - if not config_handler.entry(args['entry_name']).enabled: + if not config_handler.config_entry(args['entry_name']).enabled: print(f"Can not print metrics for disabled RouterOS entry: {args['entry_name']}\nRun 'mktxp edit' to review and enable it in the configuration file first") parser.exit() diff --git a/mktxp/cli/output/capsman_out.py b/mktxp/cli/output/capsman_out.py index f378fce..db407e8 100644 --- a/mktxp/cli/output/capsman_out.py +++ b/mktxp/cli/output/capsman_out.py @@ -12,7 +12,7 @@ ## GNU General Public License for more details. -from mktxp.processor.output import BaseOutputProcessor +from mktxp.flow.processor.output import BaseOutputProcessor from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource from mktxp.datasource.capsman_ds import CapsmanRegistrationsMetricsDataSource diff --git a/mktxp/cli/output/dhcp_out.py b/mktxp/cli/output/dhcp_out.py index e1e0aca..a5f0961 100644 --- a/mktxp/cli/output/dhcp_out.py +++ b/mktxp/cli/output/dhcp_out.py @@ -12,7 +12,7 @@ ## GNU General Public License for more details. -from mktxp.processor.output import BaseOutputProcessor +from mktxp.flow.processor.output import BaseOutputProcessor from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource diff --git a/mktxp/cli/output/wifi_out.py b/mktxp/cli/output/wifi_out.py index c7ba110..054acf0 100644 --- a/mktxp/cli/output/wifi_out.py +++ b/mktxp/cli/output/wifi_out.py @@ -12,7 +12,7 @@ ## GNU General Public License for more details. -from mktxp.processor.output import BaseOutputProcessor +from mktxp.flow.processor.output import BaseOutputProcessor from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource from mktxp.datasource.wireless_ds import WirelessMetricsDataSource diff --git a/mktxp/collector/bandwidth_collector.py b/mktxp/collector/bandwidth_collector.py index 52aa508..7c63bdb 100644 --- a/mktxp/collector/bandwidth_collector.py +++ b/mktxp/collector/bandwidth_collector.py @@ -46,7 +46,7 @@ class BandwidthCollector(BaseCollector): yield latency_metrics ts = datetime.now().timestamp() - if (ts - self.last_call_timestamp) > config_handler._entry().bandwidth_test_interval: + if (ts - self.last_call_timestamp) > config_handler.system_entry().bandwidth_test_interval: self.pool.apply_async(BandwidthCollector.bandwidth_worker, callback=get_result) self.last_call_timestamp = ts diff --git a/mktxp/collector/capsman_collector.py b/mktxp/collector/capsman_collector.py index 48e0722..9b5fcc8 100644 --- a/mktxp/collector/capsman_collector.py +++ b/mktxp/collector/capsman_collector.py @@ -13,7 +13,7 @@ from mktxp.cli.config.config import MKTXPConfigKeys -from mktxp.processor.output import BaseOutputProcessor +from mktxp.flow.processor.output import BaseOutputProcessor from mktxp.collector.base_collector import BaseCollector from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource from mktxp.datasource.capsman_ds import CapsmanCapsMetricsDataSource, CapsmanRegistrationsMetricsDataSource @@ -24,6 +24,9 @@ class CapsmanCollector(BaseCollector): ''' @staticmethod def collect(router_entry): + if not router_entry.config_entry.capsman: + return + remote_caps_labels = ['identity', 'version', 'base_mac', 'board', 'base_mac'] remote_caps_records = CapsmanCapsMetricsDataSource.metric_records(router_entry, metric_labels = remote_caps_labels) if remote_caps_records: diff --git a/mktxp/collector/dhcp_collector.py b/mktxp/collector/dhcp_collector.py index 87e934d..4c4e102 100644 --- a/mktxp/collector/dhcp_collector.py +++ b/mktxp/collector/dhcp_collector.py @@ -22,6 +22,9 @@ class DHCPCollector(BaseCollector): ''' @staticmethod def collect(router_entry): + if not router_entry.config_entry.dhcp: + return + dhcp_lease_labels = ['active_address', 'address', 'mac_address', 'host_name', 'comment', 'server', 'expires_after'] dhcp_lease_records = DHCPMetricsDataSource.metric_records(router_entry, metric_labels = dhcp_lease_labels) if dhcp_lease_records: diff --git a/mktxp/collector/firewall_collector.py b/mktxp/collector/firewall_collector.py index 2354ac4..50b7180 100644 --- a/mktxp/collector/firewall_collector.py +++ b/mktxp/collector/firewall_collector.py @@ -22,19 +22,22 @@ class FirewallCollector(BaseCollector): ''' @staticmethod def collect(router_entry): + if not router_entry.config_entry.firewall: + return + # initialize all pool counts, including those currently not used - firewall_labels = ['chain', 'action', 'bytes', 'comment'] + firewall_labels = ['chain', 'action', 'bytes', 'comment', 'log'] firewall_filter_records = FirewallMetricsDataSource.metric_records(router_entry, metric_labels = firewall_labels) if firewall_filter_records: metris_records = [FirewallCollector.metric_record(router_entry, record) for record in firewall_filter_records] - firewall_filter_metrics = BaseCollector.counter_collector('firewall_filter', 'Total amount of bytes matched by firewall rules', metris_records, 'bytes', ['name']) + firewall_filter_metrics = BaseCollector.counter_collector('firewall_filter', 'Total amount of bytes matched by firewall rules', metris_records, 'bytes', ['name', 'log']) yield firewall_filter_metrics firewall_raw_records = FirewallMetricsDataSource.metric_records(router_entry, metric_labels = firewall_labels, raw = True) if firewall_raw_records: metris_records = [FirewallCollector.metric_record(router_entry, record) for record in firewall_raw_records] - firewall_raw_metrics = BaseCollector.counter_collector('firewall_raw', 'Total amount of bytes matched by raw firewall rules', metris_records, 'bytes', ['name']) + firewall_raw_metrics = BaseCollector.counter_collector('firewall_raw', 'Total amount of bytes matched by raw firewall rules', metris_records, 'bytes', ['name', 'log']) yield firewall_raw_metrics # Helpers @@ -44,4 +47,4 @@ class FirewallCollector(BaseCollector): bytes = firewall_record['bytes'] return {MKTXPConfigKeys.ROUTERBOARD_NAME: router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME], MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS], - 'name': name, 'bytes': bytes} + 'name': name, 'log': firewall_record['log'], 'bytes': bytes} diff --git a/mktxp/collector/interface_collector.py b/mktxp/collector/interface_collector.py index 4cf8f9a..7fe341c 100644 --- a/mktxp/collector/interface_collector.py +++ b/mktxp/collector/interface_collector.py @@ -21,6 +21,9 @@ class InterfaceCollector(BaseCollector): ''' @staticmethod def collect(router_entry): + if not router_entry.config_entry.interface: + return + interface_traffic_labels = ['name', 'comment', 'rx_byte', 'tx_byte', 'rx_packet', 'tx_packet', 'rx_error', 'tx_error', 'rx_drop', 'tx_drop'] interface_traffic_records = InterfaceTrafficMetricsDataSource.metric_records(router_entry, metric_labels = interface_traffic_labels) diff --git a/mktxp/collector/monitor_collector.py b/mktxp/collector/monitor_collector.py index f247d1a..81908c6 100644 --- a/mktxp/collector/monitor_collector.py +++ b/mktxp/collector/monitor_collector.py @@ -13,7 +13,7 @@ from mktxp.collector.base_collector import BaseCollector -from mktxp.processor.output import BaseOutputProcessor +from mktxp.flow.processor.output import BaseOutputProcessor from mktxp.datasource.interface_ds import InterfaceMonitorMetricsDataSource @@ -22,6 +22,9 @@ class MonitorCollector(BaseCollector): ''' @staticmethod def collect(router_entry): + if not router_entry.config_entry.monitor: + return + monitor_labels = ('status', 'rate', 'full_duplex', 'name') monitor_records = InterfaceMonitorMetricsDataSource.metric_records(router_entry, metric_labels = monitor_labels, include_comments = True) if monitor_records: diff --git a/mktxp/collector/pool_collector.py b/mktxp/collector/pool_collector.py index 2af111c..88e58e6 100644 --- a/mktxp/collector/pool_collector.py +++ b/mktxp/collector/pool_collector.py @@ -22,6 +22,9 @@ class PoolCollector(BaseCollector): ''' @staticmethod def collect(router_entry): + if not router_entry.config_entry.pool: + return + # initialize all pool counts, including those currently not used pool_records = PoolMetricsDataSource.metric_records(router_entry, metric_labels = ['name']) if pool_records: diff --git a/mktxp/collector/resource_collector.py b/mktxp/collector/resource_collector.py index 831b0f7..e37fb6b 100644 --- a/mktxp/collector/resource_collector.py +++ b/mktxp/collector/resource_collector.py @@ -13,7 +13,7 @@ from mktxp.collector.base_collector import BaseCollector -from mktxp.processor.output import BaseOutputProcessor +from mktxp.flow.processor.output import BaseOutputProcessor from mktxp.datasource.system_resource_ds import SystemResourceMetricsDataSource diff --git a/mktxp/collector/route_collector.py b/mktxp/collector/route_collector.py index 269e816..f908609 100644 --- a/mktxp/collector/route_collector.py +++ b/mktxp/collector/route_collector.py @@ -22,6 +22,9 @@ class RouteCollector(BaseCollector): ''' @staticmethod def collect(router_entry): + if not router_entry.config_entry.route: + return + route_labels = ['connect', 'dynamic', 'static', 'bgp', 'ospf'] route_records = RouteMetricsDataSource.metric_records(router_entry, metric_labels = route_labels) if route_records: diff --git a/mktxp/collector/wlan_collector.py b/mktxp/collector/wlan_collector.py index 9f3e973..2ecec5c 100644 --- a/mktxp/collector/wlan_collector.py +++ b/mktxp/collector/wlan_collector.py @@ -12,7 +12,7 @@ ## GNU General Public License for more details. -from mktxp.processor.output import BaseOutputProcessor +from mktxp.flow.processor.output import BaseOutputProcessor from mktxp.collector.base_collector import BaseCollector from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource from mktxp.datasource.wireless_ds import WirelessMetricsDataSource @@ -24,6 +24,9 @@ class WLANCollector(BaseCollector): ''' @staticmethod def collect(router_entry): + if not router_entry.config_entry.wireless: + return + monitor_labels = ['channel', 'noise_floor', 'overall_tx_ccq', 'registered_clients'] monitor_records = InterfaceMonitorMetricsDataSource.metric_records(router_entry, metric_labels = monitor_labels, kind = 'wireless') if monitor_records: diff --git a/mktxp/datasource/firewall_ds.py b/mktxp/datasource/firewall_ds.py index 43ef67a..b18fa06 100644 --- a/mktxp/datasource/firewall_ds.py +++ b/mktxp/datasource/firewall_ds.py @@ -29,7 +29,9 @@ class FirewallMetricsDataSource: # translation rules translation_table = {} if 'comment' in metric_labels: - translation_table['comment'] = lambda c: c if c else '' + translation_table['comment'] = lambda value: value if value else '' + if 'log' in metric_labels: + translation_table['log'] = lambda value: '1' if value == 'true' else '0' return BaseDSProcessor.trimmed_records(router_entry, router_records = firewall_records, metric_labels = metric_labels, translation_table = translation_table) except Exception as exc: diff --git a/mktxp/flow/collector_handler.py b/mktxp/flow/collector_handler.py new file mode 100644 index 0000000..2ff8072 --- /dev/null +++ b/mktxp/flow/collector_handler.py @@ -0,0 +1,38 @@ +# coding=utf8 +## Copyright (c) 2020 Arseniy Kuznetsov +## +## This program is free software; you can redistribute it and/or +## modify it under the terms of the GNU General Public License +## as published by the Free Software Foundation; either version 2 +## of the License, or (at your option) any later version. +## +## This program is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. + + +from timeit import default_timer + + +class CollectorHandler: + ''' MKTXP Collectors Handler + ''' + def __init__(self, entries_handler, collector_registry): + self.entries_handler = entries_handler + self.collector_registry = collector_registry + + def collect(self): + yield from self.collector_registry.bandwidthCollector.collect() + + for router_entry in self.entries_handler.router_entries: + if not router_entry.api_connection.is_connected(): + # let's pick up on things in the next run + router_entry.api_connection.connect() + continue + + for collector_ID, collect_func in self.collector_registry.registered_collectors.items(): + start = default_timer() + yield from collect_func(router_entry) + router_entry.time_spent[collector_ID] += default_timer() - start + diff --git a/mktxp/flow/collector_registry.py b/mktxp/flow/collector_registry.py new file mode 100644 index 0000000..efc5f9b --- /dev/null +++ b/mktxp/flow/collector_registry.py @@ -0,0 +1,60 @@ +# coding=utf8 +## Copyright (c) 2020 Arseniy Kuznetsov +## +## This program is free software; you can redistribute it and/or +## modify it under the terms of the GNU General Public License +## as published by the Free Software Foundation; either version 2 +## of the License, or (at your option) any later version. +## +## This program is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. + + +from collections import OrderedDict +from mktxp.cli.config.config import MKTXPConfigKeys +from mktxp.collector.dhcp_collector import DHCPCollector +from mktxp.collector.interface_collector import InterfaceCollector +from mktxp.collector.health_collector import HealthCollector +from mktxp.collector.identity_collector import IdentityCollector +from mktxp.collector.monitor_collector import MonitorCollector +from mktxp.collector.pool_collector import PoolCollector +from mktxp.collector.resource_collector import SystemResourceCollector +from mktxp.collector.route_collector import RouteCollector +from mktxp.collector.wlan_collector import WLANCollector +from mktxp.collector.capsman_collector import CapsmanCollector +from mktxp.collector.bandwidth_collector import BandwidthCollector +from mktxp.collector.firewall_collector import FirewallCollector +from mktxp.collector.mktxp_collector import MKTXPCollector + + +class CollectorRegistry: + ''' MKTXP Collectors Registry + ''' + def __init__(self): + self.registered_collectors = OrderedDict() + + # bandwidth collector is not router-entry related, so registering directly + self.bandwidthCollector = BandwidthCollector() + + self.register('IdentityCollector', IdentityCollector.collect) + self.register('SystemResourceCollector', SystemResourceCollector.collect) + self.register('HealthCollector', HealthCollector.collect) + + self.register('DHCPCollector', DHCPCollector.collect) + self.register('PoolCollector', PoolCollector.collect) + self.register('InterfaceCollector', InterfaceCollector.collect) + + self.register('FirewallCollector', FirewallCollector.collect) + self.register('MonitorCollector', MonitorCollector.collect) + self.register('RouteCollector', RouteCollector.collect) + + self.register('WLANCollector', WLANCollector.collect) + self.register('CapsmanCollector', CapsmanCollector.collect) + + self.register('MKTXPCollector', MKTXPCollector.collect) + + def register(self, collector_ID, collect_func): + self.registered_collectors[collector_ID] = collect_func + diff --git a/mktxp/flow/collectors_handler.py b/mktxp/flow/collectors_handler.py deleted file mode 100644 index bbcb4b7..0000000 --- a/mktxp/flow/collectors_handler.py +++ /dev/null @@ -1,104 +0,0 @@ -# coding=utf8 -## Copyright (c) 2020 Arseniy Kuznetsov -## -## This program is free software; you can redistribute it and/or -## modify it under the terms of the GNU General Public License -## as published by the Free Software Foundation; either version 2 -## of the License, or (at your option) any later version. -## -## This program is distributed in the hope that it will be useful, -## but WITHOUT ANY WARRANTY; without even the implied warranty of -## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -## GNU General Public License for more details. - - -from timeit import default_timer -from mktxp.collector.dhcp_collector import DHCPCollector -from mktxp.collector.interface_collector import InterfaceCollector -from mktxp.collector.health_collector import HealthCollector -from mktxp.collector.identity_collector import IdentityCollector -from mktxp.collector.monitor_collector import MonitorCollector -from mktxp.collector.pool_collector import PoolCollector -from mktxp.collector.resource_collector import SystemResourceCollector -from mktxp.collector.route_collector import RouteCollector -from mktxp.collector.wlan_collector import WLANCollector -from mktxp.collector.capsman_collector import CapsmanCollector -from mktxp.collector.bandwidth_collector import BandwidthCollector -from mktxp.collector.firewall_collector import FirewallCollector -from mktxp.collector.mktxp_collector import MKTXPCollector - - -class CollectorsHandler: - ''' MKTXP Collectors Handler - ''' - def __init__(self, entries_handler): - self.entries_handler = entries_handler - self.bandwidthCollector = BandwidthCollector() - - def collect(self): - # process mktxp internal metrics - yield from self.bandwidthCollector.collect() - - for router_entry in self.entries_handler.router_entries: - if not router_entry.api_connection.is_connected(): - # let's pick up on things in the next run - router_entry.api_connection.connect() - continue - - start = default_timer() - yield from IdentityCollector.collect(router_entry) - router_entry.time_spent['IdentityCollector'] += default_timer() - start - - start = default_timer() - yield from SystemResourceCollector.collect(router_entry) - router_entry.time_spent['SystemResourceCollector'] += default_timer() - start - - start = default_timer() - yield from HealthCollector.collect(router_entry) - router_entry.time_spent['HealthCollector'] += default_timer() - start - - if router_entry.config_entry.dhcp: - start = default_timer() - yield from DHCPCollector.collect(router_entry) - router_entry.time_spent['DHCPCollector'] += default_timer() - start - - if router_entry.config_entry.pool: - start = default_timer() - yield from PoolCollector.collect(router_entry) - router_entry.time_spent['PoolCollector'] += default_timer() - start - - if router_entry.config_entry.interface: - start = default_timer() - yield from InterfaceCollector.collect(router_entry) - router_entry.time_spent['InterfaceCollector'] += default_timer() - start - - if router_entry.config_entry.firewall: - start = default_timer() - yield from FirewallCollector.collect(router_entry) - router_entry.time_spent['FirewallCollector'] += default_timer() - start - - if router_entry.config_entry.monitor: - start = default_timer() - yield from MonitorCollector.collect(router_entry) - router_entry.time_spent['MonitorCollector'] += default_timer() - start - - if router_entry.config_entry.route: - start = default_timer() - yield from RouteCollector.collect(router_entry) - router_entry.time_spent['RouteCollector'] += default_timer() - start - - if router_entry.config_entry.wireless: - start = default_timer() - yield from WLANCollector.collect(router_entry) - router_entry.time_spent['WLANCollector'] += default_timer() - start - - if router_entry.config_entry.capsman: - start = default_timer() - yield from CapsmanCollector.collect(router_entry) - router_entry.time_spent['CapsmanCollector'] += default_timer() - start - - yield from MKTXPCollector.collect(router_entry) - - - - diff --git a/mktxp/processor/__init__.py b/mktxp/flow/processor/__init__.py similarity index 100% rename from mktxp/processor/__init__.py rename to mktxp/flow/processor/__init__.py diff --git a/mktxp/processor/mktxp.py b/mktxp/flow/processor/base_proc.py similarity index 86% rename from mktxp/processor/mktxp.py rename to mktxp/flow/processor/base_proc.py index 2d18cd5..af2f497 100644 --- a/mktxp/processor/mktxp.py +++ b/mktxp/flow/processor/base_proc.py @@ -16,8 +16,10 @@ from http.server import HTTPServer from datetime import datetime from prometheus_client.core import REGISTRY from prometheus_client import MetricsHandler + from mktxp.cli.config.config import config_handler -from mktxp.flow.collectors_handler import CollectorsHandler +from mktxp.flow.collector_handler import CollectorHandler +from mktxp.flow.collector_registry import CollectorRegistry from mktxp.flow.router_entries_handler import RouterEntriesHandler from mktxp.cli.output.capsman_out import CapsmanOutput @@ -25,17 +27,16 @@ from mktxp.cli.output.wifi_out import WirelessOutput from mktxp.cli.output.dhcp_out import DHCPOutput -class MKTXPProcessor: +class ExportProcessor: ''' Base Export Processing ''' @staticmethod def start(): - router_entries_handler = RouterEntriesHandler() - REGISTRY.register(CollectorsHandler(router_entries_handler)) - MKTXPProcessor.run(port=config_handler._entry().port) + REGISTRY.register(CollectorHandler(RouterEntriesHandler(), CollectorRegistry())) + ExportProcessor.run(port=config_handler.system_entry().port) @staticmethod - def run(server_class=HTTPServer, handler_class=MetricsHandler, port = None): + def run(server_class=HTTPServer, handler_class=MetricsHandler, port=None): server_address = ('', port) httpd = server_class(server_address, handler_class) current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") @@ -43,7 +44,7 @@ class MKTXPProcessor: httpd.serve_forever() -class MKTXPCLIProcessor: +class OutputProcessor: ''' Base CLI Processing ''' @staticmethod diff --git a/mktxp/processor/output.py b/mktxp/flow/processor/output.py similarity index 100% rename from mktxp/processor/output.py rename to mktxp/flow/processor/output.py diff --git a/mktxp/flow/router_connection.py b/mktxp/flow/router_connection.py index 16f7e86..d355bcb 100644 --- a/mktxp/flow/router_connection.py +++ b/mktxp/flow/router_connection.py @@ -46,7 +46,7 @@ class RouterAPIConnection: ssl_verify = self.config_entry.ssl_certificate_verify, ssl_context = ctx) - self.connection.socket_timeout = config_handler._entry().socket_timeout + self.connection.socket_timeout = config_handler.system_entry().socket_timeout self.api = None def is_connected(self): @@ -91,7 +91,7 @@ class RouterAPIConnection: return False def _connect_delay(self): - mktxp_entry = config_handler._entry() + mktxp_entry = config_handler.system_entry() connect_delay = (1 + self.successive_failure_count / mktxp_entry.delay_inc_div) * mktxp_entry.initial_delay_on_failure return connect_delay if connect_delay < mktxp_entry.max_delay_on_failure else mktxp_entry.max_delay_on_failure diff --git a/mktxp/flow/router_entries_handler.py b/mktxp/flow/router_entries_handler.py index 4816876..012cee6 100644 --- a/mktxp/flow/router_entries_handler.py +++ b/mktxp/flow/router_entries_handler.py @@ -22,7 +22,7 @@ class RouterEntriesHandler: def __init__(self): self.router_entries = [] for router_name in config_handler.registered_entries(): - entry = config_handler.entry(router_name) + entry = config_handler.config_entry(router_name) if entry.enabled: self.router_entries.append(RouterEntry(router_name)) @@ -32,7 +32,7 @@ class RouterEntriesHandler: for router_name in config_handler.registered_entries(): if router_name == entry_name: if enabled_only: - entry = config_handler.entry(router_name) + entry = config_handler.config_entry(router_name) if not entry.enabled: break router_entry = RouterEntry(router_name) diff --git a/mktxp/flow/router_entry.py b/mktxp/flow/router_entry.py index 72d181d..2e51925 100644 --- a/mktxp/flow/router_entry.py +++ b/mktxp/flow/router_entry.py @@ -21,7 +21,7 @@ class RouterEntry: ''' def __init__(self, router_name): self.router_name = router_name - self.config_entry = config_handler.entry(router_name) + self.config_entry = config_handler.config_entry(router_name) self.api_connection = RouterAPIConnection(router_name, self.config_entry) self.router_id = { MKTXPConfigKeys.ROUTERBOARD_NAME: self.router_name, @@ -37,5 +37,6 @@ class RouterEntry: 'MonitorCollector': 0, 'RouteCollector': 0, 'WLANCollector': 0, - 'CapsmanCollector': 0 + 'CapsmanCollector': 0, + 'MKTXPCollector': 0 } diff --git a/setup.py b/setup.py index 133c8f4..7479872 100755 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ with open(path.join(pkg_dir, 'README.md'), encoding='utf-8') as f: setup( name='mktxp', - version='0.26', + version='0.28', url='https://github.com/akpw/mktxp',