From 408dd715c5055894d7f12b74c9c868a251caf537 Mon Sep 17 00:00:00 2001 From: Wim Fournier Date: Thu, 2 Nov 2023 21:22:48 +0100 Subject: [PATCH 1/4] Added metrics for kid-control devices --- mktxp/cli/config/config.py | 7 +- .../collector/kid_control_device_collector.py | 91 +++++++++++++++++++ mktxp/datasource/kid_control_device_ds.py | 31 +++++++ mktxp/flow/collector_registry.py | 3 + mktxp/flow/router_entry.py | 3 +- 5 files changed, 132 insertions(+), 3 deletions(-) create mode 100644 mktxp/collector/kid_control_device_collector.py create mode 100644 mktxp/datasource/kid_control_device_ds.py diff --git a/mktxp/cli/config/config.py b/mktxp/cli/config/config.py index f7dd791..7315042 100755 --- a/mktxp/cli/config/config.py +++ b/mktxp/cli/config/config.py @@ -44,6 +44,7 @@ class CollectorKeys: CAPSMAN_COLLECTOR = 'CapsmanCollector' QUEUE_TREE_COLLECTOR = 'QueueTreeCollector' QUEUE_SIMPLE_COLLECTOR = 'QueueSimpleCollector' + KID_CONTROL_DEVICE_COLLECTOR = 'KidControlCollector' USER_COLLECTOR = 'UserCollector' MKTXP_COLLECTOR = 'MKTXPCollector' @@ -90,6 +91,8 @@ class MKTXPConfigKeys: FE_CHECK_FOR_UPDATES = 'check_for_updates' + FE_KID_CONTROL_DEVICE = 'kid_control_devices' + MKTXP_SOCKET_TIMEOUT = 'socket_timeout' MKTXP_INITIAL_DELAY = 'initial_delay_on_failure' MKTXP_MAX_DELAY = 'max_delay_on_failure' @@ -129,7 +132,7 @@ class MKTXPConfigKeys: DEFAULT_MKTXP_TOTAL_MAX_SCRAPE_DURATION = 30 - BOOLEAN_KEYS_NO = {ENABLED_KEY, SSL_KEY, NO_SSL_CERTIFICATE, FE_CHECK_FOR_UPDATES, + BOOLEAN_KEYS_NO = {ENABLED_KEY, SSL_KEY, NO_SSL_CERTIFICATE, FE_CHECK_FOR_UPDATES, FE_KID_CONTROL_DEVICE, SSL_CERTIFICATE_VERIFY, FE_IPV6_FIREWALL_KEY, FE_IPV6_NEIGHBOR_KEY, FE_CONNECTION_STATS_KEY} # Feature keys enabled by default @@ -159,7 +162,7 @@ class ConfigEntry: MKTXPConfigKeys.FE_FIREWALL_KEY, MKTXPConfigKeys.FE_MONITOR_KEY, MKTXPConfigKeys.FE_ROUTE_KEY, MKTXPConfigKeys.FE_WIRELESS_KEY, MKTXPConfigKeys.FE_WIRELESS_CLIENTS_KEY, MKTXPConfigKeys.FE_IP_CONNECTIONS_KEY, MKTXPConfigKeys.FE_CONNECTION_STATS_KEY, MKTXPConfigKeys.FE_CAPSMAN_KEY, MKTXPConfigKeys.FE_CAPSMAN_CLIENTS_KEY, MKTXPConfigKeys.FE_POE_KEY, MKTXPConfigKeys.FE_NETWATCH_KEY, MKTXPConfigKeys.MKTXP_USE_COMMENTS_OVER_NAMES, MKTXPConfigKeys.FE_PUBLIC_IP_KEY, MKTXPConfigKeys.FE_IPV6_FIREWALL_KEY, MKTXPConfigKeys.FE_IPV6_NEIGHBOR_KEY, - MKTXPConfigKeys.FE_USER_KEY, MKTXPConfigKeys.FE_QUEUE_KEY, MKTXPConfigKeys.FE_REMOTE_DHCP_ENTRY, MKTXPConfigKeys.FE_CHECK_FOR_UPDATES + MKTXPConfigKeys.FE_USER_KEY, MKTXPConfigKeys.FE_QUEUE_KEY, MKTXPConfigKeys.FE_REMOTE_DHCP_ENTRY, MKTXPConfigKeys.FE_CHECK_FOR_UPDATES, MKTXPConfigKeys.FE_KID_CONTROL_DEVICE, ]) MKTXPSystemEntry = namedtuple('MKTXPSystemEntry', [MKTXPConfigKeys.PORT_KEY, MKTXPConfigKeys.MKTXP_SOCKET_TIMEOUT, MKTXPConfigKeys.MKTXP_INITIAL_DELAY, MKTXPConfigKeys.MKTXP_MAX_DELAY, diff --git a/mktxp/collector/kid_control_device_collector.py b/mktxp/collector/kid_control_device_collector.py new file mode 100644 index 0000000..0e60f24 --- /dev/null +++ b/mktxp/collector/kid_control_device_collector.py @@ -0,0 +1,91 @@ +# coding=utf8 +## Copyright (c) 2020 Arseniy Kuznetsov +## +## This program is free software; you can redistribute it and/or +## modify it under the terms of the GNU General Public License +## as published by the Free Software Foundation; either version 2 +## of the License, or (at your option) any later version. +## +## This program is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. + + +from mktxp.collector.base_collector import BaseCollector +from mktxp.flow.processor.output import BaseOutputProcessor +from mktxp.datasource.kid_control_device_ds import KidDeviceMetricsDataSource + + +class KidDeviceCollector(BaseCollector): + ''' Kid-control device Metrics collector + ''' + @staticmethod + def collect(router_entry): + if not router_entry.config_entry.kid_control_devices: + return + + labels = ['name', 'user', 'mac_address', 'ip_address', 'bytes_down', 'bytes_up', 'rate_up', 'rate_down', 'bytes_up', 'idle_time', + 'blocked', 'limited', 'inactive', 'disabled'] + info_labels = ['name', 'user', 'mac_address', 'ip_address', 'disabled'] + records = KidDeviceMetricsDataSource.metric_records(router_entry, metric_labels = labels) + + if records: + # translate records to appropriate values + for record in records: + for label in record: + value = record.get(label, None) + if value: + record[label] = KidDeviceCollector._translated_values(label, value) + + info_metrics = BaseCollector.info_collector('kid_control_device', 'Kid-control device Info', records, info_labels) + yield info_metrics + + bytes_down_metrics = BaseCollector.gauge_collector('kid_control_device_bytes_down', 'Kid-control device bytes down', records, 'bytes_down', ['name']) + yield bytes_down_metrics + + bytes_up_metrics = BaseCollector.gauge_collector('kid_control_device_bytes_up', 'Kid-control device bytes up', records, 'bytes_up', ['name']) + yield bytes_up_metrics + + rate_down_metrics = BaseCollector.gauge_collector('kid_control_device_rate_down', 'Kid-control device rate down', records, 'rate_down', ['name']) + yield rate_down_metrics + + rate_up_metrics = BaseCollector.gauge_collector('kid_control_device_rate_up', 'Kid-control device rate up', records, 'rate_up', ['name']) + yield rate_up_metrics + + idle_time_metrics = BaseCollector.gauge_collector('kid_control_device_idle_time', 'Kid-control device idle time', records, 'idle_time', ['name']) + yield idle_time_metrics + + # Helpers + @staticmethod + def _translated_values(monitor_label, value): + try: + return { + 'rate_up': lambda value: KidDeviceCollector._rates(value), + 'rate_down': lambda value: KidDeviceCollector._rates(value), + 'idle_time': lambda value: BaseOutputProcessor.parse_timedelta_seconds(value), + 'blocked': lambda value: '1' if value=='true' else '0', + 'limited': lambda value: '1' if value=='true' else '0', + 'inactive': lambda value: '1' if value=='true' else '0', + 'disabled': lambda value: '1' if value=='true' else '0', + }[monitor_label](value) + except KeyError: + return value + + @staticmethod + def _rates(rate_option): + # according mikrotik docs, an interface rate should be one of these + rate_value = { + '10Mbps': '10', + '100Mbps': '100', + '1Gbps': '1000', + '2.5Gbps': '2500', + '5Gbps': '5000', + '10Gbps': '10000', + '40Gbps': '40000' + }.get(rate_option, None) + if rate_value: + return rate_value + + # ...or just calculate in case it's not + return BaseOutputProcessor.parse_interface_rate(rate_option) diff --git a/mktxp/datasource/kid_control_device_ds.py b/mktxp/datasource/kid_control_device_ds.py new file mode 100644 index 0000000..89033e4 --- /dev/null +++ b/mktxp/datasource/kid_control_device_ds.py @@ -0,0 +1,31 @@ +# coding=utf8 +## Copyright (c) 2020 Arseniy Kuznetsov +## +## This program is free software; you can redistribute it and/or +## modify it under the terms of the GNU General Public License +## as published by the Free Software Foundation; either version 2 +## of the License, or (at your option) any later version. +## +## This program is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. + + +from mktxp.datasource.base_ds import BaseDSProcessor + + +class KidDeviceMetricsDataSource: + ''' Kid-control device Metrics data provider + ''' + @staticmethod + def metric_records(router_entry, *, metric_labels = None): + if metric_labels is None: + metric_labels = [] + try: + device_records = router_entry.api_connection.router_api().get_resource('/ip/kid-control/device').get() + return BaseDSProcessor.trimmed_records(router_entry, router_records = device_records, metric_labels = metric_labels) + except Exception as exc: + print(f'Error getting Kid-control device info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}') + return None + diff --git a/mktxp/flow/collector_registry.py b/mktxp/flow/collector_registry.py index 7659ee9..2212f7d 100644 --- a/mktxp/flow/collector_registry.py +++ b/mktxp/flow/collector_registry.py @@ -36,6 +36,7 @@ from mktxp.collector.mktxp_collector import MKTXPCollector from mktxp.collector.user_collector import UserCollector from mktxp.collector.queue_collector import QueueTreeCollector from mktxp.collector.queue_collector import QueueSimpleCollector +from mktxp.collector.kid_control_device_collector import KidDeviceCollector class CollectorRegistry: ''' MKTXP Collectors Registry @@ -72,6 +73,8 @@ class CollectorRegistry: self.register(CollectorKeys.QUEUE_TREE_COLLECTOR, QueueTreeCollector.collect) self.register(CollectorKeys.QUEUE_SIMPLE_COLLECTOR, QueueSimpleCollector.collect) + self.register(CollectorKeys.KID_CONTROL_DEVICE_COLLECTOR, KidDeviceCollector.collect) + self.register(CollectorKeys.MKTXP_COLLECTOR, MKTXPCollector.collect) def register(self, collector_ID, collect_func): diff --git a/mktxp/flow/router_entry.py b/mktxp/flow/router_entry.py index 4c4b0c1..2fad603 100644 --- a/mktxp/flow/router_entry.py +++ b/mktxp/flow/router_entry.py @@ -49,7 +49,8 @@ class RouterEntry: CollectorKeys.CAPSMAN_COLLECTOR: 0, CollectorKeys.QUEUE_TREE_COLLECTOR: 0, CollectorKeys.QUEUE_SIMPLE_COLLECTOR: 0, - CollectorKeys.USER_COLLECTOR: 0, + CollectorKeys.KID_CONTROL_DEVICE_COLLECTOR: 0, + CollectorKeys.USER_COLLECTOR: 0, CollectorKeys.MKTXP_COLLECTOR: 0 } self._dhcp_entry = None From 5e02af8be41ec5ac4e4fb00b959b579495a42eac Mon Sep 17 00:00:00 2001 From: Wim Fournier Date: Thu, 2 Nov 2023 21:28:03 +0100 Subject: [PATCH 2/4] Add MAC address to make metrics unique --- mktxp/collector/kid_control_device_collector.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mktxp/collector/kid_control_device_collector.py b/mktxp/collector/kid_control_device_collector.py index 0e60f24..ff2ec8e 100644 --- a/mktxp/collector/kid_control_device_collector.py +++ b/mktxp/collector/kid_control_device_collector.py @@ -41,19 +41,19 @@ class KidDeviceCollector(BaseCollector): info_metrics = BaseCollector.info_collector('kid_control_device', 'Kid-control device Info', records, info_labels) yield info_metrics - bytes_down_metrics = BaseCollector.gauge_collector('kid_control_device_bytes_down', 'Kid-control device bytes down', records, 'bytes_down', ['name']) + bytes_down_metrics = BaseCollector.gauge_collector('kid_control_device_bytes_down', 'Kid-control device bytes down', records, 'bytes_down', ['name', 'mac_address']) yield bytes_down_metrics - bytes_up_metrics = BaseCollector.gauge_collector('kid_control_device_bytes_up', 'Kid-control device bytes up', records, 'bytes_up', ['name']) + bytes_up_metrics = BaseCollector.gauge_collector('kid_control_device_bytes_up', 'Kid-control device bytes up', records, 'bytes_up', ['name', 'mac_address']) yield bytes_up_metrics - rate_down_metrics = BaseCollector.gauge_collector('kid_control_device_rate_down', 'Kid-control device rate down', records, 'rate_down', ['name']) + rate_down_metrics = BaseCollector.gauge_collector('kid_control_device_rate_down', 'Kid-control device rate down', records, 'rate_down', ['name', 'mac_address']) yield rate_down_metrics - rate_up_metrics = BaseCollector.gauge_collector('kid_control_device_rate_up', 'Kid-control device rate up', records, 'rate_up', ['name']) + rate_up_metrics = BaseCollector.gauge_collector('kid_control_device_rate_up', 'Kid-control device rate up', records, 'rate_up', ['name', 'mac_address']) yield rate_up_metrics - idle_time_metrics = BaseCollector.gauge_collector('kid_control_device_idle_time', 'Kid-control device idle time', records, 'idle_time', ['name']) + idle_time_metrics = BaseCollector.gauge_collector('kid_control_device_idle_time', 'Kid-control device idle time', records, 'idle_time', ['name', 'mac_address']) yield idle_time_metrics # Helpers From 79ed4bfb5e2cefdf625ac9a2c155733c7896c222 Mon Sep 17 00:00:00 2001 From: "wim.fournier" Date: Fri, 3 Nov 2023 08:49:45 +0100 Subject: [PATCH 3/4] formatting --- .../collector/kid_control_device_collector.py | 43 ++++++++----------- mktxp/datasource/kid_control_device_ds.py | 15 ++++--- 2 files changed, 25 insertions(+), 33 deletions(-) diff --git a/mktxp/collector/kid_control_device_collector.py b/mktxp/collector/kid_control_device_collector.py index ff2ec8e..de7a266 100644 --- a/mktxp/collector/kid_control_device_collector.py +++ b/mktxp/collector/kid_control_device_collector.py @@ -18,17 +18,19 @@ from mktxp.datasource.kid_control_device_ds import KidDeviceMetricsDataSource class KidDeviceCollector(BaseCollector): - ''' Kid-control device Metrics collector - ''' + """ Kid-control device Metrics collector + """ + @staticmethod def collect(router_entry): if not router_entry.config_entry.kid_control_devices: return - labels = ['name', 'user', 'mac_address', 'ip_address', 'bytes_down', 'bytes_up', 'rate_up', 'rate_down', 'bytes_up', 'idle_time', + labels = ['name', 'user', 'mac_address', 'ip_address', 'bytes_down', 'bytes_up', 'rate_up', 'rate_down', + 'bytes_up', 'idle_time', 'blocked', 'limited', 'inactive', 'disabled'] info_labels = ['name', 'user', 'mac_address', 'ip_address', 'disabled'] - records = KidDeviceMetricsDataSource.metric_records(router_entry, metric_labels = labels) + records = KidDeviceMetricsDataSource.metric_records(router_entry, metric_labels=labels) if records: # translate records to appropriate values @@ -38,23 +40,12 @@ class KidDeviceCollector(BaseCollector): if value: record[label] = KidDeviceCollector._translated_values(label, value) - info_metrics = BaseCollector.info_collector('kid_control_device', 'Kid-control device Info', records, info_labels) - yield info_metrics - - bytes_down_metrics = BaseCollector.gauge_collector('kid_control_device_bytes_down', 'Kid-control device bytes down', records, 'bytes_down', ['name', 'mac_address']) - yield bytes_down_metrics - - bytes_up_metrics = BaseCollector.gauge_collector('kid_control_device_bytes_up', 'Kid-control device bytes up', records, 'bytes_up', ['name', 'mac_address']) - yield bytes_up_metrics - - rate_down_metrics = BaseCollector.gauge_collector('kid_control_device_rate_down', 'Kid-control device rate down', records, 'rate_down', ['name', 'mac_address']) - yield rate_down_metrics - - rate_up_metrics = BaseCollector.gauge_collector('kid_control_device_rate_up', 'Kid-control device rate up', records, 'rate_up', ['name', 'mac_address']) - yield rate_up_metrics - - idle_time_metrics = BaseCollector.gauge_collector('kid_control_device_idle_time', 'Kid-control device idle time', records, 'idle_time', ['name', 'mac_address']) - yield idle_time_metrics + yield BaseCollector.info_collector('kid_control_device', 'Kid-control device Info', records, info_labels) + yield BaseCollector.gauge_collector('kid_control_device_bytes_down', 'Kid-control device bytes down', records, 'bytes_down', ['name', 'mac_address']) + yield BaseCollector.gauge_collector('kid_control_device_bytes_up', 'Kid-control device bytes up', records, 'bytes_up', ['name', 'mac_address']) + yield BaseCollector.gauge_collector('kid_control_device_rate_down', 'Kid-control device rate down', records, 'rate_down', ['name', 'mac_address']) + yield BaseCollector.gauge_collector('kid_control_device_rate_up', 'Kid-control device rate up', records, 'rate_up', ['name', 'mac_address']) + yield BaseCollector.gauge_collector('kid_control_device_idle_time', 'Kid-control device idle time', records, 'idle_time', ['name', 'mac_address']) # Helpers @staticmethod @@ -64,10 +55,10 @@ class KidDeviceCollector(BaseCollector): 'rate_up': lambda value: KidDeviceCollector._rates(value), 'rate_down': lambda value: KidDeviceCollector._rates(value), 'idle_time': lambda value: BaseOutputProcessor.parse_timedelta_seconds(value), - 'blocked': lambda value: '1' if value=='true' else '0', - 'limited': lambda value: '1' if value=='true' else '0', - 'inactive': lambda value: '1' if value=='true' else '0', - 'disabled': lambda value: '1' if value=='true' else '0', + 'blocked': lambda value: '1' if value == 'true' else '0', + 'limited': lambda value: '1' if value == 'true' else '0', + 'inactive': lambda value: '1' if value == 'true' else '0', + 'disabled': lambda value: '1' if value == 'true' else '0', }[monitor_label](value) except KeyError: return value @@ -75,7 +66,7 @@ class KidDeviceCollector(BaseCollector): @staticmethod def _rates(rate_option): # according mikrotik docs, an interface rate should be one of these - rate_value = { + rate_value = { '10Mbps': '10', '100Mbps': '100', '1Gbps': '1000', diff --git a/mktxp/datasource/kid_control_device_ds.py b/mktxp/datasource/kid_control_device_ds.py index 89033e4..b367b8a 100644 --- a/mktxp/datasource/kid_control_device_ds.py +++ b/mktxp/datasource/kid_control_device_ds.py @@ -16,16 +16,17 @@ from mktxp.datasource.base_ds import BaseDSProcessor class KidDeviceMetricsDataSource: - ''' Kid-control device Metrics data provider - ''' + """ Kid-control device Metrics data provider + """ + @staticmethod - def metric_records(router_entry, *, metric_labels = None): + def metric_records(router_entry, *, metric_labels=None): if metric_labels is None: - metric_labels = [] + metric_labels = [] try: device_records = router_entry.api_connection.router_api().get_resource('/ip/kid-control/device').get() - return BaseDSProcessor.trimmed_records(router_entry, router_records = device_records, metric_labels = metric_labels) + return BaseDSProcessor.trimmed_records(router_entry, router_records=device_records, metric_labels=metric_labels) except Exception as exc: - print(f'Error getting Kid-control device info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}') + print( + f'Error getting Kid-control device info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}') return None - From 2a55ee9780756cdf4cac9bf52b529a49bea1cd7b Mon Sep 17 00:00:00 2001 From: "wim.fournier" Date: Fri, 3 Nov 2023 08:56:06 +0100 Subject: [PATCH 4/4] add comment --- mktxp/collector/kid_control_device_collector.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mktxp/collector/kid_control_device_collector.py b/mktxp/collector/kid_control_device_collector.py index de7a266..44cf479 100644 --- a/mktxp/collector/kid_control_device_collector.py +++ b/mktxp/collector/kid_control_device_collector.py @@ -61,6 +61,7 @@ class KidDeviceCollector(BaseCollector): 'disabled': lambda value: '1' if value == 'true' else '0', }[monitor_label](value) except KeyError: + # default to just returning the value return value @staticmethod