From 5b513ee85f41195891c09bc107cd358bf8b43bb2 Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Sun, 3 Mar 2024 08:27:42 +0100 Subject: [PATCH 1/5] BGP metrics --- README.md | 2 + mktxp/cli/config/config.py | 7 ++- mktxp/cli/config/mktxp.conf | 2 + mktxp/collector/bgp_collector.py | 79 ++++++++++++++++++++++++++++++++ mktxp/datasource/bgp_ds.py | 31 +++++++++++++ mktxp/flow/collector_registry.py | 3 ++ mktxp/flow/processor/output.py | 6 ++- mktxp/flow/router_entry.py | 1 + setup.py | 2 +- 9 files changed, 129 insertions(+), 4 deletions(-) create mode 100644 mktxp/collector/bgp_collector.py create mode 100644 mktxp/datasource/bgp_ds.py diff --git a/README.md b/README.md index b9370f8..31b2b6e 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,8 @@ The default configuration file comes with a sample configuration, making it easy user = True # Active Users metrics queue = True # Queues metrics + + bgp = False # BGP sessions metrics remote_dhcp_entry = None # An MKTXP entry for remote DHCP info resolution (capsman/wireless) diff --git a/mktxp/cli/config/config.py b/mktxp/cli/config/config.py index 7315042..53ad070 100755 --- a/mktxp/cli/config/config.py +++ b/mktxp/cli/config/config.py @@ -46,6 +46,7 @@ class CollectorKeys: QUEUE_SIMPLE_COLLECTOR = 'QueueSimpleCollector' KID_CONTROL_DEVICE_COLLECTOR = 'KidControlCollector' USER_COLLECTOR = 'UserCollector' + BGP_COLLECTOR = 'BGPCollector' MKTXP_COLLECTOR = 'MKTXPCollector' @@ -87,6 +88,8 @@ class MKTXPConfigKeys: FE_USER_KEY = 'user' FE_QUEUE_KEY = 'queue' + FE_BGP_KEY = 'bgp' + FE_REMOTE_DHCP_ENTRY = 'remote_dhcp_entry' FE_CHECK_FOR_UPDATES = 'check_for_updates' @@ -133,7 +136,7 @@ class MKTXPConfigKeys: BOOLEAN_KEYS_NO = {ENABLED_KEY, SSL_KEY, NO_SSL_CERTIFICATE, FE_CHECK_FOR_UPDATES, FE_KID_CONTROL_DEVICE, - SSL_CERTIFICATE_VERIFY, FE_IPV6_FIREWALL_KEY, FE_IPV6_NEIGHBOR_KEY, FE_CONNECTION_STATS_KEY} + SSL_CERTIFICATE_VERIFY, FE_IPV6_FIREWALL_KEY, FE_IPV6_NEIGHBOR_KEY, FE_CONNECTION_STATS_KEY, FE_BGP_KEY} # Feature keys enabled by default BOOLEAN_KEYS_YES = {FE_DHCP_KEY, FE_PACKAGE_KEY, FE_DHCP_LEASE_KEY, FE_DHCP_POOL_KEY, FE_IP_CONNECTIONS_KEY, FE_INTERFACE_KEY, FE_FIREWALL_KEY, @@ -162,7 +165,7 @@ class ConfigEntry: MKTXPConfigKeys.FE_FIREWALL_KEY, MKTXPConfigKeys.FE_MONITOR_KEY, MKTXPConfigKeys.FE_ROUTE_KEY, MKTXPConfigKeys.FE_WIRELESS_KEY, MKTXPConfigKeys.FE_WIRELESS_CLIENTS_KEY, MKTXPConfigKeys.FE_IP_CONNECTIONS_KEY, MKTXPConfigKeys.FE_CONNECTION_STATS_KEY, MKTXPConfigKeys.FE_CAPSMAN_KEY, MKTXPConfigKeys.FE_CAPSMAN_CLIENTS_KEY, MKTXPConfigKeys.FE_POE_KEY, MKTXPConfigKeys.FE_NETWATCH_KEY, MKTXPConfigKeys.MKTXP_USE_COMMENTS_OVER_NAMES, MKTXPConfigKeys.FE_PUBLIC_IP_KEY, MKTXPConfigKeys.FE_IPV6_FIREWALL_KEY, MKTXPConfigKeys.FE_IPV6_NEIGHBOR_KEY, - MKTXPConfigKeys.FE_USER_KEY, MKTXPConfigKeys.FE_QUEUE_KEY, MKTXPConfigKeys.FE_REMOTE_DHCP_ENTRY, MKTXPConfigKeys.FE_CHECK_FOR_UPDATES, MKTXPConfigKeys.FE_KID_CONTROL_DEVICE, + MKTXPConfigKeys.FE_USER_KEY, MKTXPConfigKeys.FE_QUEUE_KEY, MKTXPConfigKeys.FE_REMOTE_DHCP_ENTRY, MKTXPConfigKeys.FE_CHECK_FOR_UPDATES, MKTXPConfigKeys.FE_KID_CONTROL_DEVICE, MKTXPConfigKeys.FE_BGP_KEY, ]) MKTXPSystemEntry = namedtuple('MKTXPSystemEntry', [MKTXPConfigKeys.PORT_KEY, MKTXPConfigKeys.MKTXP_SOCKET_TIMEOUT, MKTXPConfigKeys.MKTXP_INITIAL_DELAY, MKTXPConfigKeys.MKTXP_MAX_DELAY, diff --git a/mktxp/cli/config/mktxp.conf b/mktxp/cli/config/mktxp.conf index 18b9af3..1527f7e 100644 --- a/mktxp/cli/config/mktxp.conf +++ b/mktxp/cli/config/mktxp.conf @@ -52,6 +52,8 @@ user = True # Active Users metrics queue = True # Queues metrics + + bgp = False # BGP sessions metrics remote_dhcp_entry = None # An MKTXP entry for remote DHCP info resolution (capsman/wireless) diff --git a/mktxp/collector/bgp_collector.py b/mktxp/collector/bgp_collector.py new file mode 100644 index 0000000..cb55ebb --- /dev/null +++ b/mktxp/collector/bgp_collector.py @@ -0,0 +1,79 @@ +# coding=utf8 +## Copyright (c) 2020 Arseniy Kuznetsov +## +## This program is free software; you can redistribute it and/or +## modify it under the terms of the GNU General Public License +## as published by the Free Software Foundation; either version 2 +## of the License, or (at your option) any later version. +## +## This program is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. + + +from mktxp.collector.base_collector import BaseCollector +from mktxp.flow.processor.output import BaseOutputProcessor +from mktxp.datasource.bgp_ds import BGPMetricsDataSource + + +class BGPCollector(BaseCollector): + '''BGP collector''' + @staticmethod + def collect(router_entry): + if not router_entry.config_entry.bgp: + return + + bgp_labels = ['name', 'remote.address', 'remote.as', 'local.as', 'remote.afi', 'local.afi', 'remote.messages', 'remote.bytes', 'local.messages', 'local.bytes', 'prefix_count', 'established', 'uptime'] + bgp_records = BGPMetricsDataSource.metric_records(router_entry, metric_labels=bgp_labels) + + + if bgp_records: + # translate records to appropriate values + translated_fields = ['established', 'uptime'] + for bgp_record in bgp_records: + for translated_field in translated_fields: + value = bgp_record.get(translated_field, None) + if value: + bgp_record[translated_field] = BGPCollector._translated_values(translated_field, value) + + session_id_labes = ['name', 'remote.address', 'remote.as', 'local.as', 'remote.afi', 'local.afi'] + bgp_sessions_metrics = BaseCollector.info_collector('bgp_sessions_info', 'BGP sessions info', bgp_records, session_id_labes) + yield bgp_sessions_metrics + + remote_messages_metrics = BaseCollector.counter_collector('bgp_remote_messages', 'Number of remote messages', bgp_records, 'remote.messages', session_id_labes) + yield remote_messages_metrics + + + local_messages_metrics = BaseCollector.counter_collector('bgp_local_messages', 'Number of local messages', bgp_records, 'local.messages', session_id_labes) + yield local_messages_metrics + + + remote_bytes_metrics = BaseCollector.counter_collector('bgp_remote_bytes', 'Number of remote bytes', bgp_records, 'remote.bytes', session_id_labes) + yield remote_bytes_metrics + + + local_bytes_metrics = BaseCollector.counter_collector('bgp_local_bytes', 'Number of local bytes', bgp_records, 'local.bytes', session_id_labes) + yield local_bytes_metrics + + + prefix_count_metrics = BaseCollector.gauge_collector('bgp_prefix_count', 'BGP prefix count', bgp_records, 'prefix_count', session_id_labes) + yield prefix_count_metrics + + + established_metrics = BaseCollector.gauge_collector('bgp_established', 'BGP established', bgp_records, 'established', session_id_labes) + yield established_metrics + + + uptime_metrics = BaseCollector.gauge_collector('bgp_uptime', 'BGP uptime in milliseconds', bgp_records, 'uptime', session_id_labes) + yield uptime_metrics + + + # Helpers + @staticmethod + def _translated_values(translated_field, value): + return { + 'established': lambda value: '1' if value=='true' else '0', + 'uptime': lambda value: BaseOutputProcessor.parse_timedelta_milliseconds(value) + }[translated_field](value) + diff --git a/mktxp/datasource/bgp_ds.py b/mktxp/datasource/bgp_ds.py new file mode 100644 index 0000000..78740be --- /dev/null +++ b/mktxp/datasource/bgp_ds.py @@ -0,0 +1,31 @@ +# coding=utf8 +## Copyright (c) 2020 Arseniy Kuznetsov +## +## This program is free software; you can redistribute it and/or +## modify it under the terms of the GNU General Public License +## as published by the Free Software Foundation; either version 2 +## of the License, or (at your option) any later version. +## +## This program is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. + + +from mktxp.datasource.base_ds import BaseDSProcessor + + +class BGPMetricsDataSource: + ''' Wireless Metrics data provider + ''' + @staticmethod + def metric_records(router_entry, *, metric_labels = None, add_router_id = True): + if metric_labels is None: + metric_labels = [] + try: + bgp_records = router_entry.api_connection.router_api().get_resource('/routing/bgp/session').get() + return BaseDSProcessor.trimmed_records(router_entry, router_records = bgp_records, metric_labels = metric_labels, add_router_id = add_router_id) + except Exception as exc: + print(f'Error getting BGP sessions info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}') + return None + diff --git a/mktxp/flow/collector_registry.py b/mktxp/flow/collector_registry.py index 2212f7d..028d110 100644 --- a/mktxp/flow/collector_registry.py +++ b/mktxp/flow/collector_registry.py @@ -37,6 +37,7 @@ from mktxp.collector.user_collector import UserCollector from mktxp.collector.queue_collector import QueueTreeCollector from mktxp.collector.queue_collector import QueueSimpleCollector from mktxp.collector.kid_control_device_collector import KidDeviceCollector +from mktxp.collector.bgp_collector import BGPCollector class CollectorRegistry: ''' MKTXP Collectors Registry @@ -74,6 +75,8 @@ class CollectorRegistry: self.register(CollectorKeys.QUEUE_SIMPLE_COLLECTOR, QueueSimpleCollector.collect) self.register(CollectorKeys.KID_CONTROL_DEVICE_COLLECTOR, KidDeviceCollector.collect) + self.register(CollectorKeys.BGP_COLLECTOR, BGPCollector.collect) + self.register(CollectorKeys.MKTXP_COLLECTOR, MKTXPCollector.collect) diff --git a/mktxp/flow/processor/output.py b/mktxp/flow/processor/output.py index 8fb43eb..1af33c5 100644 --- a/mktxp/flow/processor/output.py +++ b/mktxp/flow/processor/output.py @@ -117,7 +117,7 @@ class BaseOutputProcessor: def parse_timedelta(time): duration_interval_rgx = config_handler.re_compiled.get('duration_interval_rgx') if not duration_interval_rgx: - duration_interval_rgx = re.compile(r'((?P\d+)w)?((?P\d+)d)?((?P\d+)h)?((?P\d+)m)?((?P\d+)s)?') + duration_interval_rgx = re.compile(r'((?P\d+)w)?((?P\d+)d)?((?P\d+)h)?((?P\d+)m)?((?P\d+)s)?((?P\d+)ms)?') config_handler.re_compiled['duration_interval_rgx'] = duration_interval_rgx time_dict = duration_interval_rgx.match(time).groupdict() return timedelta(**{key: int(value) for key, value in time_dict.items() if value}) @@ -126,6 +126,10 @@ class BaseOutputProcessor: def parse_timedelta_seconds(time): return BaseOutputProcessor.parse_timedelta(time).total_seconds() + @staticmethod + def parse_timedelta_milliseconds(time): + return BaseOutputProcessor.parse_timedelta(time) / timedelta(milliseconds=1) + @staticmethod def parse_signal_strength(signal_strength): wifi_signal_strength_rgx = config_handler.re_compiled.get('wifi_signal_strength_rgx') diff --git a/mktxp/flow/router_entry.py b/mktxp/flow/router_entry.py index 36a4837..907347a 100644 --- a/mktxp/flow/router_entry.py +++ b/mktxp/flow/router_entry.py @@ -66,6 +66,7 @@ class RouterEntry: CollectorKeys.QUEUE_SIMPLE_COLLECTOR: 0, CollectorKeys.KID_CONTROL_DEVICE_COLLECTOR: 0, CollectorKeys.USER_COLLECTOR: 0, + CollectorKeys.BGP_COLLECTOR: 0, CollectorKeys.MKTXP_COLLECTOR: 0 } self._dhcp_entry = None diff --git a/setup.py b/setup.py index 44a8a58..25cbd47 100755 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ with open(path.join(pkg_dir, 'README.md'), encoding='utf-8') as f: setup( name='mktxp', - version='1.2.2', + version='1.2.3', url='https://github.com/akpw/mktxp', From f70719d46f9cf29551005eb3bc7082c77a5786e8 Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Sun, 3 Mar 2024 14:20:03 +0100 Subject: [PATCH 2/5] BGP session metrics --- mktxp/collector/bgp_collector.py | 12 ++++++------ mktxp/datasource/base_ds.py | 17 +++++++++++++---- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/mktxp/collector/bgp_collector.py b/mktxp/collector/bgp_collector.py index cb55ebb..e82c2c0 100644 --- a/mktxp/collector/bgp_collector.py +++ b/mktxp/collector/bgp_collector.py @@ -24,7 +24,7 @@ class BGPCollector(BaseCollector): if not router_entry.config_entry.bgp: return - bgp_labels = ['name', 'remote.address', 'remote.as', 'local.as', 'remote.afi', 'local.afi', 'remote.messages', 'remote.bytes', 'local.messages', 'local.bytes', 'prefix_count', 'established', 'uptime'] + bgp_labels = ['name', 'remote_address', 'remote_as', 'local_as', 'remote_afi', 'local_afi', 'remote_messages', 'remote_bytes', 'local_messages', 'local_bytes', 'prefix_count', 'established', 'uptime'] bgp_records = BGPMetricsDataSource.metric_records(router_entry, metric_labels=bgp_labels) @@ -37,23 +37,23 @@ class BGPCollector(BaseCollector): if value: bgp_record[translated_field] = BGPCollector._translated_values(translated_field, value) - session_id_labes = ['name', 'remote.address', 'remote.as', 'local.as', 'remote.afi', 'local.afi'] + session_id_labes = ['name', 'remote_address', 'remote_as', 'local_as', 'remote_afi', 'local_afi'] bgp_sessions_metrics = BaseCollector.info_collector('bgp_sessions_info', 'BGP sessions info', bgp_records, session_id_labes) yield bgp_sessions_metrics - remote_messages_metrics = BaseCollector.counter_collector('bgp_remote_messages', 'Number of remote messages', bgp_records, 'remote.messages', session_id_labes) + remote_messages_metrics = BaseCollector.counter_collector('bgp_remote_messages', 'Number of remote messages', bgp_records, 'remote_messages', session_id_labes) yield remote_messages_metrics - local_messages_metrics = BaseCollector.counter_collector('bgp_local_messages', 'Number of local messages', bgp_records, 'local.messages', session_id_labes) + local_messages_metrics = BaseCollector.counter_collector('bgp_local_messages', 'Number of local messages', bgp_records, 'local_messages', session_id_labes) yield local_messages_metrics - remote_bytes_metrics = BaseCollector.counter_collector('bgp_remote_bytes', 'Number of remote bytes', bgp_records, 'remote.bytes', session_id_labes) + remote_bytes_metrics = BaseCollector.counter_collector('bgp_remote_bytes', 'Number of remote bytes', bgp_records, 'remote_bytes', session_id_labes) yield remote_bytes_metrics - local_bytes_metrics = BaseCollector.counter_collector('bgp_local_bytes', 'Number of local bytes', bgp_records, 'local.bytes', session_id_labes) + local_bytes_metrics = BaseCollector.counter_collector('bgp_local_bytes', 'Number of local bytes', bgp_records, 'local_bytes', session_id_labes) yield local_bytes_metrics diff --git a/mktxp/datasource/base_ds.py b/mktxp/datasource/base_ds.py index a656e42..32a6b42 100644 --- a/mktxp/datasource/base_ds.py +++ b/mktxp/datasource/base_ds.py @@ -23,15 +23,14 @@ class BaseDSProcessor: if metric_labels is None: metric_labels = [] if translation_table is None: - translation_table = {} - dash2_ = lambda x : x.replace('-', '_') + translation_table = {} if len(metric_labels) == 0 and len(router_records) > 0: - metric_labels = [dash2_(key) for key in router_records[0].keys()] + metric_labels = [BaseDSProcessor._normalise_keys(key) for key in router_records[0].keys()] metric_labels = set(metric_labels) labeled_records = [] for router_record in router_records: - translated_record = {dash2_(key): value for (key, value) in router_record.items() if dash2_(key) in metric_labels} + translated_record = {BaseDSProcessor._normalise_keys(key): value for (key, value) in router_record.items() if BaseDSProcessor._normalise_keys(key) in metric_labels} if add_router_id: for key, value in router_entry.router_id.items(): @@ -43,3 +42,13 @@ class BaseDSProcessor: labeled_records.append(translated_record) return labeled_records + + + @staticmethod + def _normalise_keys(key): + chars = ".-" + for chr in chars: + if chr in key: + key = key.replace(chr, "_") + return key + From 412b6f35112462e42878f70d11ecb179f168b55f Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Sun, 10 Mar 2024 14:28:48 +0100 Subject: [PATCH 3/5] waitress, IPv4/IPv6 listen list --- README.md | 12 +++++++----- mktxp/cli/checks/chk_pv.py | 8 ++++---- mktxp/cli/config/_mktxp.conf | 4 ++-- mktxp/cli/config/config.py | 20 +++++++++++++++----- mktxp/flow/processor/base_proc.py | 16 +++++----------- setup.py | 5 +++-- 6 files changed, 36 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 31b2b6e..cc5bab2 100644 --- a/README.md +++ b/README.md @@ -213,7 +213,7 @@ mktxp edit -i ``` [MKTXP] - port = 49090 + listen = '0.0.0.0:49090' # Space separated list of socket addresses to listen to, both IPV4 and IPV6 socket_timeout = 2 initial_delay_on_failure = 120 @@ -275,7 +275,7 @@ optional arguments: While most of the [mktxp options](https://github.com/akpw/mktxp#getting-started) are self explanatory, some might require a bit of a context. ### Remote DHCP resolution -When gathering various IP address-related metrics, MKTXP automatically resolves IP addresses whenever DHCP info is available. In many cases however, the exported devices do not have this information locally and instead rely on central DHCP servers. To improve readibility / usefulness of the exported metrics, MKTXP supports remote DHCP server calls via the following option: +When gathering various IP address-related metrics, MKTXP automatically resolves IP addresses whenever DHCP info is available. In many cases however, the exported devices do not have this information locally and instead rely on central DHCP servers. To improve readability / usefulness of the exported metrics, MKTXP supports remote DHCP server calls via the following option: ``` remote_dhcp_entry = None # An MKTXP entry for remote DHCP info resolution in capsman/wireless ``` @@ -323,11 +323,13 @@ total_max_scrape_duration = 30 # Max overall duration of all metrics collec To keeps things within expected boundaries, the last two parameters allows for controlling both individual and overall scrape durations -### mktxp port -By default, mktxp runs it's HTTP metrics endpoint on port 49090. You can change it via the following [system option](https://github.com/akpw/mktxp/blob/main/README.md#mktxp-system-configuration): +### mktxp listening socket addresses +By default, mktxp runs it's HTTP metrics endpoint on any IPv4 address on port 49090. However, it is also able to listen on multiple socket addresses, both IPv4 and IPv6. +You can configure this behaviour via the following [system option](https://github.com/akpw/mktxp/blob/main/README.md#mktxp-system-configuration), setting ```listen``` to a space-separated list of sockets to listen to, e.g.: ``` -port = 49090 +listen = '0.0.0.0:49090 [::1]:49090' ``` +A wildcard for the hostname is supported as well, and binding to both IPv4/IPv6 as available. ## Setting up MKTXP to run as a Linux Service If you've installed MKTXP on a Linux system, you can run it with system boot via adding a service. \ diff --git a/mktxp/cli/checks/chk_pv.py b/mktxp/cli/checks/chk_pv.py index c2d0985..0ff5dba 100755 --- a/mktxp/cli/checks/chk_pv.py +++ b/mktxp/cli/checks/chk_pv.py @@ -24,20 +24,20 @@ def check_version(): print(\ ''' Mikrotik Prometheus Exporter requires - Python version 3.6 or later. + Python version 3.8 or later. - You can create an isolated Python 3.6 environment + You can create an isolated Python 3.8 environment with the virtualenv tool: http://docs.python-guide.org/en/latest/dev/virtualenvs ''') sys.exit(0) - elif sys.version_info.major == 3 and sys.version_info.minor < 6: + elif sys.version_info.major == 3 and sys.version_info.minor < 8: print(\ ''' Mikrotik Prometheus Exporter requires - Python version 3.6 or later. + Python version 3.8 or later. Please upgrade to the latest Python 3.x version. diff --git a/mktxp/cli/config/_mktxp.conf b/mktxp/cli/config/_mktxp.conf index c141369..4fa589d 100644 --- a/mktxp/cli/config/_mktxp.conf +++ b/mktxp/cli/config/_mktxp.conf @@ -12,7 +12,7 @@ [MKTXP] - port = 49090 + listen = '0.0.0.0:49090' # Space separated list of socket addresses to listen to, both IPV4 and IPV6 socket_timeout = 2 initial_delay_on_failure = 120 @@ -20,7 +20,7 @@ delay_inc_div = 5 bandwidth = False # Turns metrics bandwidth metrics collection on / off - bandwidth_test_interval = 600 # Interval for colllecting bandwidth metrics + bandwidth_test_interval = 600 # Interval for collecting bandwidth metrics minimal_collect_interval = 5 # Minimal metric collection interval verbose_mode = False # Set it on for troubleshooting diff --git a/mktxp/cli/config/config.py b/mktxp/cli/config/config.py index 53ad070..562ceca 100755 --- a/mktxp/cli/config/config.py +++ b/mktxp/cli/config/config.py @@ -57,6 +57,7 @@ class MKTXPConfigKeys: ENABLED_KEY = 'enabled' HOST_KEY = 'hostname' PORT_KEY = 'port' + LISTEN_KEY = 'listen' USER_KEY = 'username' PASSWD_KEY = 'password' @@ -153,7 +154,7 @@ class MKTXPConfigKeys: MKTXP_INC_DIV, MKTXP_BANDWIDTH_TEST_INTERVAL, MKTXP_MIN_COLLECT_INTERVAL, MKTXP_MAX_WORKER_THREADS, MKTXP_MAX_SCRAPE_DURATION, MKTXP_TOTAL_MAX_SCRAPE_DURATION) - # MKTXP config entry nane + # MKTXP config entry name MKTXP_CONFIG_ENTRY_NAME = 'MKTXP' @@ -167,7 +168,7 @@ class ConfigEntry: MKTXPConfigKeys.MKTXP_USE_COMMENTS_OVER_NAMES, MKTXPConfigKeys.FE_PUBLIC_IP_KEY, MKTXPConfigKeys.FE_IPV6_FIREWALL_KEY, MKTXPConfigKeys.FE_IPV6_NEIGHBOR_KEY, MKTXPConfigKeys.FE_USER_KEY, MKTXPConfigKeys.FE_QUEUE_KEY, MKTXPConfigKeys.FE_REMOTE_DHCP_ENTRY, MKTXPConfigKeys.FE_CHECK_FOR_UPDATES, MKTXPConfigKeys.FE_KID_CONTROL_DEVICE, MKTXPConfigKeys.FE_BGP_KEY, ]) - MKTXPSystemEntry = namedtuple('MKTXPSystemEntry', [MKTXPConfigKeys.PORT_KEY, MKTXPConfigKeys.MKTXP_SOCKET_TIMEOUT, + MKTXPSystemEntry = namedtuple('MKTXPSystemEntry', [MKTXPConfigKeys.PORT_KEY, MKTXPConfigKeys.LISTEN_KEY, MKTXPConfigKeys.MKTXP_SOCKET_TIMEOUT, MKTXPConfigKeys.MKTXP_INITIAL_DELAY, MKTXPConfigKeys.MKTXP_MAX_DELAY, MKTXPConfigKeys.MKTXP_INC_DIV, MKTXPConfigKeys.MKTXP_BANDWIDTH_KEY, MKTXPConfigKeys.MKTXP_VERBOSE_MODE, MKTXPConfigKeys.MKTXP_BANDWIDTH_TEST_INTERVAL, @@ -291,10 +292,10 @@ class MKTXPConfigHandler: def _read_from_disk(self): ''' (Force-)Read conf data from disk ''' - self.config = ConfigObj(self.usr_conf_data_path) + self.config = ConfigObj(self.usr_conf_data_path, indent_type = ' ') self.config.preserve_comments = True - self._config = ConfigObj(self.mktxp_conf_path) + self._config = ConfigObj(self.mktxp_conf_path, indent_type = ' ') self._config.preserve_comments = True def _create_os_path(self, os_path, resource_path): @@ -363,7 +364,8 @@ class MKTXPConfigHandler: system_entry_reader[key] = self._config[entry_name].as_int(key) else: system_entry_reader[key] = self._default_value_for_key(key) - new_keys.append(key) # read from disk next time + if key not in (MKTXPConfigKeys.PORT_KEY): # Port key has been depricated + new_keys.append(key) # read from disk next time for key in MKTXPConfigKeys.SYSTEM_BOOLEAN_KEYS_NO.union(MKTXPConfigKeys.SYSTEM_BOOLEAN_KEYS_YES): if self._config[entry_name].get(key) is not None: @@ -372,9 +374,17 @@ class MKTXPConfigHandler: system_entry_reader[key] = True if key in MKTXPConfigKeys.SYSTEM_BOOLEAN_KEYS_YES else False new_keys.append(key) # read from disk next time + # listen + if self._config[entry_name].get(MKTXPConfigKeys.LISTEN_KEY): + system_entry_reader[MKTXPConfigKeys.LISTEN_KEY] = self._config[entry_name].get(MKTXPConfigKeys.LISTEN_KEY) + else: + system_entry_reader[MKTXPConfigKeys.LISTEN_KEY] = f'0.0.0.0:{system_entry_reader[MKTXPConfigKeys.PORT_KEY]}' + new_keys.append(MKTXPConfigKeys.LISTEN_KEY) # read from disk next time + if new_keys: self._config[entry_name] = system_entry_reader try: + self._config[entry_name].pop(MKTXPConfigKeys.PORT_KEY, None) # Port key has been depricated self._config.write() if self._config[entry_name].as_bool(MKTXPConfigKeys.MKTXP_VERBOSE_MODE): print(f'Updated system entry {entry_name} with new system keys {new_keys}') diff --git a/mktxp/flow/processor/base_proc.py b/mktxp/flow/processor/base_proc.py index f42766e..4ff724d 100644 --- a/mktxp/flow/processor/base_proc.py +++ b/mktxp/flow/processor/base_proc.py @@ -12,10 +12,9 @@ ## GNU General Public License for more details. -from http.server import HTTPServer from datetime import datetime from prometheus_client.core import REGISTRY -from prometheus_client import MetricsHandler +from prometheus_client import make_wsgi_app from mktxp.cli.config.config import config_handler from mktxp.flow.collector_handler import CollectorHandler @@ -27,6 +26,8 @@ from mktxp.cli.output.wifi_out import WirelessOutput from mktxp.cli.output.dhcp_out import DHCPOutput from mktxp.cli.output.conn_stats_out import ConnectionsStatsOutput +from waitress import serve + class ExportProcessor: ''' Base Export Processing @@ -34,16 +35,9 @@ class ExportProcessor: @staticmethod def start(): REGISTRY.register(CollectorHandler(RouterEntriesHandler(), CollectorRegistry())) - ExportProcessor.run(port=config_handler.system_entry().port) - - @staticmethod - def run(server_class=HTTPServer, handler_class=MetricsHandler, port=None): - server_address = ('', port) - httpd = server_class(server_address, handler_class) current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - print(f'{current_time} Running HTTP metrics server on port {port}') - httpd.serve_forever() - + print(f'{current_time} Running HTTP metrics server on: {config_handler.system_entry().listen}') + serve(make_wsgi_app(), listen = config_handler.system_entry().listen) class OutputProcessor: ''' Base CLI Processing diff --git a/setup.py b/setup.py index 25cbd47..633724a 100755 --- a/setup.py +++ b/setup.py @@ -48,7 +48,8 @@ setup( 'configobj>=5.0.6', 'humanize>=3.2.0', 'texttable>=1.6.3', - 'speedtest-cli>=2.1.2' + 'speedtest-cli>=2.1.2', + 'waitress>=3.0.0', ], test_suite = 'tests.mktxp_test_suite', @@ -63,7 +64,7 @@ setup( 'Development Status :: 4 - Beta', 'License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)', 'Programming Language :: Python', - 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3 :: Only', 'Intended Audience :: Developers', 'Intended Audience :: System Administrators', From eff2d2b39bebac4fc679c521d4235dc2164465b4 Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Sun, 10 Mar 2024 15:19:30 +0100 Subject: [PATCH 4/5] BGP labels (->#94) --- mktxp/collector/bgp_collector.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mktxp/collector/bgp_collector.py b/mktxp/collector/bgp_collector.py index e82c2c0..78148fc 100644 --- a/mktxp/collector/bgp_collector.py +++ b/mktxp/collector/bgp_collector.py @@ -37,10 +37,11 @@ class BGPCollector(BaseCollector): if value: bgp_record[translated_field] = BGPCollector._translated_values(translated_field, value) - session_id_labes = ['name', 'remote_address', 'remote_as', 'local_as', 'remote_afi', 'local_afi'] - bgp_sessions_metrics = BaseCollector.info_collector('bgp_sessions_info', 'BGP sessions info', bgp_records, session_id_labes) + session_info_labes = ['name', 'remote_address', 'remote_as', 'local_as', 'remote_afi', 'local_afi'] + bgp_sessions_metrics = BaseCollector.info_collector('bgp_sessions_info', 'BGP sessions info', bgp_records, session_info_labes) yield bgp_sessions_metrics + session_id_labes = ['name'] remote_messages_metrics = BaseCollector.counter_collector('bgp_remote_messages', 'Number of remote messages', bgp_records, 'remote_messages', session_id_labes) yield remote_messages_metrics From d3d782ebd4c9fea5611b3097db21355ae11753d9 Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Sun, 10 Mar 2024 18:18:11 +0100 Subject: [PATCH 5/5] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cc5bab2..61b09bc 100644 --- a/README.md +++ b/README.md @@ -323,7 +323,7 @@ total_max_scrape_duration = 30 # Max overall duration of all metrics collec To keeps things within expected boundaries, the last two parameters allows for controlling both individual and overall scrape durations -### mktxp listening socket addresses +### mktxp endpoint listen addresses By default, mktxp runs it's HTTP metrics endpoint on any IPv4 address on port 49090. However, it is also able to listen on multiple socket addresses, both IPv4 and IPv6. You can configure this behaviour via the following [system option](https://github.com/akpw/mktxp/blob/main/README.md#mktxp-system-configuration), setting ```listen``` to a space-separated list of sockets to listen to, e.g.: ```