KevinMidboe/mktxp-no-cli
1
0
Fork 0
mirror of https://github.com/KevinMidboe/mktxp-no-cli.git synced 2025-10-29 09:40:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'akpw:main' into main

Browse Source
This commit is contained in:
Hugo Rosário
2024-03-12 14:55:20 +00:00
committed by GitHub
parent 28dae306c8 d3d782ebd4
commit d1a7fce8d3
13 changed files with 179 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
README.md
View File

@@ -91,6 +91,8 @@ The default configuration file comes with a sample configuration, making it easy
user = True # Active Users metrics
queue = True # Queues metrics
bgp = False # BGP sessions metrics
remote_dhcp_entry = None # An MKTXP entry for remote DHCP info resolution (capsman/wireless)
@@ -211,7 +213,7 @@ mktxp edit -i
```
[MKTXP]
port = 49090
listen = '0.0.0.0:49090' # Space separated list of socket addresses to listen to, both IPV4 and IPV6
socket_timeout = 2
initial_delay_on_failure = 120
@@ -273,7 +275,7 @@ optional arguments:
While most of the [mktxp options](https://github.com/akpw/mktxp#getting-started) are self explanatory, some might require a bit of a context.
### Remote DHCP resolution
When gathering various IP address-related metrics, MKTXP automatically resolves IP addresses whenever DHCP info is available. In many cases however, the exported devices do not have this information locally and instead rely on central DHCP servers. To improve readibility / usefulness of the exported metrics, MKTXP supports remote DHCP server calls via the following option:
When gathering various IP address-related metrics, MKTXP automatically resolves IP addresses whenever DHCP info is available. In many cases however, the exported devices do not have this information locally and instead rely on central DHCP servers. To improve readability / usefulness of the exported metrics, MKTXP supports remote DHCP server calls via the following option:
```
remote_dhcp_entry = None # An MKTXP entry for remote DHCP info resolution in capsman/wireless
```
@@ -321,11 +323,13 @@ total_max_scrape_duration = 30 # Max overall duration of all metrics collec
To keeps things within expected boundaries, the last two parameters allows for controlling both individual and overall scrape durations
### mktxp port
By default, mktxp runs it's HTTP metrics endpoint on port 49090. You can change it via the following [system option](https://github.com/akpw/mktxp/blob/main/README.md#mktxp-system-configuration):
### mktxp endpoint listen addresses
By default, mktxp runs it's HTTP metrics endpoint on any IPv4 address on port 49090. However, it is also able to listen on multiple socket addresses, both IPv4 and IPv6.
You can configure this behaviour via the following [system option](https://github.com/akpw/mktxp/blob/main/README.md#mktxp-system-configuration), setting ```listen``` to a space-separated list of sockets to listen to, e.g.:
```
port = 49090
listen = '0.0.0.0:49090 [::1]:49090'
```
A wildcard for the hostname is supported as well, and binding to both IPv4/IPv6 as available.
## Setting up MKTXP to run as a Linux Service
If you've installed MKTXP on a Linux system, you can run it with system boot via adding a service. \

8
mktxp/cli/checks/chk_pv.py
View File

@@ -24,20 +24,20 @@ def check_version():
print(\
'''
Mikrotik Prometheus Exporter requires
Python version 3.6 or later.
Python version 3.8 or later.
You can create an isolated Python 3.6 environment
You can create an isolated Python 3.8 environment
with the virtualenv tool:
http://docs.python-guide.org/en/latest/dev/virtualenvs
''')
sys.exit(0)
elif sys.version_info.major == 3 and sys.version_info.minor < 6:
elif sys.version_info.major == 3 and sys.version_info.minor < 8:
print(\
'''
Mikrotik Prometheus Exporter requires
Python version 3.6 or later.
Python version 3.8 or later.
Please upgrade to the latest Python 3.x version.

4
mktxp/cli/config/_mktxp.conf
View File

@@ -12,7 +12,7 @@
[MKTXP]
port = 49090
listen = '0.0.0.0:49090' # Space separated list of socket addresses to listen to, both IPV4 and IPV6
socket_timeout = 2
initial_delay_on_failure = 120
@@ -20,7 +20,7 @@
delay_inc_div = 5
bandwidth = False # Turns metrics bandwidth metrics collection on / off
bandwidth_test_interval = 600 # Interval for colllecting bandwidth metrics
bandwidth_test_interval = 600 # Interval for collecting bandwidth metrics
minimal_collect_interval = 5 # Minimal metric collection interval
verbose_mode = False # Set it on for troubleshooting

27
mktxp/cli/config/config.py
View File

@@ -46,6 +46,7 @@ class CollectorKeys:
QUEUE_SIMPLE_COLLECTOR = 'QueueSimpleCollector'
KID_CONTROL_DEVICE_COLLECTOR = 'KidControlCollector'
USER_COLLECTOR = 'UserCollector'
BGP_COLLECTOR = 'BGPCollector'
MKTXP_COLLECTOR = 'MKTXPCollector'
@@ -56,6 +57,7 @@ class MKTXPConfigKeys:
ENABLED_KEY = 'enabled'
HOST_KEY = 'hostname'
PORT_KEY = 'port'
LISTEN_KEY = 'listen'
USER_KEY = 'username'
PASSWD_KEY = 'password'
@@ -87,6 +89,8 @@ class MKTXPConfigKeys:
FE_USER_KEY = 'user'
FE_QUEUE_KEY = 'queue'
FE_BGP_KEY = 'bgp'
FE_REMOTE_DHCP_ENTRY = 'remote_dhcp_entry'
FE_CHECK_FOR_UPDATES = 'check_for_updates'
@@ -133,7 +137,7 @@ class MKTXPConfigKeys:
BOOLEAN_KEYS_NO = {ENABLED_KEY, SSL_KEY, NO_SSL_CERTIFICATE, FE_CHECK_FOR_UPDATES, FE_KID_CONTROL_DEVICE,
SSL_CERTIFICATE_VERIFY, FE_IPV6_FIREWALL_KEY, FE_IPV6_NEIGHBOR_KEY, FE_CONNECTION_STATS_KEY}
SSL_CERTIFICATE_VERIFY, FE_IPV6_FIREWALL_KEY, FE_IPV6_NEIGHBOR_KEY, FE_CONNECTION_STATS_KEY, FE_BGP_KEY}
# Feature keys enabled by default
BOOLEAN_KEYS_YES = {FE_DHCP_KEY, FE_PACKAGE_KEY, FE_DHCP_LEASE_KEY, FE_DHCP_POOL_KEY, FE_IP_CONNECTIONS_KEY, FE_INTERFACE_KEY, FE_FIREWALL_KEY,
@@ -150,7 +154,7 @@ class MKTXPConfigKeys:
MKTXP_INC_DIV, MKTXP_BANDWIDTH_TEST_INTERVAL, MKTXP_MIN_COLLECT_INTERVAL,
MKTXP_MAX_WORKER_THREADS, MKTXP_MAX_SCRAPE_DURATION, MKTXP_TOTAL_MAX_SCRAPE_DURATION)
# MKTXP config entry nane
# MKTXP config entry name
MKTXP_CONFIG_ENTRY_NAME = 'MKTXP'
@@ -162,9 +166,9 @@ class ConfigEntry:
MKTXPConfigKeys.FE_FIREWALL_KEY, MKTXPConfigKeys.FE_MONITOR_KEY, MKTXPConfigKeys.FE_ROUTE_KEY, MKTXPConfigKeys.FE_WIRELESS_KEY, MKTXPConfigKeys.FE_WIRELESS_CLIENTS_KEY,
MKTXPConfigKeys.FE_IP_CONNECTIONS_KEY, MKTXPConfigKeys.FE_CONNECTION_STATS_KEY, MKTXPConfigKeys.FE_CAPSMAN_KEY, MKTXPConfigKeys.FE_CAPSMAN_CLIENTS_KEY, MKTXPConfigKeys.FE_POE_KEY, MKTXPConfigKeys.FE_NETWATCH_KEY,
MKTXPConfigKeys.MKTXP_USE_COMMENTS_OVER_NAMES, MKTXPConfigKeys.FE_PUBLIC_IP_KEY, MKTXPConfigKeys.FE_IPV6_FIREWALL_KEY, MKTXPConfigKeys.FE_IPV6_NEIGHBOR_KEY,
MKTXPConfigKeys.FE_USER_KEY, MKTXPConfigKeys.FE_QUEUE_KEY, MKTXPConfigKeys.FE_REMOTE_DHCP_ENTRY, MKTXPConfigKeys.FE_CHECK_FOR_UPDATES, MKTXPConfigKeys.FE_KID_CONTROL_DEVICE,
MKTXPConfigKeys.FE_USER_KEY, MKTXPConfigKeys.FE_QUEUE_KEY, MKTXPConfigKeys.FE_REMOTE_DHCP_ENTRY, MKTXPConfigKeys.FE_CHECK_FOR_UPDATES, MKTXPConfigKeys.FE_KID_CONTROL_DEVICE, MKTXPConfigKeys.FE_BGP_KEY,
])
MKTXPSystemEntry = namedtuple('MKTXPSystemEntry', [MKTXPConfigKeys.PORT_KEY, MKTXPConfigKeys.MKTXP_SOCKET_TIMEOUT,
MKTXPSystemEntry = namedtuple('MKTXPSystemEntry', [MKTXPConfigKeys.PORT_KEY, MKTXPConfigKeys.LISTEN_KEY, MKTXPConfigKeys.MKTXP_SOCKET_TIMEOUT,
MKTXPConfigKeys.MKTXP_INITIAL_DELAY, MKTXPConfigKeys.MKTXP_MAX_DELAY,
MKTXPConfigKeys.MKTXP_INC_DIV, MKTXPConfigKeys.MKTXP_BANDWIDTH_KEY,
MKTXPConfigKeys.MKTXP_VERBOSE_MODE, MKTXPConfigKeys.MKTXP_BANDWIDTH_TEST_INTERVAL,
@@ -288,10 +292,10 @@ class MKTXPConfigHandler:
def _read_from_disk(self):
''' (Force-)Read conf data from disk
'''
self.config = ConfigObj(self.usr_conf_data_path)
self.config = ConfigObj(self.usr_conf_data_path, indent_type = ' ')
self.config.preserve_comments = True
self._config = ConfigObj(self.mktxp_conf_path)
self._config = ConfigObj(self.mktxp_conf_path, indent_type = ' ')
self._config.preserve_comments = True
def _create_os_path(self, os_path, resource_path):
@@ -360,7 +364,8 @@ class MKTXPConfigHandler:
system_entry_reader[key] = self._config[entry_name].as_int(key)
else:
system_entry_reader[key] = self._default_value_for_key(key)
new_keys.append(key) # read from disk next time
if key not in (MKTXPConfigKeys.PORT_KEY): # Port key has been depricated
new_keys.append(key) # read from disk next time
for key in MKTXPConfigKeys.SYSTEM_BOOLEAN_KEYS_NO.union(MKTXPConfigKeys.SYSTEM_BOOLEAN_KEYS_YES):
if self._config[entry_name].get(key) is not None:
@@ -369,9 +374,17 @@ class MKTXPConfigHandler:
system_entry_reader[key] = True if key in MKTXPConfigKeys.SYSTEM_BOOLEAN_KEYS_YES else False
new_keys.append(key) # read from disk next time
# listen
if self._config[entry_name].get(MKTXPConfigKeys.LISTEN_KEY):
system_entry_reader[MKTXPConfigKeys.LISTEN_KEY] = self._config[entry_name].get(MKTXPConfigKeys.LISTEN_KEY)
else:
system_entry_reader[MKTXPConfigKeys.LISTEN_KEY] = f'0.0.0.0:{system_entry_reader[MKTXPConfigKeys.PORT_KEY]}'
new_keys.append(MKTXPConfigKeys.LISTEN_KEY) # read from disk next time
if new_keys:
self._config[entry_name] = system_entry_reader
try:
self._config[entry_name].pop(MKTXPConfigKeys.PORT_KEY, None) # Port key has been depricated
self._config.write()
if self._config[entry_name].as_bool(MKTXPConfigKeys.MKTXP_VERBOSE_MODE):
print(f'Updated system entry {entry_name} with new system keys {new_keys}')

2
mktxp/cli/config/mktxp.conf
View File

@@ -52,6 +52,8 @@
user = True # Active Users metrics
queue = True # Queues metrics
bgp = False # BGP sessions metrics
remote_dhcp_entry = None # An MKTXP entry for remote DHCP info resolution (capsman/wireless)

80
mktxp/collector/bgp_collector.py Normal file
View File

@@ -0,0 +1,80 @@
# coding=utf8
## Copyright (c) 2020 Arseniy Kuznetsov
##
## This program is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License
## as published by the Free Software Foundation; either version 2
## of the License, or (at your option) any later version.
##
## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
## GNU General Public License for more details.
from mktxp.collector.base_collector import BaseCollector
from mktxp.flow.processor.output import BaseOutputProcessor
from mktxp.datasource.bgp_ds import BGPMetricsDataSource
class BGPCollector(BaseCollector):
'''BGP collector'''
@staticmethod
def collect(router_entry):
if not router_entry.config_entry.bgp:
return
bgp_labels = ['name', 'remote_address', 'remote_as', 'local_as', 'remote_afi', 'local_afi', 'remote_messages', 'remote_bytes', 'local_messages', 'local_bytes', 'prefix_count', 'established', 'uptime']
bgp_records = BGPMetricsDataSource.metric_records(router_entry, metric_labels=bgp_labels)
if bgp_records:
# translate records to appropriate values
translated_fields = ['established', 'uptime']
for bgp_record in bgp_records:
for translated_field in translated_fields:
value = bgp_record.get(translated_field, None)
if value:
bgp_record[translated_field] = BGPCollector._translated_values(translated_field, value)
session_info_labes = ['name', 'remote_address', 'remote_as', 'local_as', 'remote_afi', 'local_afi']
bgp_sessions_metrics = BaseCollector.info_collector('bgp_sessions_info', 'BGP sessions info', bgp_records, session_info_labes)
yield bgp_sessions_metrics
session_id_labes = ['name']
remote_messages_metrics = BaseCollector.counter_collector('bgp_remote_messages', 'Number of remote messages', bgp_records, 'remote_messages', session_id_labes)
yield remote_messages_metrics
local_messages_metrics = BaseCollector.counter_collector('bgp_local_messages', 'Number of local messages', bgp_records, 'local_messages', session_id_labes)
yield local_messages_metrics
remote_bytes_metrics = BaseCollector.counter_collector('bgp_remote_bytes', 'Number of remote bytes', bgp_records, 'remote_bytes', session_id_labes)
yield remote_bytes_metrics
local_bytes_metrics = BaseCollector.counter_collector('bgp_local_bytes', 'Number of local bytes', bgp_records, 'local_bytes', session_id_labes)
yield local_bytes_metrics
prefix_count_metrics = BaseCollector.gauge_collector('bgp_prefix_count', 'BGP prefix count', bgp_records, 'prefix_count', session_id_labes)
yield prefix_count_metrics
established_metrics = BaseCollector.gauge_collector('bgp_established', 'BGP established', bgp_records, 'established', session_id_labes)
yield established_metrics
uptime_metrics = BaseCollector.gauge_collector('bgp_uptime', 'BGP uptime in milliseconds', bgp_records, 'uptime', session_id_labes)
yield uptime_metrics
# Helpers
@staticmethod
def _translated_values(translated_field, value):
return {
'established': lambda value: '1' if value=='true' else '0',
'uptime': lambda value: BaseOutputProcessor.parse_timedelta_milliseconds(value)
}[translated_field](value)

17
mktxp/datasource/base_ds.py
View File

@@ -23,15 +23,14 @@ class BaseDSProcessor:
if metric_labels is None:
metric_labels = []
if translation_table is None:
translation_table = {}
dash2_ = lambda x : x.replace('-', '_')
translation_table = {}
if len(metric_labels) == 0 and len(router_records) > 0:
metric_labels = [dash2_(key) for key in router_records[0].keys()]
metric_labels = [BaseDSProcessor._normalise_keys(key) for key in router_records[0].keys()]
metric_labels = set(metric_labels)
labeled_records = []
for router_record in router_records:
translated_record = {dash2_(key): value for (key, value) in router_record.items() if dash2_(key) in metric_labels}
translated_record = {BaseDSProcessor._normalise_keys(key): value for (key, value) in router_record.items() if BaseDSProcessor._normalise_keys(key) in metric_labels}
if add_router_id:
for key, value in router_entry.router_id.items():
@@ -43,3 +42,13 @@ class BaseDSProcessor:
labeled_records.append(translated_record)
return labeled_records
@staticmethod
def _normalise_keys(key):
chars = ".-"
for chr in chars:
if chr in key:
key = key.replace(chr, "_")
return key

31
mktxp/datasource/bgp_ds.py Normal file
View File

@@ -0,0 +1,31 @@
# coding=utf8
## Copyright (c) 2020 Arseniy Kuznetsov
##
## This program is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License
## as published by the Free Software Foundation; either version 2
## of the License, or (at your option) any later version.
##
## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
## GNU General Public License for more details.
from mktxp.datasource.base_ds import BaseDSProcessor
class BGPMetricsDataSource:
''' Wireless Metrics data provider
'''
@staticmethod
def metric_records(router_entry, *, metric_labels = None, add_router_id = True):
if metric_labels is None:
metric_labels = []
try:
bgp_records = router_entry.api_connection.router_api().get_resource('/routing/bgp/session').get()
return BaseDSProcessor.trimmed_records(router_entry, router_records = bgp_records, metric_labels = metric_labels, add_router_id = add_router_id)
except Exception as exc:
print(f'Error getting BGP sessions info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}')
return None

3
mktxp/flow/collector_registry.py
View File

@@ -37,6 +37,7 @@ from mktxp.collector.user_collector import UserCollector
from mktxp.collector.queue_collector import QueueTreeCollector
from mktxp.collector.queue_collector import QueueSimpleCollector
from mktxp.collector.kid_control_device_collector import KidDeviceCollector
from mktxp.collector.bgp_collector import BGPCollector
class CollectorRegistry:
''' MKTXP Collectors Registry
@@ -74,6 +75,8 @@ class CollectorRegistry:
self.register(CollectorKeys.QUEUE_SIMPLE_COLLECTOR, QueueSimpleCollector.collect)
self.register(CollectorKeys.KID_CONTROL_DEVICE_COLLECTOR, KidDeviceCollector.collect)
self.register(CollectorKeys.BGP_COLLECTOR, BGPCollector.collect)
self.register(CollectorKeys.MKTXP_COLLECTOR, MKTXPCollector.collect)

16
mktxp/flow/processor/base_proc.py
View File

@@ -12,10 +12,9 @@
## GNU General Public License for more details.
from http.server import HTTPServer
from datetime import datetime
from prometheus_client.core import REGISTRY
from prometheus_client import MetricsHandler
from prometheus_client import make_wsgi_app
from mktxp.cli.config.config import config_handler
from mktxp.flow.collector_handler import CollectorHandler
@@ -27,6 +26,8 @@ from mktxp.cli.output.wifi_out import WirelessOutput
from mktxp.cli.output.dhcp_out import DHCPOutput
from mktxp.cli.output.conn_stats_out import ConnectionsStatsOutput
from waitress import serve
class ExportProcessor:
''' Base Export Processing
@@ -34,16 +35,9 @@ class ExportProcessor:
@staticmethod
def start():
REGISTRY.register(CollectorHandler(RouterEntriesHandler(), CollectorRegistry()))
ExportProcessor.run(port=config_handler.system_entry().port)
@staticmethod
def run(server_class=HTTPServer, handler_class=MetricsHandler, port=None):
server_address = ('', port)
httpd = server_class(server_address, handler_class)
current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
print(f'{current_time} Running HTTP metrics server on port {port}')
httpd.serve_forever()
print(f'{current_time} Running HTTP metrics server on: {config_handler.system_entry().listen}')
serve(make_wsgi_app(), listen = config_handler.system_entry().listen)
class OutputProcessor:
''' Base CLI Processing

6
mktxp/flow/processor/output.py
View File

@@ -117,7 +117,7 @@ class BaseOutputProcessor:
def parse_timedelta(time):
duration_interval_rgx = config_handler.re_compiled.get('duration_interval_rgx')
if not duration_interval_rgx:
duration_interval_rgx = re.compile(r'((?P<weeks>\d+)w)?((?P<days>\d+)d)?((?P<hours>\d+)h)?((?P<minutes>\d+)m)?((?P<seconds>\d+)s)?')
duration_interval_rgx = re.compile(r'((?P<weeks>\d+)w)?((?P<days>\d+)d)?((?P<hours>\d+)h)?((?P<minutes>\d+)m)?((?P<seconds>\d+)s)?((?P<milliseconds>\d+)ms)?')
config_handler.re_compiled['duration_interval_rgx'] = duration_interval_rgx
time_dict = duration_interval_rgx.match(time).groupdict()
return timedelta(**{key: int(value) for key, value in time_dict.items() if value})
@@ -126,6 +126,10 @@ class BaseOutputProcessor:
def parse_timedelta_seconds(time):
return BaseOutputProcessor.parse_timedelta(time).total_seconds()
@staticmethod
def parse_timedelta_milliseconds(time):
return BaseOutputProcessor.parse_timedelta(time) / timedelta(milliseconds=1)
@staticmethod
def parse_signal_strength(signal_strength):
wifi_signal_strength_rgx = config_handler.re_compiled.get('wifi_signal_strength_rgx')

1
mktxp/flow/router_entry.py
View File

@@ -66,6 +66,7 @@ class RouterEntry:
CollectorKeys.QUEUE_SIMPLE_COLLECTOR: 0,
CollectorKeys.KID_CONTROL_DEVICE_COLLECTOR: 0,
CollectorKeys.USER_COLLECTOR: 0,
CollectorKeys.BGP_COLLECTOR: 0,
CollectorKeys.MKTXP_COLLECTOR: 0
}
self._dhcp_entry = None

7
setup.py
View File

@@ -20,7 +20,7 @@ with open(path.join(pkg_dir, 'README.md'), encoding='utf-8') as f:
setup(
name='mktxp',
version='1.2.2',
version='1.2.3',
url='https://github.com/akpw/mktxp',
@@ -48,7 +48,8 @@ setup(
'configobj>=5.0.6',
'humanize>=3.2.0',
'texttable>=1.6.3',
'speedtest-cli>=2.1.2'
'speedtest-cli>=2.1.2',
'waitress>=3.0.0',
],
test_suite = 'tests.mktxp_test_suite',
@@ -63,7 +64,7 @@ setup(
'Development Status :: 4 - Beta',
'License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)',
'Programming Language :: Python',
'Programming Language :: Python :: 3.6',
'Programming Language :: Python :: 3.8',
'Programming Language :: Python :: 3 :: Only',
'Intended Audience :: Developers',
'Intended Audience :: System Administrators',