KevinMidboe/mktxp-no-cli
1
0
Fork 0
mirror of https://github.com/KevinMidboe/mktxp-no-cli.git synced 2025-10-29 17:50:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

Shared default config (->#119)

Browse Source
This commit is contained in:
Arseniy Kuznetsov
2024-03-17 14:36:50 +01:00
parent eff2d2b39b
commit 64b91ec829
10 changed files with 168 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
README.md
View File

@@ -50,11 +50,20 @@ To get started with MKTXP, you need to edit its main configuration file. This es
The default configuration file comes with a sample configuration, making it easy to copy / edit parameters for your RouterOS devices as needed: The default configuration file comes with a sample configuration, making it easy to copy / edit parameters for your RouterOS devices as needed:
``` ```
[Sample-Router] [Sample-Router-1]
enabled = False # turns metrics collection for this RouterOS device on / off # for specific configuration on the router level, overload the defaults here
hostname = 192.168.88.1
[Sample-Router-2]
# for specific configuration on the router level, overload the defaults here
hostname = 192.168.88.2
[default]
# this affects configuration of all routers, unless overloaded on their specific levels
enabled = True # turns metrics collection for this RouterOS device on / off
hostname = localhost # RouterOS IP address hostname = localhost # RouterOS IP address
port = 8728 # RouterOS API / API-SSL service port port = 8728 # RouterOS IP Port
username = username # RouterOS user, needs to have 'read' and 'api' permissions username = username # RouterOS user, needs to have 'read' and 'api' permissions
password = password password = password
@@ -102,7 +111,7 @@ The default configuration file comes with a sample configuration, making it easy
``` ```
Most options are easy to understand at first glance, and some are described in more details [later](https://github.com/akpw/mktxp#advanced-features). Most options are easy to understand at first glance, and some are described in more details [later](https://github.com/akpw/mktxp#advanced-features).
<sup>💡</sup> To automatically optimise older format of the `mktxp.conf` for existing installs, set `compact_default_conf_values = True` in the system config `_mktxp.conf`
#### Local install #### Local install
If you have a local MKTXP installation, you can edit the configuration file with your default system editor directly from mktxp: If you have a local MKTXP installation, you can edit the configuration file with your default system editor directly from mktxp:
@@ -230,6 +239,8 @@ mktxp edit -i
max_worker_threads = 5 # Max number of worker threads that can fetch routers (parallel fetch only) max_worker_threads = 5 # Max number of worker threads that can fetch routers (parallel fetch only)
max_scrape_duration = 10 # Max duration of individual routers' metrics collection (parallel fetch only) max_scrape_duration = 10 # Max duration of individual routers' metrics collection (parallel fetch only)
total_max_scrape_duration = 30 # Max overall duration of all metrics collection (parallel fetch only) total_max_scrape_duration = 30 # Max overall duration of all metrics collection (parallel fetch only)
compact_default_conf_values = True # Compact mktxp.conf, so only specific values are kept on the individual routers' level
``` ```
<sup>💡</sup> *When changing the default mktxp port for [docker image installs](https://github.com/akpw/mktxp#docker-image-install), you'll need to adjust the `docker run ... -p 49090:49090 ...` command to reflect the new port* <sup>💡</sup> *When changing the default mktxp port for [docker image installs](https://github.com/akpw/mktxp#docker-image-install), you'll need to adjust the `docker run ... -p 49090:49090 ...` command to reflect the new port*

2
mktxp/cli/config/_mktxp.conf
View File

@@ -29,3 +29,5 @@
max_worker_threads = 5 # Max number of worker threads that can fetch routers (parallel fetch only) max_worker_threads = 5 # Max number of worker threads that can fetch routers (parallel fetch only)
max_scrape_duration = 10 # Max duration of individual routers' metrics collection (parallel fetch only) max_scrape_duration = 10 # Max duration of individual routers' metrics collection (parallel fetch only)
total_max_scrape_duration = 30 # Max overall duration of all metrics collection (parallel fetch only) total_max_scrape_duration = 30 # Max overall duration of all metrics collection (parallel fetch only)
compact_default_conf_values = True # Compact mktxp.conf, so only specific values are kept on the individual routers' level

178
mktxp/cli/config/config.py
View File

@@ -109,6 +109,7 @@ class MKTXPConfigKeys:
MKTXP_MAX_WORKER_THREADS = 'max_worker_threads' MKTXP_MAX_WORKER_THREADS = 'max_worker_threads'
MKTXP_MAX_SCRAPE_DURATION = 'max_scrape_duration' MKTXP_MAX_SCRAPE_DURATION = 'max_scrape_duration'
MKTXP_TOTAL_MAX_SCRAPE_DURATION = 'total_max_scrape_duration' MKTXP_TOTAL_MAX_SCRAPE_DURATION = 'total_max_scrape_duration'
MKTXP_COMPACT_CONFIG = 'compact_default_conf_values'
# UnRegistered entries placeholder # UnRegistered entries placeholder
NO_ENTRIES_REGISTERED = 'NoEntriesRegistered' NO_ENTRIES_REGISTERED = 'NoEntriesRegistered'
@@ -120,6 +121,10 @@ class MKTXPConfigKeys:
ROUTERBOARD_ADDRESS = 'routerboard_address' ROUTERBOARD_ADDRESS = 'routerboard_address'
# Default values # Default values
DEFAULT_HOST_KEY = 'localhost'
DEFAULT_USER_KEY = 'user'
DEFAULT_PASSWORD_KEY = 'password'
DEFAULT_API_PORT = 8728 DEFAULT_API_PORT = 8728
DEFAULT_API_SSL_PORT = 8729 DEFAULT_API_SSL_PORT = 8729
DEFAULT_FE_REMOTE_DHCP_ENTRY = 'None' DEFAULT_FE_REMOTE_DHCP_ENTRY = 'None'
@@ -130,7 +135,6 @@ class MKTXPConfigKeys:
DEFAULT_MKTXP_INC_DIV = 5 DEFAULT_MKTXP_INC_DIV = 5
DEFAULT_MKTXP_BANDWIDTH_TEST_INTERVAL = 420 DEFAULT_MKTXP_BANDWIDTH_TEST_INTERVAL = 420
DEFAULT_MKTXP_MIN_COLLECT_INTERVAL = 5 DEFAULT_MKTXP_MIN_COLLECT_INTERVAL = 5
DEFAULT_MKTXP_FETCH_IN_PARALLEL = False
DEFAULT_MKTXP_MAX_WORKER_THREADS = 5 DEFAULT_MKTXP_MAX_WORKER_THREADS = 5
DEFAULT_MKTXP_MAX_SCRAPE_DURATION = 10 DEFAULT_MKTXP_MAX_SCRAPE_DURATION = 10
DEFAULT_MKTXP_TOTAL_MAX_SCRAPE_DURATION = 30 DEFAULT_MKTXP_TOTAL_MAX_SCRAPE_DURATION = 30
@@ -146,7 +150,7 @@ class MKTXPConfigKeys:
FE_NETWATCH_KEY, FE_PUBLIC_IP_KEY, FE_USER_KEY, FE_QUEUE_KEY} FE_NETWATCH_KEY, FE_PUBLIC_IP_KEY, FE_USER_KEY, FE_QUEUE_KEY}
SYSTEM_BOOLEAN_KEYS_YES = set() SYSTEM_BOOLEAN_KEYS_YES = set()
SYSTEM_BOOLEAN_KEYS_NO = {MKTXP_BANDWIDTH_KEY, MKTXP_VERBOSE_MODE, MKTXP_FETCH_IN_PARALLEL} SYSTEM_BOOLEAN_KEYS_NO = {MKTXP_BANDWIDTH_KEY, MKTXP_VERBOSE_MODE, MKTXP_FETCH_IN_PARALLEL, MKTXP_COMPACT_CONFIG}
STR_KEYS = (HOST_KEY, USER_KEY, PASSWD_KEY, FE_REMOTE_DHCP_ENTRY) STR_KEYS = (HOST_KEY, USER_KEY, PASSWD_KEY, FE_REMOTE_DHCP_ENTRY)
INT_KEYS = () INT_KEYS = ()
@@ -154,7 +158,8 @@ class MKTXPConfigKeys:
MKTXP_INC_DIV, MKTXP_BANDWIDTH_TEST_INTERVAL, MKTXP_MIN_COLLECT_INTERVAL, MKTXP_INC_DIV, MKTXP_BANDWIDTH_TEST_INTERVAL, MKTXP_MIN_COLLECT_INTERVAL,
MKTXP_MAX_WORKER_THREADS, MKTXP_MAX_SCRAPE_DURATION, MKTXP_TOTAL_MAX_SCRAPE_DURATION) MKTXP_MAX_WORKER_THREADS, MKTXP_MAX_SCRAPE_DURATION, MKTXP_TOTAL_MAX_SCRAPE_DURATION)
# MKTXP config entry name # MKTXP configs entry names
DEFAULT_ENTRY_KEY = 'default'
MKTXP_CONFIG_ENTRY_NAME = 'MKTXP' MKTXP_CONFIG_ENTRY_NAME = 'MKTXP'
@@ -174,7 +179,7 @@ class ConfigEntry:
MKTXPConfigKeys.MKTXP_VERBOSE_MODE, MKTXPConfigKeys.MKTXP_BANDWIDTH_TEST_INTERVAL, MKTXPConfigKeys.MKTXP_VERBOSE_MODE, MKTXPConfigKeys.MKTXP_BANDWIDTH_TEST_INTERVAL,
MKTXPConfigKeys.MKTXP_MIN_COLLECT_INTERVAL, MKTXPConfigKeys.MKTXP_FETCH_IN_PARALLEL, MKTXPConfigKeys.MKTXP_MIN_COLLECT_INTERVAL, MKTXPConfigKeys.MKTXP_FETCH_IN_PARALLEL,
MKTXPConfigKeys.MKTXP_MAX_WORKER_THREADS, MKTXPConfigKeys.MKTXP_MAX_SCRAPE_DURATION, MKTXPConfigKeys.MKTXP_MAX_WORKER_THREADS, MKTXPConfigKeys.MKTXP_MAX_SCRAPE_DURATION,
MKTXPConfigKeys.MKTXP_TOTAL_MAX_SCRAPE_DURATION]) MKTXPConfigKeys.MKTXP_TOTAL_MAX_SCRAPE_DURATION, MKTXPConfigKeys.MKTXP_COMPACT_CONFIG])
class OSConfig(metaclass=ABCMeta): class OSConfig(metaclass=ABCMeta):
@@ -265,11 +270,14 @@ class MKTXPConfigHandler:
self._read_from_disk() self._read_from_disk()
self.default_config_entry_reader = self._default_config_entry_reader()
self.system_entry = self._system_entry()
# MKTXP entries # MKTXP entries
def registered_entries(self): def registered_entries(self):
''' All MKTXP registered entries ''' All MKTXP registered entries
''' '''
return (entry_name for entry_name in self.config.keys()) return (entry_name for entry_name in self.config.keys() if entry_name != MKTXPConfigKeys.DEFAULT_ENTRY_KEY )
def registered_entry(self, entry_name): def registered_entry(self, entry_name):
''' A specific MKTXP registered entry by name ''' A specific MKTXP registered entry by name
@@ -282,13 +290,13 @@ class MKTXPConfigHandler:
entry_reader = self._config_entry_reader(entry_name) entry_reader = self._config_entry_reader(entry_name)
return ConfigEntry.MKTXPConfigEntry(**entry_reader) if entry_reader else None return ConfigEntry.MKTXPConfigEntry(**entry_reader) if entry_reader else None
def system_entry(self): # Helpers
def _system_entry(self):
''' MKTXP internal config entry ''' MKTXP internal config entry
''' '''
_entry_reader = self._system_entry_reader() _entry_reader = self._system_entry_reader()
return ConfigEntry.MKTXPSystemEntry(**_entry_reader) return ConfigEntry.MKTXPSystemEntry(**_entry_reader)
# Helpers
def _read_from_disk(self): def _read_from_disk(self):
''' (Force-)Read conf data from disk ''' (Force-)Read conf data from disk
''' '''
@@ -305,55 +313,6 @@ class MKTXPConfigHandler:
Requirement.parse("mktxp"), resource_path) Requirement.parse("mktxp"), resource_path)
shutil.copy(lookup_path, os_path) shutil.copy(lookup_path, os_path)
def _config_entry_reader(self, entry_name):
config_entry_reader = {}
new_keys = []
for key in MKTXPConfigKeys.BOOLEAN_KEYS_NO.union(MKTXPConfigKeys.BOOLEAN_KEYS_YES):
if self.config[entry_name].get(key) is not None:
config_entry_reader[key] = self.config[entry_name].as_bool(key)
else:
config_entry_reader[key] = True if key in MKTXPConfigKeys.BOOLEAN_KEYS_YES else False
new_keys.append(key) # read from disk next time
for key in MKTXPConfigKeys.STR_KEYS:
if self.config[entry_name].get(key):
config_entry_reader[key] = self.config[entry_name].get(key)
else:
config_entry_reader[key] = self._default_value_for_key(key)
new_keys.append(key) # read from disk next time
if key is MKTXPConfigKeys.PASSWD_KEY and type(config_entry_reader[key]) is list:
config_entry_reader[key] = ','.join(config_entry_reader[key])
for key in MKTXPConfigKeys.INT_KEYS:
if self.config[entry_name].get(key):
config_entry_reader[key] = self.config[entry_name].as_int(key)
else:
config_entry_reader[key] = self._default_value_for_key(key)
new_keys.append(key) # read from disk next time
# port
if self.config[entry_name].get(MKTXPConfigKeys.PORT_KEY):
config_entry_reader[MKTXPConfigKeys.PORT_KEY] = self.config[entry_name].as_int(
MKTXPConfigKeys.PORT_KEY)
else:
config_entry_reader[MKTXPConfigKeys.PORT_KEY] = self._default_value_for_key(
MKTXPConfigKeys.SSL_KEY, config_entry_reader[MKTXPConfigKeys.SSL_KEY])
new_keys.append(MKTXPConfigKeys.PORT_KEY) # read from disk next time
if new_keys:
self.config[entry_name] = config_entry_reader
try:
self.config.write()
if self._config[MKTXPConfigKeys.MKTXP_CONFIG_ENTRY_NAME].as_bool(MKTXPConfigKeys.MKTXP_VERBOSE_MODE):
print(f'Updated router entry {entry_name} with new feature keys {new_keys}')
except Exception as exc:
print(f'Error updating router entry {entry_name} with new feature keys {new_keys}: {exc}')
print('Please update mktxp.conf to its latest version manually')
return config_entry_reader
def _system_entry_reader(self): def _system_entry_reader(self):
system_entry_reader = {} system_entry_reader = {}
entry_name = MKTXPConfigKeys.MKTXP_CONFIG_ENTRY_NAME entry_name = MKTXPConfigKeys.MKTXP_CONFIG_ENTRY_NAME
@@ -394,9 +353,115 @@ class MKTXPConfigHandler:
return system_entry_reader return system_entry_reader
def _config_entry_reader(self, entry_name):
config_entry_reader = {}
compact_config = self._config[MKTXPConfigKeys.MKTXP_CONFIG_ENTRY_NAME].as_bool(MKTXPConfigKeys.MKTXP_COMPACT_CONFIG)
drop_keys = []
for key in MKTXPConfigKeys.BOOLEAN_KEYS_NO.union(MKTXPConfigKeys.BOOLEAN_KEYS_YES):
if self.config[entry_name].get(key) is not None:
config_entry_reader[key] = self.config[entry_name].as_bool(key)
if compact_config and config_entry_reader[key] == self.default_config_entry_reader[key]:
drop_keys.append(key)
else:
config_entry_reader[key] = self.default_config_entry_reader[key]
for key in MKTXPConfigKeys.STR_KEYS:
if self.config[entry_name].get(key):
config_entry_reader[key] = self.config[entry_name].get(key)
if key is MKTXPConfigKeys.PASSWD_KEY and type(config_entry_reader[key]) is list:
config_entry_reader[key] = ','.join(config_entry_reader[key])
if compact_config and config_entry_reader[key] == self.default_config_entry_reader[key]:
drop_keys.append(key)
else:
config_entry_reader[key] = self.default_config_entry_reader[key]
for key in MKTXPConfigKeys.INT_KEYS:
if self.config[entry_name].get(key):
config_entry_reader[key] = self.config[entry_name].as_int(key)
if compact_config and config_entry_reader[key] == self.default_config_entry_reader[key]:
drop_keys.append(key)
else:
config_entry_reader[key] = self.default_config_entry_reader[key]
# port
if self.config[entry_name].get(MKTXPConfigKeys.PORT_KEY):
config_entry_reader[MKTXPConfigKeys.PORT_KEY] = self.config[entry_name].as_int(MKTXPConfigKeys.PORT_KEY)
if compact_config and config_entry_reader[MKTXPConfigKeys.PORT_KEY] == self.default_config_entry_reader[MKTXPConfigKeys.PORT_KEY]:
drop_keys.append(MKTXPConfigKeys.PORT_KEY)
else:
config_entry_reader[MKTXPConfigKeys.PORT_KEY] = self.default_config_entry_reader[key]
# If allowed, compact mktxp.conf entry
if drop_keys and compact_config:
for key in drop_keys:
self.config[entry_name].pop(key, None)
try:
self.config.write()
if self._config[MKTXPConfigKeys.MKTXP_CONFIG_ENTRY_NAME].as_bool(MKTXPConfigKeys.MKTXP_VERBOSE_MODE):
print(f'compacted router entry {entry_name} for default values of the feature keys {drop_keys}')
except Exception as exc:
print(f'Error compacting router entry {entry_name} for default values of feature keys {drop_keys}: {exc}')
print(f'Error compacting router entry {entry_name} for default values of feature keys {drop_keys}: {exc}')
print('Please compact mktxp.conf manually')
return config_entry_reader
def _default_config_entry_reader(self):
default_config_entry_reader = {}
new_keys = []
if not self.config.get(MKTXPConfigKeys.DEFAULT_ENTRY_KEY):
self.config[MKTXPConfigKeys.DEFAULT_ENTRY_KEY] = {}
for key in MKTXPConfigKeys.BOOLEAN_KEYS_NO.union(MKTXPConfigKeys.BOOLEAN_KEYS_YES):
if self.config[MKTXPConfigKeys.DEFAULT_ENTRY_KEY].get(key) is not None:
default_config_entry_reader[key] = self.config[MKTXPConfigKeys.DEFAULT_ENTRY_KEY].as_bool(key)
else:
default_config_entry_reader[key] = True if key in MKTXPConfigKeys.BOOLEAN_KEYS_YES else False
new_keys.append(key) # read from disk next time
for key in MKTXPConfigKeys.STR_KEYS:
if self.config[MKTXPConfigKeys.DEFAULT_ENTRY_KEY].get(key):
default_config_entry_reader[key] = self.config[MKTXPConfigKeys.DEFAULT_ENTRY_KEY].get(key)
else:
default_config_entry_reader[key] = self._default_value_for_key(key)
new_keys.append(key) # read from disk next time
for key in MKTXPConfigKeys.INT_KEYS:
if self.config[MKTXPConfigKeys.DEFAULT_ENTRY_KEY].get(key):
default_config_entry_reader[key] = self.config[MKTXPConfigKeys.DEFAULT_ENTRY_KEY].as_int(key)
else:
default_config_entry_reader[key] = self._default_value_for_key(key)
new_keys.append(key) # read from disk next time
# port
if self.config[MKTXPConfigKeys.DEFAULT_ENTRY_KEY].get(MKTXPConfigKeys.PORT_KEY):
default_config_entry_reader[MKTXPConfigKeys.PORT_KEY] = self.config[MKTXPConfigKeys.DEFAULT_ENTRY_KEY].as_int(MKTXPConfigKeys.PORT_KEY)
else:
default_config_entry_reader[MKTXPConfigKeys.PORT_KEY] = self._default_value_for_key(
MKTXPConfigKeys.SSL_KEY, default_config_entry_reader[MKTXPConfigKeys.SSL_KEY])
new_keys.append(MKTXPConfigKeys.PORT_KEY) # read from disk next time
if new_keys:
self.config[MKTXPConfigKeys.DEFAULT_ENTRY_KEY] = default_config_entry_reader
try:
self.config.write()
if self._config[MKTXPConfigKeys.MKTXP_CONFIG_ENTRY_NAME].as_bool(MKTXPConfigKeys.MKTXP_VERBOSE_MODE):
print(f'Updated default router entry with new feature keys {new_keys}')
except Exception as exc:
print(f'Error updating default router entry with new feature keys {new_keys}: {exc}')
print('Please update mktxp.conf to its latest version manually')
return default_config_entry_reader
def _default_value_for_key(self, key, value=None): def _default_value_for_key(self, key, value=None):
return { return {
MKTXPConfigKeys.SSL_KEY: lambda value: MKTXPConfigKeys.DEFAULT_API_SSL_PORT if value else MKTXPConfigKeys.DEFAULT_API_PORT, MKTXPConfigKeys.SSL_KEY: lambda value: MKTXPConfigKeys.DEFAULT_API_SSL_PORT if value else MKTXPConfigKeys.DEFAULT_API_PORT,
MKTXPConfigKeys.HOST_KEY: lambda _: MKTXPConfigKeys.DEFAULT_HOST_KEY,
MKTXPConfigKeys.USER_KEY: lambda _: MKTXPConfigKeys.DEFAULT_USER_KEY,
MKTXPConfigKeys.PASSWD_KEY: lambda _: MKTXPConfigKeys.DEFAULT_PASSWORD_KEY,
MKTXPConfigKeys.PORT_KEY: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_PORT, MKTXPConfigKeys.PORT_KEY: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_PORT,
MKTXPConfigKeys.FE_REMOTE_DHCP_ENTRY: lambda _: MKTXPConfigKeys.DEFAULT_FE_REMOTE_DHCP_ENTRY, MKTXPConfigKeys.FE_REMOTE_DHCP_ENTRY: lambda _: MKTXPConfigKeys.DEFAULT_FE_REMOTE_DHCP_ENTRY,
MKTXPConfigKeys.MKTXP_SOCKET_TIMEOUT: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_SOCKET_TIMEOUT, MKTXPConfigKeys.MKTXP_SOCKET_TIMEOUT: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_SOCKET_TIMEOUT,
@@ -405,7 +470,6 @@ class MKTXPConfigHandler:
MKTXPConfigKeys.MKTXP_INC_DIV: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_INC_DIV, MKTXPConfigKeys.MKTXP_INC_DIV: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_INC_DIV,
MKTXPConfigKeys.MKTXP_BANDWIDTH_TEST_INTERVAL: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_BANDWIDTH_TEST_INTERVAL, MKTXPConfigKeys.MKTXP_BANDWIDTH_TEST_INTERVAL: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_BANDWIDTH_TEST_INTERVAL,
MKTXPConfigKeys.MKTXP_MIN_COLLECT_INTERVAL: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_MIN_COLLECT_INTERVAL, MKTXPConfigKeys.MKTXP_MIN_COLLECT_INTERVAL: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_MIN_COLLECT_INTERVAL,
MKTXPConfigKeys.MKTXP_FETCH_IN_PARALLEL: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_FETCH_IN_PARALLEL,
MKTXPConfigKeys.MKTXP_MAX_WORKER_THREADS: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_MAX_WORKER_THREADS, MKTXPConfigKeys.MKTXP_MAX_WORKER_THREADS: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_MAX_WORKER_THREADS,
MKTXPConfigKeys.MKTXP_MAX_SCRAPE_DURATION: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_MAX_SCRAPE_DURATION, MKTXPConfigKeys.MKTXP_MAX_SCRAPE_DURATION: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_MAX_SCRAPE_DURATION,
MKTXPConfigKeys.MKTXP_TOTAL_MAX_SCRAPE_DURATION: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_TOTAL_MAX_SCRAPE_DURATION, MKTXPConfigKeys.MKTXP_TOTAL_MAX_SCRAPE_DURATION: lambda _: MKTXPConfigKeys.DEFAULT_MKTXP_TOTAL_MAX_SCRAPE_DURATION,

7
mktxp/cli/config/mktxp.conf
View File

@@ -12,8 +12,13 @@
[Sample-Router] [Sample-Router]
enabled = False # turns metrics collection for this RouterOS device on / off # for specific configuration on the router level, change here the defaults values from below
hostname = 192.168.88.1
[default]
# this affects configuration of all routers, unless overloaded on their specific levels
enabled = True # turns metrics collection for this RouterOS device on / off
hostname = localhost # RouterOS IP address hostname = localhost # RouterOS IP address
port = 8728 # RouterOS IP Port port = 8728 # RouterOS IP Port

7
mktxp/cli/options.py
View File

@@ -13,11 +13,10 @@
import os import os
import pkg_resources
from argparse import ArgumentParser, HelpFormatter from argparse import ArgumentParser, HelpFormatter
from mktxp.cli.config.config import config_handler, MKTXPConfigKeys, CustomConfig from mktxp.cli.config.config import config_handler, MKTXPConfigKeys, CustomConfig
from mktxp.utils.utils import FSHelper, UniquePartialMatchList, run_cmd from mktxp.utils.utils import FSHelper, UniquePartialMatchList, run_cmd
from importlib.metadata import version as Version
class MKTXPCommands: class MKTXPCommands:
INFO = 'info' INFO = 'info'
@@ -41,7 +40,7 @@ class MKTXPOptionsParser:
''' '''
def __init__(self): def __init__(self):
self._script_name = f'MKTXP' self._script_name = f'MKTXP'
version = pkg_resources.require("mktxp")[0].version version = Version('mktxp')
self._description = \ self._description = \
f''' f'''
Prometheus Exporter for Mikrotik RouterOS, version {version} Prometheus Exporter for Mikrotik RouterOS, version {version}
@@ -232,7 +231,7 @@ Selected metrics info can be printed on the command line. For more information,
return editor return editor
commands = ['which nano', 'which vi', 'which vim'] commands = ['which nano', 'which vi', 'which vim']
quiet = not config_handler.system_entry().verbose_mode quiet = not config_handler.system_entry.verbose_mode
for command in commands: for command in commands:
editor = run_cmd(command, quiet = quiet).rstrip() editor = run_cmd(command, quiet = quiet).rstrip()
if editor: if editor:

4
mktxp/collector/bandwidth_collector.py
View File

@@ -33,7 +33,7 @@ class BandwidthCollector(BaseCollector):
self.last_call_timestamp = 0 self.last_call_timestamp = 0
def collect(self): def collect(self):
if not config_handler.system_entry().bandwidth: if not config_handler.system_entry.bandwidth:
return return
if result_list: if result_list:
@@ -49,7 +49,7 @@ class BandwidthCollector(BaseCollector):
yield latency_metrics yield latency_metrics
ts = datetime.now().timestamp() ts = datetime.now().timestamp()
if (ts - self.last_call_timestamp) > config_handler.system_entry().bandwidth_test_interval: if (ts - self.last_call_timestamp) > config_handler.system_entry.bandwidth_test_interval:
self.pool.apply_async(BandwidthCollector.bandwidth_worker, callback=get_result) self.pool.apply_async(BandwidthCollector.bandwidth_worker, callback=get_result)
self.last_call_timestamp = ts self.last_call_timestamp = ts

14
mktxp/flow/collector_handler.py
View File

@@ -77,7 +77,7 @@ class CollectorHandler:
# overall scrape duration # overall scrape duration
total_scrape_timeout_event = Event() total_scrape_timeout_event = Event()
total_scrape_timer = Timer(config_handler.system_entry().total_max_scrape_duration, timeout, args=(total_scrape_timeout_event,)) total_scrape_timer = Timer(config_handler.system_entry.total_max_scrape_duration, timeout, args=(total_scrape_timeout_event,))
total_scrape_timer.start() total_scrape_timer.start()
with ThreadPoolExecutor(max_workers=max_worker_threads) as executor: with ThreadPoolExecutor(max_workers=max_worker_threads) as executor:
@@ -94,7 +94,7 @@ class CollectorHandler:
# Duration of individual scrapes # Duration of individual scrapes
scrape_timeout_event = Event() scrape_timeout_event = Event()
scrape_timer = Timer(config_handler.system_entry().max_scrape_duration, timeout, args=(scrape_timeout_event,)) scrape_timer = Timer(config_handler.system_entry.max_scrape_duration, timeout, args=(scrape_timeout_event,))
scrape_timer.start() scrape_timer.start()
futures[executor.submit(self.collect_router_entry_async, router_entry, scrape_timeout_event, total_scrape_timeout_event)] = scrape_timer futures[executor.submit(self.collect_router_entry_async, router_entry, scrape_timeout_event, total_scrape_timeout_event)] = scrape_timer
@@ -117,8 +117,8 @@ class CollectorHandler:
# all other collectors # all other collectors
# Check whether to run in parallel by looking at the mktxp system configuration # Check whether to run in parallel by looking at the mktxp system configuration
parallel = config_handler.system_entry().fetch_routers_in_parallel parallel = config_handler.system_entry.fetch_routers_in_parallel
max_worker_threads = config_handler.system_entry().max_worker_threads max_worker_threads = config_handler.system_entry.max_worker_threads
if parallel: if parallel:
yield from self.collect_async(max_worker_threads=max_worker_threads) yield from self.collect_async(max_worker_threads=max_worker_threads)
else: else:
@@ -128,9 +128,9 @@ class CollectorHandler:
def _valid_collect_interval(self): def _valid_collect_interval(self):
now = datetime.now().timestamp() now = datetime.now().timestamp()
diff = now - self.last_collect_timestamp diff = now - self.last_collect_timestamp
if diff < config_handler.system_entry().minimal_collect_interval: if diff < config_handler.system_entry.minimal_collect_interval:
if config_handler.system_entry().verbose_mode: if config_handler.system_entry.verbose_mode:
print(f'An attemp to collect metrics within minimal metrics collection interval: {diff} < {config_handler.system_entry().minimal_collect_interval}') print(f'An attemp to collect metrics within minimal metrics collection interval: {diff} < {config_handler.system_entry.minimal_collect_interval}')
print('deferring..') print('deferring..')
return False return False

4
mktxp/flow/processor/base_proc.py
View File

@@ -36,8 +36,8 @@ class ExportProcessor:
def start(): def start():
REGISTRY.register(CollectorHandler(RouterEntriesHandler(), CollectorRegistry())) REGISTRY.register(CollectorHandler(RouterEntriesHandler(), CollectorRegistry()))
current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
print(f'{current_time} Running HTTP metrics server on: {config_handler.system_entry().listen}') print(f'{current_time} Running HTTP metrics server on: {config_handler.system_entry.listen}')
serve(make_wsgi_app(), listen = config_handler.system_entry().listen) serve(make_wsgi_app(), listen = config_handler.system_entry.listen)
class OutputProcessor: class OutputProcessor:
''' Base CLI Processing ''' Base CLI Processing

8
mktxp/flow/router_connection.py
View File

@@ -59,7 +59,7 @@ class RouterAPIConnection:
ssl_verify = self.config_entry.ssl_certificate_verify, ssl_verify = self.config_entry.ssl_certificate_verify,
ssl_context = ctx) ssl_context = ctx)
self.connection.socket_timeout = config_handler.system_entry().socket_timeout self.connection.socket_timeout = config_handler.system_entry.socket_timeout
self.api = None self.api = None
def is_connected(self): def is_connected(self):
@@ -92,11 +92,11 @@ class RouterAPIConnection:
def _in_connect_timeout(self, connect_timestamp): def _in_connect_timeout(self, connect_timestamp):
connect_delay = self._connect_delay() connect_delay = self._connect_delay()
if (connect_timestamp - self.last_failure_timestamp) < connect_delay: if (connect_timestamp - self.last_failure_timestamp) < connect_delay:
if config_handler.system_entry().verbose_mode: if config_handler.system_entry.verbose_mode:
print(f'{self.router_name}@{self.config_entry.hostname}: in connect timeout, {int(connect_delay - (connect_timestamp - self.last_failure_timestamp))}secs remaining') print(f'{self.router_name}@{self.config_entry.hostname}: in connect timeout, {int(connect_delay - (connect_timestamp - self.last_failure_timestamp))}secs remaining')
print(f'Successive failure count: {self.successive_failure_count}') print(f'Successive failure count: {self.successive_failure_count}')
return True return True
if config_handler.system_entry().verbose_mode: if config_handler.system_entry.verbose_mode:
print(f'{self.router_name}@{self.config_entry.hostname}: OK to connect') print(f'{self.router_name}@{self.config_entry.hostname}: OK to connect')
if self.last_failure_timestamp > 0: if self.last_failure_timestamp > 0:
print(f'Seconds since last failure: {connect_timestamp - self.last_failure_timestamp}') print(f'Seconds since last failure: {connect_timestamp - self.last_failure_timestamp}')
@@ -104,7 +104,7 @@ class RouterAPIConnection:
return False return False
def _connect_delay(self): def _connect_delay(self):
mktxp_entry = config_handler.system_entry() mktxp_entry = config_handler.system_entry
connect_delay = (1 + self.successive_failure_count / mktxp_entry.delay_inc_div) * mktxp_entry.initial_delay_on_failure connect_delay = (1 + self.successive_failure_count / mktxp_entry.delay_inc_div) * mktxp_entry.initial_delay_on_failure
return connect_delay if connect_delay < mktxp_entry.max_delay_on_failure else mktxp_entry.max_delay_on_failure return connect_delay if connect_delay < mktxp_entry.max_delay_on_failure else mktxp_entry.max_delay_on_failure

2
setup.py
View File

@@ -20,7 +20,7 @@ with open(path.join(pkg_dir, 'README.md'), encoding='utf-8') as f:
setup( setup(
name='mktxp', name='mktxp',
version='1.2.3', version='1.2.4',
url='https://github.com/akpw/mktxp', url='https://github.com/akpw/mktxp',