KevinMidboe/mktxp-no-cli
1
0
Fork 0
mirror of https://github.com/KevinMidboe/mktxp-no-cli.git synced 2025-10-29 17:50:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

Collector registry, fixes/optimizations

Browse Source
This commit is contained in:
Arseniy Kuznetsov
2021-02-14 09:03:08 +01:00
parent 6533a20d47
commit df84ada9c7
28 changed files with 237 additions and 234 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
mktxp/cli/config/_mktxp.conf
View File

@@ -12,10 +12,11 @@
[MKTXP]
port = 49090 # default metrics HTTP server port
bandwidth_test_interval = 420 # Interval between periodic bandwidth tests
socket_timeout = 2 # Socket connection timeout
initial_delay_on_failure = 120 # Delay untill next connection attempt to a RouterOS Device
max_delay_on_failure = 900 # Max delay untill next connection attempt to a RouterOS Device
delay_inc_div = 5 # Delay increment factor
port = 49090
socket_timeout = 2
initial_delay_on_failure = 120
max_delay_on_failure = 900
delay_inc_div = 5
bandwidth_test_interval = 420
collectors = IdentityCollector, SystemResourceCollector, HealthCollector, DHCPCollector, PoolCollector, InterfaceCollector, FirewallCollector, MonitorCollector, RouteCollector, WLANCollector, CapsmanCollector, MKTXPCollector

155
mktxp/cli/config/config.py
View File

@@ -54,6 +54,7 @@ class MKTXPConfigKeys:
MKTXP_MAX_DELAY = 'max_delay_on_failure'
MKTXP_INC_DIV = 'delay_inc_div'
MKTXP_BANDWIDTH_TEST_INTERVAL = 'bandwidth_test_interval'
MKTXP_COLLECTORS = 'collectors'
# UnRegistered entries placeholder
NO_ENTRIES_REGISTERED = 'NoEntriesRegistered'
@@ -74,6 +75,8 @@ class MKTXPConfigKeys:
DEFAULT_MKTXP_MAX_DELAY = 900
DEFAULT_MKTXP_INC_DIV = 5
DEFAULT_MKTXP_BANDWIDTH_TEST_INTERVAL = 420
DEFAULT_MKTXP_COLLECTORS = ['IdentityCollector', 'SystemResourceCollector', 'HealthCollector', 'DHCPCollector', 'PoolCollector',
'InterfaceCollector', 'FirewallCollector', 'MonitorCollector', 'RouteCollector', 'WLANCollector', 'CapsmanCollector', 'MKTXPCollector']
BOOLEAN_KEYS = (ENABLED_KEY, SSL_KEY, NO_SSL_CERTIFICATE, SSL_CERTIFICATE_VERIFY,
FE_DHCP_KEY, FE_DHCP_LEASE_KEY, FE_DHCP_POOL_KEY, FE_INTERFACE_KEY, FE_FIREWALL_KEY,
@@ -88,17 +91,17 @@ class MKTXPConfigKeys:
class ConfigEntry:
MKTXPEntry = namedtuple('MKTXPEntry', [MKTXPConfigKeys.ENABLED_KEY, MKTXPConfigKeys.HOST_KEY, MKTXPConfigKeys.PORT_KEY,
MKTXPConfigKeys.USER_KEY, MKTXPConfigKeys.PASSWD_KEY,
MKTXPConfigKeys.SSL_KEY, MKTXPConfigKeys.NO_SSL_CERTIFICATE, MKTXPConfigKeys.SSL_CERTIFICATE_VERIFY,
MKTXPConfigKeys.FE_DHCP_KEY, MKTXPConfigKeys.FE_DHCP_LEASE_KEY, MKTXPConfigKeys.FE_DHCP_POOL_KEY, MKTXPConfigKeys.FE_INTERFACE_KEY, MKTXPConfigKeys.FE_FIREWALL_KEY,
MKTXPConfigKeys.FE_MONITOR_KEY, MKTXPConfigKeys.FE_ROUTE_KEY, MKTXPConfigKeys.FE_WIRELESS_KEY, MKTXPConfigKeys.FE_WIRELESS_CLIENTS_KEY,
MKTXPConfigKeys.FE_CAPSMAN_KEY, MKTXPConfigKeys.FE_CAPSMAN_CLIENTS_KEY, MKTXPConfigKeys.MKTXP_USE_COMMENTS_OVER_NAMES
])
_MKTXPEntry = namedtuple('_MKTXPEntry', [MKTXPConfigKeys.PORT_KEY, MKTXPConfigKeys.MKTXP_SOCKET_TIMEOUT,
MKTXPConfigKeys.MKTXP_INITIAL_DELAY, MKTXPConfigKeys.MKTXP_MAX_DELAY,
MKTXPConfigKeys.MKTXP_INC_DIV, MKTXPConfigKeys.MKTXP_BANDWIDTH_TEST_INTERVAL])
MKTXPConfigEntry = namedtuple('MKTXPConfigEntry', [MKTXPConfigKeys.ENABLED_KEY, MKTXPConfigKeys.HOST_KEY, MKTXPConfigKeys.PORT_KEY,
MKTXPConfigKeys.USER_KEY, MKTXPConfigKeys.PASSWD_KEY,
MKTXPConfigKeys.SSL_KEY, MKTXPConfigKeys.NO_SSL_CERTIFICATE, MKTXPConfigKeys.SSL_CERTIFICATE_VERIFY,
MKTXPConfigKeys.FE_DHCP_KEY, MKTXPConfigKeys.FE_DHCP_LEASE_KEY, MKTXPConfigKeys.FE_DHCP_POOL_KEY, MKTXPConfigKeys.FE_INTERFACE_KEY, MKTXPConfigKeys.FE_FIREWALL_KEY,
MKTXPConfigKeys.FE_MONITOR_KEY, MKTXPConfigKeys.FE_ROUTE_KEY, MKTXPConfigKeys.FE_WIRELESS_KEY, MKTXPConfigKeys.FE_WIRELESS_CLIENTS_KEY,
MKTXPConfigKeys.FE_CAPSMAN_KEY, MKTXPConfigKeys.FE_CAPSMAN_CLIENTS_KEY, MKTXPConfigKeys.MKTXP_USE_COMMENTS_OVER_NAMES
])
MKTXPSystemEntry = namedtuple('MKTXPSystemEntry', [MKTXPConfigKeys.PORT_KEY, MKTXPConfigKeys.MKTXP_SOCKET_TIMEOUT,
MKTXPConfigKeys.MKTXP_INITIAL_DELAY, MKTXPConfigKeys.MKTXP_MAX_DELAY,
MKTXPConfigKeys.MKTXP_INC_DIV, MKTXPConfigKeys.MKTXP_BANDWIDTH_TEST_INTERVAL, MKTXPConfigKeys.MKTXP_COLLECTORS])
class OSConfig(metaclass = ABCMeta):
@@ -158,9 +161,32 @@ class MKTXPConfigHandler:
self._create_os_path(self.mktxp_conf_path, 'mktxp/cli/config/_mktxp.conf')
self.re_compiled = {}
self._read_from_disk()
# MKTXP entries
def registered_entries(self):
''' All MKTXP registered entries
'''
registered_entries = [entry_name for entry_name in self.config.keys()]
if not registered_entries:
registered_entries = [MKTXPConfigKeys.NO_ENTRIES_REGISTERED]
return registered_entries
def config_entry(self, entry_name):
''' Given an entry name, reads and returns the entry info
'''
entry_reader = self._config_entry_reader(entry_name)
return ConfigEntry.MKTXPConfigEntry(**entry_reader)
def system_entry(self):
''' MKTXP internal config entry
'''
_entry_reader = self._system_entry_reader()
return ConfigEntry.MKTXPSystemEntry(**_entry_reader)
# Helpers
def _read_from_disk(self):
''' (Force-)Read conf data from disk
'''
@@ -173,112 +199,67 @@ class MKTXPConfigHandler:
lookup_path = resource_filename(Requirement.parse("mktxp"), resource_path)
shutil.copy(lookup_path, os_path)
# MKTXP entries
##############
def register_entry(self, entry_name, entry_args, quiet = False):
''' Registers MKTXP conf entry
'''
if entry_name in self.registered_entries():
if not quiet:
print('"{0}": entry name already registered'.format(entry_name))
return False
else:
self.config[entry_name] = entry_args
self.config.write()
if not quiet:
print('Entry registered: {0}'.format(entry_name))
return True
def unregister_entry(self, entry_name, quiet = False):
''' Un-registers MKTXP conf entry
'''
if self.config[entry_name]:
del(self.config[entry_name])
self.config.write()
if not quiet:
print('Unregistered entry: {}'.format(entry_name))
return True
else:
if not quiet:
print('Entry is not registered: {}'.format(entry_name))
return False
def registered_entries(self):
''' All MKTXP registered entries
'''
registered_entries = [entry_name for entry_name in self.config.keys()]
if not registered_entries:
registered_entries = [MKTXPConfigKeys.NO_ENTRIES_REGISTERED]
return registered_entries
def entry(self, entry_name):
''' Given an entry name, reads and returns the entry info
'''
entry_reader = self.entry_reader(entry_name)
return ConfigEntry.MKTXPEntry(**entry_reader)
def _entry(self):
''' MKTXP internal config entry
'''
_entry_reader = self._entry_reader()
return ConfigEntry._MKTXPEntry(**_entry_reader)
# Helpers
def entry_reader(self, entry_name):
entry_reader = {}
def _config_entry_reader(self, entry_name):
config_entry_reader = {}
write_needed = False
for key in MKTXPConfigKeys.BOOLEAN_KEYS:
if self.config[entry_name].get(key):
entry_reader[key] = self.config[entry_name].as_bool(key)
config_entry_reader[key] = self.config[entry_name].as_bool(key)
else:
entry_reader[key] = False
config_entry_reader[key] = False
write_needed = True # read from disk next time
for key in MKTXPConfigKeys.STR_KEYS:
entry_reader[key] = self.config[entry_name][key]
config_entry_reader[key] = self.config[entry_name][key]
# port
if self.config[entry_name].get(MKTXPConfigKeys.PORT_KEY):
entry_reader[MKTXPConfigKeys.PORT_KEY] = self.config[entry_name].as_int(MKTXPConfigKeys.PORT_KEY)
config_entry_reader[MKTXPConfigKeys.PORT_KEY] = self.config[entry_name].as_int(MKTXPConfigKeys.PORT_KEY)
else:
entry_reader[MKTXPConfigKeys.PORT_KEY] = self._default_value_for_key(MKTXPConfigKeys.SSL_KEY, entry_reader[MKTXPConfigKeys.SSL_KEY])
config_entry_reader[MKTXPConfigKeys.PORT_KEY] = self._default_value_for_key(MKTXPConfigKeys.SSL_KEY, config_entry_reader[MKTXPConfigKeys.SSL_KEY])
write_needed = True # read from disk next time
if write_needed:
self.config[entry_name] = entry_reader
self.config[entry_name] = config_entry_reader
self.config.write()
return entry_reader
return config_entry_reader
def _entry_reader(self):
_entry_reader = {}
def _system_entry_reader(self):
system_entry_reader = {}
entry_name = MKTXPConfigKeys.MKTXP_CONFIG_ENTRY_NAME
write_needed = False
for key in MKTXPConfigKeys.MKTXP_INT_KEYS:
if self._config[entry_name].get(key):
_entry_reader[key] = self._config[entry_name].as_int(key)
system_entry_reader[key] = self._config[entry_name].as_int(key)
else:
_entry_reader[key] = self._default_value_for_key(key)
system_entry_reader[key] = self._default_value_for_key(key)
write_needed = True # read from disk next time
# Collectors
if self._config[entry_name].get(MKTXPConfigKeys.MKTXP_COLLECTORS):
system_entry_reader[MKTXPConfigKeys.MKTXP_COLLECTORS] = self._config[entry_name][MKTXPConfigKeys.MKTXP_COLLECTORS]
else:
system_entry_reader[MKTXPConfigKeys.MKTXP_COLLECTORS] = self._default_value_for_key(MKTXPConfigKeys.MKTXP_COLLECTORS)
write_needed = True # read from disk next time
if write_needed:
self._config[entry_name] = _entry_reader
self._config[entry_name] = system_entry_reader
self._config.write()
return _entry_reader
return system_entry_reader
def _default_value_for_key(self, key, value = None):
return {
MKTXPConfigKeys.SSL_KEY: lambda value: MKTXPConfigKeys.DEFAULT_API_SSL_PORT if value else MKTXPConfigKeys.DEFAULT_API_PORT,
MKTXPConfigKeys.PORT_KEY: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_PORT,
MKTXPConfigKeys.MKTXP_SOCKET_TIMEOUT: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_SOCKET_TIMEOUT,
MKTXPConfigKeys.MKTXP_INITIAL_DELAY: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_INITIAL_DELAY,
MKTXPConfigKeys.MKTXP_MAX_DELAY: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_MAX_DELAY,
MKTXPConfigKeys.MKTXP_INC_DIV: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_INC_DIV,
MKTXPConfigKeys.MKTXP_BANDWIDTH_TEST_INTERVAL: lambda value: MKTXPConfigKeys.DEFAULT_MKTXP_BANDWIDTH_TEST_INTERVAL
MKTXPConfigKeys.PORT_KEY: MKTXPConfigKeys.DEFAULT_MKTXP_PORT,
MKTXPConfigKeys.MKTXP_SOCKET_TIMEOUT: MKTXPConfigKeys.DEFAULT_MKTXP_SOCKET_TIMEOUT,
MKTXPConfigKeys.MKTXP_INITIAL_DELAY: MKTXPConfigKeys.DEFAULT_MKTXP_INITIAL_DELAY,
MKTXPConfigKeys.MKTXP_MAX_DELAY: MKTXPConfigKeys.DEFAULT_MKTXP_MAX_DELAY,
MKTXPConfigKeys.MKTXP_INC_DIV: MKTXPConfigKeys.DEFAULT_MKTXP_INC_DIV,
MKTXPConfigKeys.MKTXP_BANDWIDTH_TEST_INTERVAL: MKTXPConfigKeys.DEFAULT_MKTXP_BANDWIDTH_TEST_INTERVAL,
MKTXPConfigKeys.MKTXP_COLLECTORS: MKTXPConfigKeys.DEFAULT_MKTXP_COLLECTORS
}[key](value)

15
mktxp/cli/dispatch.py
View File

@@ -1,4 +1,4 @@
#!/usr/bin/env python
#!.usr/bin/env python
# coding=utf8
## Copyright (c) 2020 Arseniy Kuznetsov
##
@@ -16,7 +16,7 @@
import subprocess
from mktxp.cli.config.config import config_handler
from mktxp.cli.options import MKTXPOptionsParser, MKTXPCommands
from mktxp.processor.mktxp import MKTXPProcessor, MKTXPCLIProcessor
from mktxp.flow.processor.base_proc import ExportProcessor, OutputProcessor
class MKTXPDispatcher:
@@ -60,12 +60,11 @@ class MKTXPDispatcher:
if args['config']:
print(f'MKTXP data config: {config_handler.usr_conf_data_path}')
print(f'MKTXP internal config: {config_handler.mktxp_conf_path}')
else:
for entryname in config_handler.registered_entries():
if args['entry_name'] and entryname != args['entry_name']:
continue
entry = config_handler.entry(entryname)
entry = config_handler.config_entry(entryname)
print(f'[{entryname}]')
divider_fields = set(['username', 'use_ssl', 'dhcp'])
for field in entry._fields:
@@ -87,20 +86,20 @@ class MKTXPDispatcher:
subprocess.check_call([editor, config_handler.usr_conf_data_path])
def start_export(self, args):
MKTXPProcessor.start()
ExportProcessor.start()
def print(self, args):
if not (args['wifi_clients'] or args['capsman_clients']):
print("Select metric option(s) to print out, or run 'mktxp print -h' to find out more")
if args['wifi_clients']:
MKTXPCLIProcessor.wifi_clients(args['entry_name'])
OutputProcessor.wifi_clients(args['entry_name'])
if args['capsman_clients']:
MKTXPCLIProcessor.capsman_clients(args['entry_name'])
OutputProcessor.capsman_clients(args['entry_name'])
if args['dhcp_clients']:
MKTXPCLIProcessor.dhcp_clients(args['entry_name'])
OutputProcessor.dhcp_clients(args['entry_name'])
def main():

2
mktxp/cli/options.py
View File

@@ -150,7 +150,7 @@ Selected metrics info can be printed on the command line. For more information,
args['entry_name'] = UniquePartialMatchList(config_handler.registered_entries()).find(args['entry_name'])
if args['sub_cmd'] == MKTXPCommands.PRINT:
if not config_handler.entry(args['entry_name']).enabled:
if not config_handler.config_entry(args['entry_name']).enabled:
print(f"Can not print metrics for disabled RouterOS entry: {args['entry_name']}\nRun 'mktxp edit' to review and enable it in the configuration file first")
parser.exit()

2
mktxp/cli/output/capsman_out.py
View File

@@ -12,7 +12,7 @@
## GNU General Public License for more details.
from mktxp.processor.output import BaseOutputProcessor
from mktxp.flow.processor.output import BaseOutputProcessor
from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource
from mktxp.datasource.capsman_ds import CapsmanRegistrationsMetricsDataSource

2
mktxp/cli/output/dhcp_out.py
View File

@@ -12,7 +12,7 @@
## GNU General Public License for more details.
from mktxp.processor.output import BaseOutputProcessor
from mktxp.flow.processor.output import BaseOutputProcessor
from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource

2
mktxp/cli/output/wifi_out.py
View File

@@ -12,7 +12,7 @@
## GNU General Public License for more details.
from mktxp.processor.output import BaseOutputProcessor
from mktxp.flow.processor.output import BaseOutputProcessor
from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource
from mktxp.datasource.wireless_ds import WirelessMetricsDataSource

2
mktxp/collector/bandwidth_collector.py
View File

@@ -46,7 +46,7 @@ class BandwidthCollector(BaseCollector):
yield latency_metrics
ts = datetime.now().timestamp()
if (ts - self.last_call_timestamp) > config_handler._entry().bandwidth_test_interval:
if (ts - self.last_call_timestamp) > config_handler.system_entry().bandwidth_test_interval:
self.pool.apply_async(BandwidthCollector.bandwidth_worker, callback=get_result)
self.last_call_timestamp = ts

5
mktxp/collector/capsman_collector.py
View File

@@ -13,7 +13,7 @@
from mktxp.cli.config.config import MKTXPConfigKeys
from mktxp.processor.output import BaseOutputProcessor
from mktxp.flow.processor.output import BaseOutputProcessor
from mktxp.collector.base_collector import BaseCollector
from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource
from mktxp.datasource.capsman_ds import CapsmanCapsMetricsDataSource, CapsmanRegistrationsMetricsDataSource
@@ -24,6 +24,9 @@ class CapsmanCollector(BaseCollector):
'''
@staticmethod
def collect(router_entry):
if not router_entry.config_entry.capsman:
return
remote_caps_labels = ['identity', 'version', 'base_mac', 'board', 'base_mac']
remote_caps_records = CapsmanCapsMetricsDataSource.metric_records(router_entry, metric_labels = remote_caps_labels)
if remote_caps_records:

3
mktxp/collector/dhcp_collector.py
View File

@@ -22,6 +22,9 @@ class DHCPCollector(BaseCollector):
'''
@staticmethod
def collect(router_entry):
if not router_entry.config_entry.dhcp:
return
dhcp_lease_labels = ['active_address', 'address', 'mac_address', 'host_name', 'comment', 'server', 'expires_after']
dhcp_lease_records = DHCPMetricsDataSource.metric_records(router_entry, metric_labels = dhcp_lease_labels)
if dhcp_lease_records:

11
mktxp/collector/firewall_collector.py
View File

@@ -22,19 +22,22 @@ class FirewallCollector(BaseCollector):
'''
@staticmethod
def collect(router_entry):
if not router_entry.config_entry.firewall:
return
# initialize all pool counts, including those currently not used
firewall_labels = ['chain', 'action', 'bytes', 'comment']
firewall_labels = ['chain', 'action', 'bytes', 'comment', 'log']
firewall_filter_records = FirewallMetricsDataSource.metric_records(router_entry, metric_labels = firewall_labels)
if firewall_filter_records:
metris_records = [FirewallCollector.metric_record(router_entry, record) for record in firewall_filter_records]
firewall_filter_metrics = BaseCollector.counter_collector('firewall_filter', 'Total amount of bytes matched by firewall rules', metris_records, 'bytes', ['name'])
firewall_filter_metrics = BaseCollector.counter_collector('firewall_filter', 'Total amount of bytes matched by firewall rules', metris_records, 'bytes', ['name', 'log'])
yield firewall_filter_metrics
firewall_raw_records = FirewallMetricsDataSource.metric_records(router_entry, metric_labels = firewall_labels, raw = True)
if firewall_raw_records:
metris_records = [FirewallCollector.metric_record(router_entry, record) for record in firewall_raw_records]
firewall_raw_metrics = BaseCollector.counter_collector('firewall_raw', 'Total amount of bytes matched by raw firewall rules', metris_records, 'bytes', ['name'])
firewall_raw_metrics = BaseCollector.counter_collector('firewall_raw', 'Total amount of bytes matched by raw firewall rules', metris_records, 'bytes', ['name', 'log'])
yield firewall_raw_metrics
# Helpers
@@ -44,4 +47,4 @@ class FirewallCollector(BaseCollector):
bytes = firewall_record['bytes']
return {MKTXPConfigKeys.ROUTERBOARD_NAME: router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_entry.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
'name': name, 'bytes': bytes}
'name': name, 'log': firewall_record['log'], 'bytes': bytes}

3
mktxp/collector/interface_collector.py
View File

@@ -21,6 +21,9 @@ class InterfaceCollector(BaseCollector):
'''
@staticmethod
def collect(router_entry):
if not router_entry.config_entry.interface:
return
interface_traffic_labels = ['name', 'comment', 'rx_byte', 'tx_byte', 'rx_packet', 'tx_packet', 'rx_error', 'tx_error', 'rx_drop', 'tx_drop']
interface_traffic_records = InterfaceTrafficMetricsDataSource.metric_records(router_entry, metric_labels = interface_traffic_labels)

5
mktxp/collector/monitor_collector.py
View File

@@ -13,7 +13,7 @@
from mktxp.collector.base_collector import BaseCollector
from mktxp.processor.output import BaseOutputProcessor
from mktxp.flow.processor.output import BaseOutputProcessor
from mktxp.datasource.interface_ds import InterfaceMonitorMetricsDataSource
@@ -22,6 +22,9 @@ class MonitorCollector(BaseCollector):
'''
@staticmethod
def collect(router_entry):
if not router_entry.config_entry.monitor:
return
monitor_labels = ('status', 'rate', 'full_duplex', 'name')
monitor_records = InterfaceMonitorMetricsDataSource.metric_records(router_entry, metric_labels = monitor_labels, include_comments = True)
if monitor_records:

3
mktxp/collector/pool_collector.py
View File

@@ -22,6 +22,9 @@ class PoolCollector(BaseCollector):
'''
@staticmethod
def collect(router_entry):
if not router_entry.config_entry.pool:
return
# initialize all pool counts, including those currently not used
pool_records = PoolMetricsDataSource.metric_records(router_entry, metric_labels = ['name'])
if pool_records:

2
mktxp/collector/resource_collector.py
View File

@@ -13,7 +13,7 @@
from mktxp.collector.base_collector import BaseCollector
from mktxp.processor.output import BaseOutputProcessor
from mktxp.flow.processor.output import BaseOutputProcessor
from mktxp.datasource.system_resource_ds import SystemResourceMetricsDataSource

3
mktxp/collector/route_collector.py
View File

@@ -22,6 +22,9 @@ class RouteCollector(BaseCollector):
'''
@staticmethod
def collect(router_entry):
if not router_entry.config_entry.route:
return
route_labels = ['connect', 'dynamic', 'static', 'bgp', 'ospf']
route_records = RouteMetricsDataSource.metric_records(router_entry, metric_labels = route_labels)
if route_records:

5
mktxp/collector/wlan_collector.py
View File

@@ -12,7 +12,7 @@
## GNU General Public License for more details.
from mktxp.processor.output import BaseOutputProcessor
from mktxp.flow.processor.output import BaseOutputProcessor
from mktxp.collector.base_collector import BaseCollector
from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource
from mktxp.datasource.wireless_ds import WirelessMetricsDataSource
@@ -24,6 +24,9 @@ class WLANCollector(BaseCollector):
'''
@staticmethod
def collect(router_entry):
if not router_entry.config_entry.wireless:
return
monitor_labels = ['channel', 'noise_floor', 'overall_tx_ccq', 'registered_clients']
monitor_records = InterfaceMonitorMetricsDataSource.metric_records(router_entry, metric_labels = monitor_labels, kind = 'wireless')
if monitor_records:

4
mktxp/datasource/firewall_ds.py
View File

@@ -29,7 +29,9 @@ class FirewallMetricsDataSource:
# translation rules
translation_table = {}
if 'comment' in metric_labels:
translation_table['comment'] = lambda c: c if c else ''
translation_table['comment'] = lambda value: value if value else ''
if 'log' in metric_labels:
translation_table['log'] = lambda value: '1' if value == 'true' else '0'
return BaseDSProcessor.trimmed_records(router_entry, router_records = firewall_records, metric_labels = metric_labels, translation_table = translation_table)
except Exception as exc:

38
mktxp/flow/collector_handler.py Normal file
View File

@@ -0,0 +1,38 @@
# coding=utf8
## Copyright (c) 2020 Arseniy Kuznetsov
##
## This program is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License
## as published by the Free Software Foundation; either version 2
## of the License, or (at your option) any later version.
##
## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
## GNU General Public License for more details.
from timeit import default_timer
class CollectorHandler:
''' MKTXP Collectors Handler
'''
def __init__(self, entries_handler, collector_registry):
self.entries_handler = entries_handler
self.collector_registry = collector_registry
def collect(self):
yield from self.collector_registry.bandwidthCollector.collect()
for router_entry in self.entries_handler.router_entries:
if not router_entry.api_connection.is_connected():
# let's pick up on things in the next run
router_entry.api_connection.connect()
continue
for collector_ID, collect_func in self.collector_registry.registered_collectors.items():
start = default_timer()
yield from collect_func(router_entry)
router_entry.time_spent[collector_ID] += default_timer() - start

60
mktxp/flow/collector_registry.py Normal file
View File

@@ -0,0 +1,60 @@
# coding=utf8
## Copyright (c) 2020 Arseniy Kuznetsov
##
## This program is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License
## as published by the Free Software Foundation; either version 2
## of the License, or (at your option) any later version.
##
## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
## GNU General Public License for more details.
from collections import OrderedDict
from mktxp.cli.config.config import MKTXPConfigKeys
from mktxp.collector.dhcp_collector import DHCPCollector
from mktxp.collector.interface_collector import InterfaceCollector
from mktxp.collector.health_collector import HealthCollector
from mktxp.collector.identity_collector import IdentityCollector
from mktxp.collector.monitor_collector import MonitorCollector
from mktxp.collector.pool_collector import PoolCollector
from mktxp.collector.resource_collector import SystemResourceCollector
from mktxp.collector.route_collector import RouteCollector
from mktxp.collector.wlan_collector import WLANCollector
from mktxp.collector.capsman_collector import CapsmanCollector
from mktxp.collector.bandwidth_collector import BandwidthCollector
from mktxp.collector.firewall_collector import FirewallCollector
from mktxp.collector.mktxp_collector import MKTXPCollector
class CollectorRegistry:
''' MKTXP Collectors Registry
'''
def __init__(self):
self.registered_collectors = OrderedDict()
# bandwidth collector is not router-entry related, so registering directly
self.bandwidthCollector = BandwidthCollector()
self.register('IdentityCollector', IdentityCollector.collect)
self.register('SystemResourceCollector', SystemResourceCollector.collect)
self.register('HealthCollector', HealthCollector.collect)
self.register('DHCPCollector', DHCPCollector.collect)
self.register('PoolCollector', PoolCollector.collect)
self.register('InterfaceCollector', InterfaceCollector.collect)
self.register('FirewallCollector', FirewallCollector.collect)
self.register('MonitorCollector', MonitorCollector.collect)
self.register('RouteCollector', RouteCollector.collect)
self.register('WLANCollector', WLANCollector.collect)
self.register('CapsmanCollector', CapsmanCollector.collect)
self.register('MKTXPCollector', MKTXPCollector.collect)
def register(self, collector_ID, collect_func):
self.registered_collectors[collector_ID] = collect_func

104
mktxp/flow/collectors_handler.py
View File

@@ -1,104 +0,0 @@
# coding=utf8
## Copyright (c) 2020 Arseniy Kuznetsov
##
## This program is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License
## as published by the Free Software Foundation; either version 2
## of the License, or (at your option) any later version.
##
## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
## GNU General Public License for more details.
from timeit import default_timer
from mktxp.collector.dhcp_collector import DHCPCollector
from mktxp.collector.interface_collector import InterfaceCollector
from mktxp.collector.health_collector import HealthCollector
from mktxp.collector.identity_collector import IdentityCollector
from mktxp.collector.monitor_collector import MonitorCollector
from mktxp.collector.pool_collector import PoolCollector
from mktxp.collector.resource_collector import SystemResourceCollector
from mktxp.collector.route_collector import RouteCollector
from mktxp.collector.wlan_collector import WLANCollector
from mktxp.collector.capsman_collector import CapsmanCollector
from mktxp.collector.bandwidth_collector import BandwidthCollector
from mktxp.collector.firewall_collector import FirewallCollector
from mktxp.collector.mktxp_collector import MKTXPCollector
class CollectorsHandler:
''' MKTXP Collectors Handler
'''
def __init__(self, entries_handler):
self.entries_handler = entries_handler
self.bandwidthCollector = BandwidthCollector()
def collect(self):
# process mktxp internal metrics
yield from self.bandwidthCollector.collect()
for router_entry in self.entries_handler.router_entries:
if not router_entry.api_connection.is_connected():
# let's pick up on things in the next run
router_entry.api_connection.connect()
continue
start = default_timer()
yield from IdentityCollector.collect(router_entry)
router_entry.time_spent['IdentityCollector'] += default_timer() - start
start = default_timer()
yield from SystemResourceCollector.collect(router_entry)
router_entry.time_spent['SystemResourceCollector'] += default_timer() - start
start = default_timer()
yield from HealthCollector.collect(router_entry)
router_entry.time_spent['HealthCollector'] += default_timer() - start
if router_entry.config_entry.dhcp:
start = default_timer()
yield from DHCPCollector.collect(router_entry)
router_entry.time_spent['DHCPCollector'] += default_timer() - start
if router_entry.config_entry.pool:
start = default_timer()
yield from PoolCollector.collect(router_entry)
router_entry.time_spent['PoolCollector'] += default_timer() - start
if router_entry.config_entry.interface:
start = default_timer()
yield from InterfaceCollector.collect(router_entry)
router_entry.time_spent['InterfaceCollector'] += default_timer() - start
if router_entry.config_entry.firewall:
start = default_timer()
yield from FirewallCollector.collect(router_entry)
router_entry.time_spent['FirewallCollector'] += default_timer() - start
if router_entry.config_entry.monitor:
start = default_timer()
yield from MonitorCollector.collect(router_entry)
router_entry.time_spent['MonitorCollector'] += default_timer() - start
if router_entry.config_entry.route:
start = default_timer()
yield from RouteCollector.collect(router_entry)
router_entry.time_spent['RouteCollector'] += default_timer() - start
if router_entry.config_entry.wireless:
start = default_timer()
yield from WLANCollector.collect(router_entry)
router_entry.time_spent['WLANCollector'] += default_timer() - start
if router_entry.config_entry.capsman:
start = default_timer()
yield from CapsmanCollector.collect(router_entry)
router_entry.time_spent['CapsmanCollector'] += default_timer() - start
yield from MKTXPCollector.collect(router_entry)

0
mktxp/processor/__init__.py → mktxp/flow/processor/__init__.py
View File

15
mktxp/processor/mktxp.py → mktxp/flow/processor/base_proc.py
View File

@@ -16,8 +16,10 @@ from http.server import HTTPServer
from datetime import datetime
from prometheus_client.core import REGISTRY
from prometheus_client import MetricsHandler
from mktxp.cli.config.config import config_handler
from mktxp.flow.collectors_handler import CollectorsHandler
from mktxp.flow.collector_handler import CollectorHandler
from mktxp.flow.collector_registry import CollectorRegistry
from mktxp.flow.router_entries_handler import RouterEntriesHandler
from mktxp.cli.output.capsman_out import CapsmanOutput
@@ -25,17 +27,16 @@ from mktxp.cli.output.wifi_out import WirelessOutput
from mktxp.cli.output.dhcp_out import DHCPOutput
class MKTXPProcessor:
class ExportProcessor:
''' Base Export Processing
'''
@staticmethod
def start():
router_entries_handler = RouterEntriesHandler()
REGISTRY.register(CollectorsHandler(router_entries_handler))
MKTXPProcessor.run(port=config_handler._entry().port)
REGISTRY.register(CollectorHandler(RouterEntriesHandler(), CollectorRegistry()))
ExportProcessor.run(port=config_handler.system_entry().port)
@staticmethod
def run(server_class=HTTPServer, handler_class=MetricsHandler, port = None):
def run(server_class=HTTPServer, handler_class=MetricsHandler, port=None):
server_address = ('', port)
httpd = server_class(server_address, handler_class)
current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
@@ -43,7 +44,7 @@ class MKTXPProcessor:
httpd.serve_forever()
class MKTXPCLIProcessor:
class OutputProcessor:
''' Base CLI Processing
'''
@staticmethod

0
mktxp/processor/output.py → mktxp/flow/processor/output.py
View File

4
mktxp/flow/router_connection.py
View File

@@ -46,7 +46,7 @@ class RouterAPIConnection:
ssl_verify = self.config_entry.ssl_certificate_verify,
ssl_context = ctx)
self.connection.socket_timeout = config_handler._entry().socket_timeout
self.connection.socket_timeout = config_handler.system_entry().socket_timeout
self.api = None
def is_connected(self):
@@ -91,7 +91,7 @@ class RouterAPIConnection:
return False
def _connect_delay(self):
mktxp_entry = config_handler._entry()
mktxp_entry = config_handler.system_entry()
connect_delay = (1 + self.successive_failure_count / mktxp_entry.delay_inc_div) * mktxp_entry.initial_delay_on_failure
return connect_delay if connect_delay < mktxp_entry.max_delay_on_failure else mktxp_entry.max_delay_on_failure

4
mktxp/flow/router_entries_handler.py
View File

@@ -22,7 +22,7 @@ class RouterEntriesHandler:
def __init__(self):
self.router_entries = []
for router_name in config_handler.registered_entries():
entry = config_handler.entry(router_name)
entry = config_handler.config_entry(router_name)
if entry.enabled:
self.router_entries.append(RouterEntry(router_name))
@@ -32,7 +32,7 @@ class RouterEntriesHandler:
for router_name in config_handler.registered_entries():
if router_name == entry_name:
if enabled_only:
entry = config_handler.entry(router_name)
entry = config_handler.config_entry(router_name)
if not entry.enabled:
break
router_entry = RouterEntry(router_name)

5
mktxp/flow/router_entry.py
View File

@@ -21,7 +21,7 @@ class RouterEntry:
'''
def __init__(self, router_name):
self.router_name = router_name
self.config_entry = config_handler.entry(router_name)
self.config_entry = config_handler.config_entry(router_name)
self.api_connection = RouterAPIConnection(router_name, self.config_entry)
self.router_id = {
MKTXPConfigKeys.ROUTERBOARD_NAME: self.router_name,
@@ -37,5 +37,6 @@ class RouterEntry:
'MonitorCollector': 0,
'RouteCollector': 0,
'WLANCollector': 0,
'CapsmanCollector': 0
'CapsmanCollector': 0,
'MKTXPCollector': 0
}