From cb2ff3c1a5606f1f4a13d237347534b60d352113 Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Sun, 7 Feb 2021 15:40:49 +0100 Subject: [PATCH] DS refactor, fixes/optimizations --- README.md | 2 +- mktxp/cli/config/_mktxp.conf | 4 +-- mktxp/cli/config/mktxp.conf | 2 +- mktxp/cli/dispatch.py | 10 ++----- mktxp/cli/output/capsman_out.py | 28 ++++++++++++------- mktxp/cli/output/dhcp_out.py | 26 +++++++++-------- mktxp/cli/output/wifi_out.py | 27 +++++++++++------- mktxp/collector/capsman_collector.py | 2 +- mktxp/collector/monitor_collector.py | 2 +- mktxp/collector/resource_collector.py | 2 +- mktxp/collector/wlan_collector.py | 2 +- mktxp/{ => flow}/__init__.py | 0 mktxp/{ => flow}/collectors_handler.py | 0 mktxp/{ => flow}/router_connection.py | 0 mktxp/{ => flow}/router_entries_handler.py | 2 +- mktxp/{ => flow}/router_entry.py | 2 +- mktxp/processor/__init__.py | 0 mktxp/{basep.py => processor/mktxp.py} | 6 ++-- .../base_out.py => processor/output.py} | 20 ++++++++++++- setup.py | 4 +-- 20 files changed, 87 insertions(+), 54 deletions(-) rename mktxp/{ => flow}/__init__.py (100%) rename mktxp/{ => flow}/collectors_handler.py (100%) rename mktxp/{ => flow}/router_connection.py (100%) rename mktxp/{ => flow}/router_entries_handler.py (96%) rename mktxp/{ => flow}/router_entry.py (96%) create mode 100644 mktxp/processor/__init__.py rename mktxp/{basep.py => processor/mktxp.py} (92%) rename mktxp/{cli/output/base_out.py => processor/output.py} (89%) diff --git a/README.md b/README.md index 6d713b2..0011695 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ ## Description MKTXP is a Prometheus Exporter for Mikrotik RouterOS devices.\ -It gathers a rich set of supported metrics across multiple routers, all easily configurable via built-in CLI interface. +It gathers and exports a rich set of metrics across multiple routers, all easily configurable via built-in CLI interface. Apart from exporting to Prometheus, MKTXP can also print some of the metrics directly on the command line (see an example below). diff --git a/mktxp/cli/config/_mktxp.conf b/mktxp/cli/config/_mktxp.conf index 567c776..1f987ae 100644 --- a/mktxp/cli/config/_mktxp.conf +++ b/mktxp/cli/config/_mktxp.conf @@ -12,10 +12,10 @@ [MKTXP] - port = 49090 # default metrics HTTP server port + port = 49090 # default metrics HTTP server port bandwidth_test_interval = 420 # Interval between periodic bandwidth tests - socket_timeout = 2 # Socket connection timeout + socket_timeout = 2 # Socket connection timeout initial_delay_on_failure = 120 # Delay untill next connection attempt to a RouterOS Device max_delay_on_failure = 900 # Max delay untill next connection attempt to a RouterOS Device delay_inc_div = 5 # Delay increment factor diff --git a/mktxp/cli/config/mktxp.conf b/mktxp/cli/config/mktxp.conf index 8d0267c..b0c70ab 100644 --- a/mktxp/cli/config/mktxp.conf +++ b/mktxp/cli/config/mktxp.conf @@ -36,4 +36,4 @@ capsman = True # CAPsMAN general metrics capsman_clients = True # CAPsMAN clients metrics - use_comments_over_names = False # when available, forces using comments over the interfaces names \ No newline at end of file + use_comments_over_names = True # when available, forces using comments over the interfaces names \ No newline at end of file diff --git a/mktxp/cli/dispatch.py b/mktxp/cli/dispatch.py index 42f8cad..f6e84cc 100755 --- a/mktxp/cli/dispatch.py +++ b/mktxp/cli/dispatch.py @@ -13,13 +13,10 @@ ## GNU General Public License for more details. -import sys import subprocess -import mktxp.cli.checks.chk_pv -from mktxp.utils.utils import run_cmd +from mktxp.cli.config.config import config_handler from mktxp.cli.options import MKTXPOptionsParser, MKTXPCommands -from mktxp.cli.config.config import config_handler, ConfigEntry -from mktxp.basep import MKTXPProcessor, MKTXPCLIProcessor +from mktxp.processor.mktxp import MKTXPProcessor, MKTXPCLIProcessor class MKTXPDispatcher: @@ -104,8 +101,7 @@ class MKTXPDispatcher: if args['dhcp_clients']: MKTXPCLIProcessor.dhcp_clients(args['entry_name']) - - + def main(): MKTXPDispatcher().dispatch() diff --git a/mktxp/cli/output/capsman_out.py b/mktxp/cli/output/capsman_out.py index 28fcfd5..c2f3c28 100644 --- a/mktxp/cli/output/capsman_out.py +++ b/mktxp/cli/output/capsman_out.py @@ -12,8 +12,7 @@ ## GNU General Public License for more details. -from tabulate import tabulate -from mktxp.cli.output.base_out import BaseOutputProcessor +from mktxp.processor.output import BaseOutputProcessor from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource from mktxp.datasource.capsman_ds import CapsmanRegistrationsMetricsDataSource @@ -43,14 +42,23 @@ class CapsmanOutput: else: dhcp_rt_by_interface[interface] = [registration_record] - num_records = 0 - output_table = [] + + output_entry = BaseOutputProcessor.OutputCapsmanEntry + output_table = BaseOutputProcessor.output_table(output_entry) + + output_records = 0 + registration_records = len(registration_records) for key in dhcp_rt_by_interface.keys(): for record in dhcp_rt_by_interface[key]: - output_table.append(BaseOutputProcessor.OutputCapsmanEntry(**record)) - num_records += 1 - output_table.append({}) - print() - print(tabulate(output_table, headers = "keys", tablefmt="github")) - print(tabulate([{0:'Connected Wifi Devices:', 1:num_records}], tablefmt="text")) + output_table.add_row(output_entry(**record)) + output_records += 1 + if output_records < registration_records: + output_table.add_row(output_entry()) + + print (output_table.draw()) + + for server in dhcp_rt_by_interface.keys(): + print(f'{server} clients: {len(dhcp_rt_by_interface[server])}') + print(f'Total connected CAPsMAN clients: {output_records}', '\n') + diff --git a/mktxp/cli/output/dhcp_out.py b/mktxp/cli/output/dhcp_out.py index 74d2eeb..c2e9888 100644 --- a/mktxp/cli/output/dhcp_out.py +++ b/mktxp/cli/output/dhcp_out.py @@ -12,8 +12,7 @@ ## GNU General Public License for more details. -from tabulate import tabulate -from mktxp.cli.output.base_out import BaseOutputProcessor +from mktxp.processor.output import BaseOutputProcessor from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource @@ -35,17 +34,22 @@ class DHCPOutput: dhcp_by_server[server].append(dhcp_lease_record) else: dhcp_by_server[server] = [dhcp_lease_record] + + output_entry = BaseOutputProcessor.OutputDHCPEntry + output_table = BaseOutputProcessor.output_table(output_entry) - num_records = 0 - output_table = [] + output_records = 0 + lease_records = len(dhcp_lease_records) for key in dhcp_by_server.keys(): for record in dhcp_by_server[key]: record['host_name'] = BaseOutputProcessor.dhcp_name(router_entry, record, drop_comment = True) - output_table.append(BaseOutputProcessor.OutputDHCPEntry(**record)) - num_records += 1 - output_table.append({}) - print() - print(tabulate(output_table, headers = "keys", tablefmt="github")) - print(tabulate([{0:'DHCP Clients:', 1:num_records}], tablefmt="text")) - + output_table.add_row(output_entry(**record)) + output_records += 1 + if output_records < lease_records: + output_table.add_row(output_entry()) + + print (output_table.draw()) + for server in dhcp_by_server.keys(): + print(f'{server} clients: {len(dhcp_by_server[server])}') + print(f'Total DHCP clients: {output_records}', '\n') diff --git a/mktxp/cli/output/wifi_out.py b/mktxp/cli/output/wifi_out.py index cd5d97f..ccae725 100644 --- a/mktxp/cli/output/wifi_out.py +++ b/mktxp/cli/output/wifi_out.py @@ -12,8 +12,7 @@ ## GNU General Public License for more details. -from tabulate import tabulate -from mktxp.cli.output.base_out import BaseOutputProcessor +from mktxp.processor.output import BaseOutputProcessor from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource from mktxp.datasource.wireless_ds import WirelessMetricsDataSource @@ -43,13 +42,21 @@ class WirelessOutput: else: dhcp_rt_by_interface[interface] = [registration_record] - num_records = 0 - output_table = [] + output_entry = BaseOutputProcessor.OutputWiFiEntry + output_table = BaseOutputProcessor.output_table(output_entry) + + output_records = 0 + registration_records = len(registration_records) for key in dhcp_rt_by_interface.keys(): for record in dhcp_rt_by_interface[key]: - output_table.append(BaseOutputProcessor.OutputWiFiEntry(**record)) - num_records += 1 - output_table.append({}) - print() - print(tabulate(output_table, headers = "keys", tablefmt="github")) - print(tabulate([{0:'Connected Wifi Devices:', 1:num_records}], tablefmt="text")) + output_table.add_row(output_entry(**record)) + output_records += 1 + if output_records < registration_records: + output_table.add_row(output_entry()) + + print (output_table.draw()) + + for server in dhcp_rt_by_interface.keys(): + print(f'{server} clients: {len(dhcp_rt_by_interface[server])}') + print(f'Total connected WiFi devices: {output_records}', '\n') + diff --git a/mktxp/collector/capsman_collector.py b/mktxp/collector/capsman_collector.py index d6fc1b1..48e0722 100644 --- a/mktxp/collector/capsman_collector.py +++ b/mktxp/collector/capsman_collector.py @@ -12,8 +12,8 @@ ## GNU General Public License for more details. -from mktxp.cli.output.base_out import BaseOutputProcessor from mktxp.cli.config.config import MKTXPConfigKeys +from mktxp.processor.output import BaseOutputProcessor from mktxp.collector.base_collector import BaseCollector from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource from mktxp.datasource.capsman_ds import CapsmanCapsMetricsDataSource, CapsmanRegistrationsMetricsDataSource diff --git a/mktxp/collector/monitor_collector.py b/mktxp/collector/monitor_collector.py index 76b6edb..f247d1a 100644 --- a/mktxp/collector/monitor_collector.py +++ b/mktxp/collector/monitor_collector.py @@ -13,7 +13,7 @@ from mktxp.collector.base_collector import BaseCollector -from mktxp.cli.output.base_out import BaseOutputProcessor +from mktxp.processor.output import BaseOutputProcessor from mktxp.datasource.interface_ds import InterfaceMonitorMetricsDataSource diff --git a/mktxp/collector/resource_collector.py b/mktxp/collector/resource_collector.py index 82be43c..831b0f7 100644 --- a/mktxp/collector/resource_collector.py +++ b/mktxp/collector/resource_collector.py @@ -13,7 +13,7 @@ from mktxp.collector.base_collector import BaseCollector -from mktxp.cli.output.base_out import BaseOutputProcessor +from mktxp.processor.output import BaseOutputProcessor from mktxp.datasource.system_resource_ds import SystemResourceMetricsDataSource diff --git a/mktxp/collector/wlan_collector.py b/mktxp/collector/wlan_collector.py index bdb4b6d..9f3e973 100644 --- a/mktxp/collector/wlan_collector.py +++ b/mktxp/collector/wlan_collector.py @@ -12,7 +12,7 @@ ## GNU General Public License for more details. -from mktxp.cli.output.base_out import BaseOutputProcessor +from mktxp.processor.output import BaseOutputProcessor from mktxp.collector.base_collector import BaseCollector from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource from mktxp.datasource.wireless_ds import WirelessMetricsDataSource diff --git a/mktxp/__init__.py b/mktxp/flow/__init__.py similarity index 100% rename from mktxp/__init__.py rename to mktxp/flow/__init__.py diff --git a/mktxp/collectors_handler.py b/mktxp/flow/collectors_handler.py similarity index 100% rename from mktxp/collectors_handler.py rename to mktxp/flow/collectors_handler.py diff --git a/mktxp/router_connection.py b/mktxp/flow/router_connection.py similarity index 100% rename from mktxp/router_connection.py rename to mktxp/flow/router_connection.py diff --git a/mktxp/router_entries_handler.py b/mktxp/flow/router_entries_handler.py similarity index 96% rename from mktxp/router_entries_handler.py rename to mktxp/flow/router_entries_handler.py index d9992bc..4816876 100644 --- a/mktxp/router_entries_handler.py +++ b/mktxp/flow/router_entries_handler.py @@ -13,7 +13,7 @@ from mktxp.cli.config.config import config_handler -from mktxp.router_entry import RouterEntry +from mktxp.flow.router_entry import RouterEntry class RouterEntriesHandler: diff --git a/mktxp/router_entry.py b/mktxp/flow/router_entry.py similarity index 96% rename from mktxp/router_entry.py rename to mktxp/flow/router_entry.py index 3bb222f..72d181d 100644 --- a/mktxp/router_entry.py +++ b/mktxp/flow/router_entry.py @@ -13,7 +13,7 @@ from mktxp.cli.config.config import config_handler, MKTXPConfigKeys -from mktxp.router_connection import RouterAPIConnection +from mktxp.flow.router_connection import RouterAPIConnection class RouterEntry: diff --git a/mktxp/processor/__init__.py b/mktxp/processor/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mktxp/basep.py b/mktxp/processor/mktxp.py similarity index 92% rename from mktxp/basep.py rename to mktxp/processor/mktxp.py index a72700b..2d18cd5 100644 --- a/mktxp/basep.py +++ b/mktxp/processor/mktxp.py @@ -15,10 +15,10 @@ from http.server import HTTPServer from datetime import datetime from prometheus_client.core import REGISTRY -from prometheus_client import MetricsHandler, start_http_server +from prometheus_client import MetricsHandler from mktxp.cli.config.config import config_handler -from mktxp.collectors_handler import CollectorsHandler -from mktxp.router_entries_handler import RouterEntriesHandler +from mktxp.flow.collectors_handler import CollectorsHandler +from mktxp.flow.router_entries_handler import RouterEntriesHandler from mktxp.cli.output.capsman_out import CapsmanOutput from mktxp.cli.output.wifi_out import WirelessOutput diff --git a/mktxp/cli/output/base_out.py b/mktxp/processor/output.py similarity index 89% rename from mktxp/cli/output/base_out.py rename to mktxp/processor/output.py index fceba6a..821f986 100644 --- a/mktxp/cli/output/base_out.py +++ b/mktxp/processor/output.py @@ -11,17 +11,23 @@ ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ## GNU General Public License for more details. -import re +import re, os from datetime import timedelta from collections import namedtuple +from texttable import Texttable from humanize import naturaldelta from mktxp.cli.config.config import config_handler class BaseOutputProcessor: OutputCapsmanEntry = namedtuple('OutputCapsmanEntry', ['dhcp_name', 'dhcp_address', 'mac_address', 'rx_signal', 'interface', 'ssid', 'tx_rate', 'rx_rate', 'uptime']) + OutputCapsmanEntry.__new__.__defaults__ = ('',) * len(OutputCapsmanEntry._fields) + OutputWiFiEntry = namedtuple('OutputWiFiEntry', ['dhcp_name', 'dhcp_address', 'mac_address', 'signal_strength', 'signal_to_noise', 'interface', 'tx_rate', 'rx_rate', 'uptime']) + OutputWiFiEntry.__new__.__defaults__ = ('',) * len(OutputWiFiEntry._fields) + OutputDHCPEntry = namedtuple('OutputDHCPEntry', ['host_name', 'server', 'mac_address', 'address', 'active_address', 'expires_after']) + OutputDHCPEntry.__new__.__defaults__ = ('',) * len(OutputDHCPEntry._fields) @staticmethod def augment_record(router_entry, registration_record, dhcp_lease_records): @@ -110,3 +116,15 @@ class BaseOutputProcessor: config_handler.re_compiled['interface_rate_rgx'] = interface_rate_rgx rate = lambda interface_rate: 1000 if interface_rate.find('Mbps') < 0 else 1 return(int(float(interface_rate_rgx.sub('', interface_rate)) * rate(interface_rate))) + + @staticmethod + def output_table(outputEntry = None): + table = Texttable(max_width = os.get_terminal_size().columns) + table.set_deco(Texttable.HEADER | Texttable.BORDER | Texttable.VLINES ) + if outputEntry: + table.header(outputEntry._fields) + table.set_cols_align(['l']+ ['c']*(len(outputEntry._fields)-1)) + return table + + + diff --git a/setup.py b/setup.py index ce0a50e..46bf09d 100755 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ with open(path.join(pkg_dir, 'README.md'), encoding='utf-8') as f: setup( name='mktxp', - version='0.24', + version='0.25', url='https://github.com/akpw/mktxp', @@ -47,7 +47,7 @@ setup( 'RouterOS-api>=0.17.0', 'configobj>=5.0.6', 'humanize>=3.2.0', - 'tabulate>=0.8.7', + 'texttable>=1.6.3', 'speedtest-cli>=2.1.2' ],