DS refactor, fixes/optimizations

README.md

@@ -7,7 +7,7 @@
 
 ## Description
 MKTXP is a Prometheus Exporter for Mikrotik RouterOS devices.\
-It gathers a rich set of supported metrics across multiple routers, all easily configurable via built-in CLI interface. 
+It gathers and exports a rich set of metrics across multiple routers, all easily configurable via built-in CLI interface. 
 
 Apart from exporting to Prometheus, MKTXP can also print some of the metrics directly on the command line (see an example below).
 

mktxp/cli/config/mktxp.conf

@@ -36,4 +36,4 @@
     capsman = True                  # CAPsMAN general metrics
     capsman_clients = True          # CAPsMAN clients metrics
 
-    use_comments_over_names = False  # when available, forces using comments over the interfaces names 
+    use_comments_over_names = True  # when available, forces using comments over the interfaces names 

mktxp/cli/dispatch.py

@@ -13,13 +13,10 @@
 ## GNU General Public License for more details.
 
 
-import sys
 import subprocess
-import mktxp.cli.checks.chk_pv
+from mktxp.cli.config.config import config_handler
-from mktxp.utils.utils import run_cmd
 from mktxp.cli.options import MKTXPOptionsParser, MKTXPCommands
-from mktxp.cli.config.config import config_handler, ConfigEntry
+from mktxp.processor.mktxp import MKTXPProcessor, MKTXPCLIProcessor
-from mktxp.basep import MKTXPProcessor, MKTXPCLIProcessor
 
 
 class MKTXPDispatcher:
@@ -106,7 +103,6 @@ class MKTXPDispatcher:
             MKTXPCLIProcessor.dhcp_clients(args['entry_name'])
             
 
-
 def main():
     MKTXPDispatcher().dispatch()
 

mktxp/cli/output/capsman_out.py

@@ -12,8 +12,7 @@
 ## GNU General Public License for more details.
 
 
-from tabulate import tabulate   
+from mktxp.processor.output import BaseOutputProcessor
-from mktxp.cli.output.base_out import BaseOutputProcessor
 from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource
 from mktxp.datasource.capsman_ds import CapsmanRegistrationsMetricsDataSource
 
@@ -43,14 +42,23 @@ class CapsmanOutput:
             else:
                 dhcp_rt_by_interface[interface] = [registration_record]         
 
-        num_records = 0
+
-        output_table = []
+        output_entry = BaseOutputProcessor.OutputCapsmanEntry
+        output_table = BaseOutputProcessor.output_table(output_entry)
+
+        output_records = 0
+        registration_records = len(registration_records)                
         for key in dhcp_rt_by_interface.keys():
             for record in dhcp_rt_by_interface[key]:
-                output_table.append(BaseOutputProcessor.OutputCapsmanEntry(**record))
+                output_table.add_row(output_entry(**record))
-                num_records += 1
+                output_records += 1
-            output_table.append({})
+            if output_records < registration_records:
-        print()
+                output_table.add_row(output_entry())
-        print(tabulate(output_table, headers = "keys",  tablefmt="github"))
+                
-        print(tabulate([{0:'Connected Wifi Devices:', 1:num_records}], tablefmt="text"))
+        print (output_table.draw())
+
+        for server in dhcp_rt_by_interface.keys():
+            print(f'{server} clients: {len(dhcp_rt_by_interface[server])}')
+        print(f'Total connected CAPsMAN clients: {output_records}', '\n')
+
 

mktxp/cli/output/dhcp_out.py

@@ -12,8 +12,7 @@
 ## GNU General Public License for more details.
 
 
-from tabulate import tabulate   
+from mktxp.processor.output import BaseOutputProcessor
-from mktxp.cli.output.base_out import BaseOutputProcessor
 from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource
 
 
@@ -36,16 +35,21 @@ class DHCPOutput:
             else:
                 dhcp_by_server[server] = [dhcp_lease_record]         
         
-        num_records = 0
+        output_entry = BaseOutputProcessor.OutputDHCPEntry
-        output_table = []
+        output_table = BaseOutputProcessor.output_table(output_entry)
+
+        output_records = 0
+        lease_records = len(dhcp_lease_records)        
         for key in dhcp_by_server.keys():
             for record in dhcp_by_server[key]:
                 record['host_name'] = BaseOutputProcessor.dhcp_name(router_entry, record, drop_comment = True)
-                output_table.append(BaseOutputProcessor.OutputDHCPEntry(**record))
+                output_table.add_row(output_entry(**record))
-                num_records += 1
+                output_records += 1
-            output_table.append({})
+            if output_records < lease_records:
-        print()
+                output_table.add_row(output_entry())
-        print(tabulate(output_table, headers = "keys",  tablefmt="github"))
-        print(tabulate([{0:'DHCP Clients:', 1:num_records}], tablefmt="text"))
                 
+        print (output_table.draw())
 
+        for server in dhcp_by_server.keys():
+            print(f'{server} clients: {len(dhcp_by_server[server])}')
+        print(f'Total DHCP clients: {output_records}', '\n')

mktxp/cli/output/wifi_out.py

@@ -12,8 +12,7 @@
 ## GNU General Public License for more details.
 
 
-from tabulate import tabulate   
+from mktxp.processor.output import BaseOutputProcessor
-from mktxp.cli.output.base_out import BaseOutputProcessor
 from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource
 from mktxp.datasource.wireless_ds import WirelessMetricsDataSource
 
@@ -43,13 +42,21 @@ class WirelessOutput:
             else:
                 dhcp_rt_by_interface[interface] = [registration_record]         
 
-        num_records = 0
+        output_entry = BaseOutputProcessor.OutputWiFiEntry
-        output_table = []
+        output_table = BaseOutputProcessor.output_table(output_entry)
+
+        output_records = 0
+        registration_records = len(registration_records)                
         for key in dhcp_rt_by_interface.keys():
             for record in dhcp_rt_by_interface[key]:
-                output_table.append(BaseOutputProcessor.OutputWiFiEntry(**record))
+                output_table.add_row(output_entry(**record))
-                num_records += 1
+                output_records += 1
-            output_table.append({})
+            if output_records < registration_records:
-        print()
+                output_table.add_row(output_entry())
-        print(tabulate(output_table, headers = "keys",  tablefmt="github"))
+                
-        print(tabulate([{0:'Connected Wifi Devices:', 1:num_records}], tablefmt="text"))
+        print (output_table.draw())
+
+        for server in dhcp_rt_by_interface.keys():
+            print(f'{server} clients: {len(dhcp_rt_by_interface[server])}')
+        print(f'Total connected WiFi devices: {output_records}', '\n')
+

mktxp/collector/capsman_collector.py

@@ -12,8 +12,8 @@
 ## GNU General Public License for more details.
 
 
-from mktxp.cli.output.base_out import BaseOutputProcessor
 from mktxp.cli.config.config import MKTXPConfigKeys
+from mktxp.processor.output import BaseOutputProcessor
 from mktxp.collector.base_collector import BaseCollector
 from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource
 from mktxp.datasource.capsman_ds import CapsmanCapsMetricsDataSource, CapsmanRegistrationsMetricsDataSource

mktxp/collector/monitor_collector.py

@@ -13,7 +13,7 @@
 
 
 from mktxp.collector.base_collector import BaseCollector
-from mktxp.cli.output.base_out import BaseOutputProcessor
+from mktxp.processor.output import BaseOutputProcessor
 from mktxp.datasource.interface_ds import InterfaceMonitorMetricsDataSource
 
 

mktxp/collector/resource_collector.py

@@ -13,7 +13,7 @@
 
 
 from mktxp.collector.base_collector import BaseCollector
-from mktxp.cli.output.base_out import BaseOutputProcessor
+from mktxp.processor.output import BaseOutputProcessor
 from mktxp.datasource.system_resource_ds import SystemResourceMetricsDataSource
 
 

mktxp/collector/wlan_collector.py

@@ -12,7 +12,7 @@
 ## GNU General Public License for more details.
 
 
-from mktxp.cli.output.base_out import BaseOutputProcessor
+from mktxp.processor.output import BaseOutputProcessor
 from mktxp.collector.base_collector import BaseCollector
 from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource
 from mktxp.datasource.wireless_ds import WirelessMetricsDataSource

mktxp/flow/router_entries_handler.py

@@ -13,7 +13,7 @@
 
 
 from mktxp.cli.config.config import config_handler
-from mktxp.router_entry import RouterEntry
+from mktxp.flow.router_entry import RouterEntry
 
 
 class RouterEntriesHandler:

mktxp/flow/router_entry.py

@@ -13,7 +13,7 @@
 
 
 from mktxp.cli.config.config import config_handler, MKTXPConfigKeys
-from mktxp.router_connection import RouterAPIConnection
+from mktxp.flow.router_connection import RouterAPIConnection
 
 
 class RouterEntry:

mktxp/processor/mktxp.py

@@ -15,10 +15,10 @@
 from http.server import HTTPServer
 from datetime import datetime
 from prometheus_client.core import REGISTRY
-from prometheus_client import MetricsHandler, start_http_server
+from prometheus_client import MetricsHandler
 from mktxp.cli.config.config import config_handler
-from mktxp.collectors_handler import CollectorsHandler
+from mktxp.flow.collectors_handler import CollectorsHandler
-from mktxp.router_entries_handler import RouterEntriesHandler
+from mktxp.flow.router_entries_handler import RouterEntriesHandler
 
 from mktxp.cli.output.capsman_out import CapsmanOutput
 from mktxp.cli.output.wifi_out import WirelessOutput

mktxp/processor/output.py

@@ -11,17 +11,23 @@
 ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 ## GNU General Public License for more details.
 
-import re
+import re, os
 from datetime import timedelta
 from collections import namedtuple
+from texttable import Texttable
 from humanize import naturaldelta
 from mktxp.cli.config.config import config_handler
 
 
 class BaseOutputProcessor:
     OutputCapsmanEntry = namedtuple('OutputCapsmanEntry', ['dhcp_name', 'dhcp_address', 'mac_address', 'rx_signal', 'interface', 'ssid', 'tx_rate', 'rx_rate', 'uptime'])
+    OutputCapsmanEntry.__new__.__defaults__ = ('',) * len(OutputCapsmanEntry._fields)
+
     OutputWiFiEntry = namedtuple('OutputWiFiEntry', ['dhcp_name', 'dhcp_address', 'mac_address', 'signal_strength', 'signal_to_noise', 'interface', 'tx_rate', 'rx_rate', 'uptime'])
+    OutputWiFiEntry.__new__.__defaults__ = ('',) * len(OutputWiFiEntry._fields)
+
     OutputDHCPEntry = namedtuple('OutputDHCPEntry', ['host_name', 'server', 'mac_address', 'address', 'active_address', 'expires_after'])
+    OutputDHCPEntry.__new__.__defaults__ = ('',) * len(OutputDHCPEntry._fields)
 
     @staticmethod
     def augment_record(router_entry, registration_record, dhcp_lease_records):
@@ -110,3 +116,15 @@ class BaseOutputProcessor:
             config_handler.re_compiled['interface_rate_rgx'] = interface_rate_rgx
         rate = lambda interface_rate: 1000 if interface_rate.find('Mbps') < 0 else 1
         return(int(float(interface_rate_rgx.sub('', interface_rate)) * rate(interface_rate)))
+
+    @staticmethod
+    def output_table(outputEntry = None):
+        table = Texttable(max_width = os.get_terminal_size().columns)
+        table.set_deco(Texttable.HEADER | Texttable.BORDER | Texttable.VLINES )        
+        if outputEntry:
+            table.header(outputEntry._fields)
+            table.set_cols_align(['l']+ ['c']*(len(outputEntry._fields)-1))
+        return table
+
+
+

setup.py

@@ -20,7 +20,7 @@ with open(path.join(pkg_dir, 'README.md'), encoding='utf-8') as f:
 
 setup(
     name='mktxp',
-    version='0.24',
+    version='0.25',
 
     url='https://github.com/akpw/mktxp',
 
@@ -47,7 +47,7 @@ setup(
                         'RouterOS-api>=0.17.0', 
                         'configobj>=5.0.6',
                         'humanize>=3.2.0',
-                        'tabulate>=0.8.7',
+                        'texttable>=1.6.3',
                         'speedtest-cli>=2.1.2'
                         ],