Initial commit

mktxp/basep.py

@@ -0,0 +1,35 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from http.server import HTTPServer
+from prometheus_client.core import REGISTRY
+from prometheus_client import MetricsHandler, start_http_server
+from mktxp.collectors_handler import CollectorsHandler
+from mktxp.metrics_handler import RouterMetricsHandler
+
+
+class MKTXPProcessor:
+    ''' Base Export Processing
+    '''    
+    @staticmethod
+    def start():
+        router_metrics_handler = RouterMetricsHandler()
+        REGISTRY.register(CollectorsHandler(router_metrics_handler))
+        MKTXPProcessor.run()
+
+    @staticmethod
+    def run(server_class=HTTPServer, handler_class=MetricsHandler, port=8000):
+        server_address = ('', port)
+        httpd = server_class(server_address, handler_class)
+        httpd.serve_forever()

mktxp/cli/checks/chk_pv.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+''' Python version check
+'''
+
+from __future__ import print_function
+
+import sys
+
+def check_version():
+    if sys.version_info.major < 3:
+        print(\
+        '''
+        Mikrotik Prometheus Exporter requires
+                           Python version 3.6 or later.
+
+        You can create an isolated Python 3.6 environment
+        with the virtualenv tool:
+          http://docs.python-guide.org/en/latest/dev/virtualenvs
+
+        ''')
+        sys.exit(0)
+    elif sys.version_info.major == 3 and sys.version_info.minor < 6:
+        print(\
+        '''
+
+        Mikrotik Prometheus Exporter requires
+                            Python version 3.6 or later.
+
+        Please upgrade to the latest Python 3.x version.
+
+        ''')
+        sys.exit(0)
+
+# check
+check_version()

mktxp/cli/config/config.py

@@ -0,0 +1,209 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+import os, sys, shutil
+from collections import namedtuple
+from configobj import ConfigObj
+from abc import ABCMeta, abstractmethod
+from pkg_resources import Requirement, resource_filename
+from mktxp.utils.utils import FSHelper
+
+
+''' MKTXP conf file handling
+'''
+
+class MKTXPConfigKeys:
+    ''' MKTXP config file keys
+    '''
+    # Section Keys
+    ENABLED_KEY = 'enabled'
+    HOST_KEY = 'hostname'
+    PORT_KEY = 'port'
+    USER_KEY = 'username'
+    PASSWD_KEY = 'password'
+
+    SSL_KEY = 'use_ssl'
+    SSL_CERTIFICATE = 'ssl_certificate'
+
+    FE_DHCP_KEY = 'dhcp'
+    FE_DHCP_LEASE_KEY = 'dhcp_lease'    
+    FE_DHCP_POOL_KEY = 'pool'    
+    FE_INTERFACE_KEY = 'interface'
+    FE_MONITOR_KEY = 'monitor'
+    FE_ROUTE_KEY = 'route'
+    FE_WIRELESS_KEY = 'wireless'
+    FE_CAPSMAN_KEY = 'capsman'
+
+
+    # UnRegistered entries placeholder
+    NO_ENTRIES_REGISTERED = 'NoEntriesRegistered'
+
+    # Base router id labels
+    ROUTERBOARD_NAME = 'routerboard_name'
+    ROUTERBOARD_ADDRESS = 'routerboard_address'
+
+    # Default ports
+    DEFAULT_API_PORT = 8728
+    DEFAULT_API_SSL_PORT = 8729
+
+    BOOLEAN_KEYS = [ENABLED_KEY, SSL_KEY, SSL_CERTIFICATE,  
+                      FE_DHCP_KEY, FE_DHCP_LEASE_KEY, FE_DHCP_POOL_KEY, FE_INTERFACE_KEY, 
+                      FE_MONITOR_KEY, FE_ROUTE_KEY, FE_WIRELESS_KEY, FE_CAPSMAN_KEY]
+    STR_KEYS = [HOST_KEY, USER_KEY, PASSWD_KEY]
+
+
+class ConfigEntry:
+    MKTXPEntry = namedtuple('MKTXPEntry', [MKTXPConfigKeys.ENABLED_KEY, MKTXPConfigKeys.HOST_KEY, MKTXPConfigKeys.PORT_KEY, 
+                         MKTXPConfigKeys.USER_KEY, MKTXPConfigKeys.PASSWD_KEY, 
+                         MKTXPConfigKeys.SSL_KEY, MKTXPConfigKeys.SSL_CERTIFICATE, 
+                         
+                         MKTXPConfigKeys.FE_DHCP_KEY, MKTXPConfigKeys.FE_DHCP_LEASE_KEY, MKTXPConfigKeys.FE_DHCP_POOL_KEY, MKTXPConfigKeys.FE_INTERFACE_KEY, 
+                         MKTXPConfigKeys.FE_MONITOR_KEY, MKTXPConfigKeys.FE_ROUTE_KEY, MKTXPConfigKeys.FE_WIRELESS_KEY, MKTXPConfigKeys.FE_CAPSMAN_KEY
+                         ])
+
+class OSConfig(metaclass = ABCMeta):
+    ''' OS-related config
+    '''
+    @staticmethod
+    def os_config(quiet = False):
+        ''' Factory method
+        '''
+        if sys.platform == 'linux':
+            return LinuxConfig()
+        elif sys.platform == 'darwin':
+            return OSXConfig()
+        else:
+            if not quiet:
+                print('Non-supported platform: {}'.format(sys.platform))
+            return None
+
+    @property
+    @abstractmethod
+    def mktxp_user_dir_path(self):
+        pass
+
+
+class OSXConfig(OSConfig):
+    ''' OSX-related config
+    '''
+    @property
+    def mktxp_user_dir_path(self):
+        return FSHelper.full_path('~/mktxp')
+
+
+class LinuxConfig(OSConfig):
+    ''' Linux-related config
+    '''
+    @property
+    def mktxp_user_dir_path(self):
+        return FSHelper.full_path('/etc/mktxp')
+
+
+class MKTXPConfigHandler:
+    def __init__(self):
+        self.os_config = OSConfig.os_config()
+        if not self.os_config:
+            sys.exit(1)
+
+        # mktxp user config folder
+        if not os.path.exists(self.os_config.mktxp_user_dir_path):
+            os.makedirs(self.os_config.mktxp_user_dir_path)
+
+        # if needed, stage the user config data
+        self.usr_conf_data_path = os.path.join(self.os_config.mktxp_user_dir_path, 'mktxp.conf')
+        if not os.path.exists(self.usr_conf_data_path):
+            # stage from the mktxp conf template
+            lookup_path = resource_filename(Requirement.parse("mktxp"), "mktxp/cli/config/mktxp.conf")
+            shutil.copy(lookup_path, self.usr_conf_data_path)
+
+        self.read_from_disk()
+
+    def read_from_disk(self):
+        ''' (Force-)Read conf data from disk
+        '''
+        self.config = ConfigObj(self.usr_conf_data_path)
+
+
+    # MKTXP entries
+    ##############
+    def register_entry(self, entry_name, entry_info, quiet = False):
+        ''' Registers MKTXP conf entry
+        '''
+        if entry_name in self.registered_entries():
+            if not quiet:
+                print('"{0}": entry name already registered'.format(entry_name))
+            return False
+        else:
+            self.config[entry_name] = dict(entry_info._asdict())
+            print(f'adding entry: {self.config[entry_name]}')
+            self.config.write()
+            if not quiet:
+                print('Entry registered: {0}'.format(entry_name))
+            return True
+
+    def unregister_entry(self, entry_name, quiet = False):
+        ''' Un-registers MKTXP conf entry
+        '''
+        if self.config[entry_name]:
+            del(self.config[entry_name])
+            self.config.write()
+            if not quiet:
+                print('Unregistered entry: {}'.format(entry_name))
+            return True
+        else:
+            if not quiet:
+                print('Entry is not registered: {}'.format(entry_name))
+            return False
+
+    def registered_entries(self):
+        ''' All MKTXP registered entries
+        '''
+        registered_entries = [entry_name for entry_name in self.config.keys()]
+        if not registered_entries:
+            registered_entries = [MKTXPConfigKeys.NO_ENTRIES_REGISTERED]
+
+        return registered_entries
+
+    def entry(self, entry_name):
+        ''' Given an entry name, reads and returns the entry info
+        '''
+        entry_reader = self._entry_reader(entry_name)
+        return ConfigEntry.MKTXPEntry(**entry_reader)
+
+    # Helpers
+    def _entry_reader(self, entry_name):
+        entry = {}
+        for key in MKTXPConfigKeys.BOOLEAN_KEYS:
+            if self.config[entry_name].get(key):
+                entry[key] = self.config[entry_name].as_bool(key)
+            else:
+                entry[key] = False
+
+        for key in MKTXPConfigKeys.STR_KEYS:
+            entry[key] = self.config[entry_name][key]
+
+        # port
+        if self.config[entry_name].get(MKTXPConfigKeys.PORT_KEY):
+            entry[MKTXPConfigKeys.PORT_KEY] = self.config[entry_name].as_int(MKTXPConfigKeys.PORT_KEY)
+        else:
+            if entry[MKTXPConfigKeys.SSL_KEY]:
+                entry[MKTXPConfigKeys.PORT_KEY] = MKTXPConfigKeys.DEFAULT_API_SSL_PORT
+            else:
+                entry[MKTXPConfigKeys.PORT_KEY] = MKTXPConfigKeys.DEFAULT_API_PORT
+
+        return entry
+
+
+# Simplest possible Singleton impl
+config_handler = MKTXPConfigHandler()
+

mktxp/cli/config/mktxp.conf

@@ -0,0 +1,36 @@
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+[Sample_RouterBoard]
+    enabled = False
+
+    hostname = localhost
+    port = 8728
+
+    username = user
+    password = password
+
+    use_ssl = False
+    ssl_certificate = False
+
+    identity = True
+    resource = True
+    health = True
+    dhcp = True
+    dhcp_lease = True
+    pool = True
+    interface = True
+    monitor = True
+    route = True
+    wireless = True
+    caps_man = True

mktxp/cli/dispatch.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+import sys
+import pkg_resources
+import mktxp.cli.checks.chk_pv
+from mktxp.cli.options import MKTXPOptionsParser, MKTXPCommands
+from mktxp.cli.config.config import config_handler, ConfigEntry
+from mktxp.basep import MKTXPProcessor
+
+class MKTXPDispatcher:
+    ''' Base MKTXP Commands Dispatcher
+    '''
+    def __init__(self):
+        self.option_parser = MKTXPOptionsParser()
+
+    # Dispatcher
+    def dispatch(self):
+        args = self.option_parser.parse_options()
+
+        if args['sub_cmd'] == MKTXPCommands.VERSION:
+            self.print_version()
+
+        elif args['sub_cmd'] == MKTXPCommands.INFO:
+            self.print_info()
+
+        elif args['sub_cmd'] == MKTXPCommands.SHOW:
+            self.show_entries()
+
+        elif args['sub_cmd'] == MKTXPCommands.ADD:
+            self.add_entry(args)
+
+        elif args['sub_cmd'] == MKTXPCommands.EDIT:
+            self.edit_entry(args)
+
+        elif args['sub_cmd'] == MKTXPCommands.DELETE:
+            self.delete_entry(args)
+
+        elif args['sub_cmd'] == MKTXPCommands.START:
+            self.start_export(args)
+
+        else:
+            # nothing to dispatch
+            return False
+
+        return True
+
+    # Dispatched methods
+    def print_version(self):
+        ''' Prints MKTXP version info
+        '''
+        version = pkg_resources.require("mktxp")[0].version
+        print('Mikrotik RouterOS Prometheus Exporter version {}'.format(version))
+
+    def print_info(self):
+        ''' Prints MKTXP general info
+        '''
+        print('Mikrotik RouterOS Prometheus Exporter: {}'.format(self.option_parser.script_name))
+        print(self.option_parser.description)
+
+
+    def show_entries(self):
+        for entryname in config_handler.registered_entries():
+            entry = config_handler.entry(entryname)
+
+            print('[{}]'.format(entryname))
+            for field in entry._fields:
+                print('    {}: {}'.format(field, getattr(entry, field)))
+            print()
+
+    def add_entry(self, args):
+        args.pop('sub_cmd', None)
+        entry_name = args['entry_name']
+        args.pop('entry_name', None)
+
+        entry_info = ConfigEntry.MKTXPEntry(**args)
+        config_handler.register_entry(entry_name = entry_name, entry_info = entry_info)
+
+
+    def edit_entry(self, args):
+        pass
+
+    def delete_entry(self, args):
+        config_handler.unregister_entry(entry_name = args['entry_name'])
+        
+
+    def start_export(self, args):
+        MKTXPProcessor.start()
+
+
+def main():
+    MKTXPDispatcher().dispatch()
+
+if __name__ == '__main__':
+    main()
+

mktxp/cli/options.py

@@ -0,0 +1,287 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+import os
+from argparse import ArgumentParser, HelpFormatter
+from mktxp.cli.config.config import config_handler, MKTXPConfigKeys
+from mktxp.utils.utils import FSHelper, UniquePartialMatchList
+
+
+class MKTXPCommands:
+    INFO = 'info'
+    VERSION = 'version'
+    SHOW = 'show'
+    ADD = 'add'
+    EDIT = 'edit'
+    DELETE = 'delete' 
+    START = 'start'
+
+    @classmethod
+    def commands_meta(cls):
+        return ''.join(('{',
+                        '{}, '.format(cls.INFO),
+                        '{}'.format(cls.VERSION),
+                        '{}'.format(cls.SHOW),
+                        '{}'.format(cls.ADD),
+                        '{}'.format(cls.EDIT),
+                        '{}'.format(cls.DELETE),
+                        '{}'.format(cls.START),                        
+                        '}'))
+
+class MKTXPOptionsParser:
+    ''' Base MKTXP Options Parser
+    '''
+    def __init__(self):
+        self._script_name = 'MKTXP'
+        self._description = \
+    '''
+    Prometheus Exporter for Mikrotik RouterOS Devices
+    '''
+
+    @property
+    def description(self):
+        return self._description
+
+    @property
+    def script_name(self):
+        return self._script_name
+
+    # Options Parsing Workflow
+    def parse_options(self):
+        ''' General Options parsing workflow
+        '''
+        parser = ArgumentParser(prog = self._script_name,
+                                description = self._description,
+                                formatter_class=MKTXPHelpFormatter)
+
+        self.parse_global_options(parser)
+        self.parse_commands(parser)
+        args = vars(parser.parse_args())
+
+        self._check_args(args, parser)
+
+        return args
+
+    def parse_global_options(self, parser):
+        ''' Parses global options
+        '''
+        pass
+
+    def parse_commands(self, parser):
+        ''' Commands parsing
+        '''
+        subparsers = parser.add_subparsers(dest = 'sub_cmd',
+                                           title = 'MKTXP commands',
+                                           metavar = MKTXPCommands.commands_meta())
+        
+        # Info command
+        subparsers.add_parser(MKTXPCommands.INFO,
+                                        description = 'Displays MKTXP info',
+                                        formatter_class=MKTXPHelpFormatter)
+        # Version command
+        subparsers.add_parser(MKTXPCommands.VERSION,
+                                        description = 'Displays MKTXP version info',
+                                        formatter_class=MKTXPHelpFormatter)
+
+        # Show command
+        subparsers.add_parser(MKTXPCommands.SHOW,
+                                        description = 'Displays MKTXP router entries',
+                                        formatter_class=MKTXPHelpFormatter)
+
+        # Add command
+        add_parser = subparsers.add_parser(MKTXPCommands.ADD,
+                                        description = 'Adds a new MKTXP router entry',
+                                        formatter_class=MKTXPHelpFormatter)
+        required_args_group = add_parser.add_argument_group('Required Arguments')
+        self._add_entry_name(required_args_group, registered_only = False, help = "Config entry name")
+        required_args_group.add_argument('-host', '--hostname', dest='hostname',
+                help = "IP address of RouterOS device to export metrics from",
+                type = str,
+                required=True)
+        required_args_group.add_argument('-usr', '--username', dest='username',
+                help = "username",
+                type = str,
+                required=True)
+        required_args_group.add_argument('-pwd', '--password', dest='password',
+                help = "password",
+                type = str,
+                required=True)
+
+        optional_args_group = add_parser.add_argument_group('Optional Arguments')
+        optional_args_group.add_argument('-e', dest='enabled',
+                help = "Enables entry for metrics processing",
+                action = 'store_false')
+
+        optional_args_group.add_argument('-port', dest='port',
+                help = "port",
+                default = MKTXPConfigKeys.DEFAULT_API_PORT,
+                type = int)
+
+        optional_args_group.add_argument('-ssl', '--use-ssl', dest='use_ssl',
+                help = "Connect via RouterOS api-ssl service",
+                action = 'store_true')
+        optional_args_group.add_argument('-ssl-cert', '--use-ssl-certificate', dest='ssl_certificate',
+                help = "Connect with configured RouterOS SSL ceritficate",
+                action = 'store_true')
+
+        optional_args_group.add_argument('-dhcp', '--export_dhcp', dest='dhcp',
+                help = "Export DHCP metrics",
+                action = 'store_true')
+        optional_args_group.add_argument('-dhcp_lease', '--export_dhcp_lease', dest='dhcp_lease',
+                help = "Export DHCP Lease metrics",
+                action = 'store_true')
+        optional_args_group.add_argument('-pool', '--export_pool', dest='pool',
+                help = "Export IP Pool metrics",
+                action = 'store_true')
+        optional_args_group.add_argument('-interface', '--export_interface', dest='interface',
+                help = "Export Interface metrics",
+                action = 'store_true')
+        optional_args_group.add_argument('-monitor', '--export_monitor', dest='monitor',
+                help = "Export Interface Monitor metrics",
+                action = 'store_true')
+        optional_args_group.add_argument('-route', '--export_route', dest='route',
+                help = "Export IP Route metrics",
+                action = 'store_true')
+        optional_args_group.add_argument('-wireless', '--export_wireless', dest='wireless',
+                help = "Export Wireless metrics",
+                action = 'store_true')
+        optional_args_group.add_argument('-capsman', '--export_capsman', dest='capsman',
+                help = "Export CAPsMAN metrics",
+                action = 'store_true')
+
+#'hostname', 'port', 'username', 'password', 'use_ssl', 'ssl_certificate', 'dhcp', 'dhcp_lease', 'pool', 'interface', 'monitor', 'route', 'wireless', and 'capsman'
+
+        # Edit command
+        edit_parser = subparsers.add_parser(MKTXPCommands.EDIT,
+                                        description = 'Edits an existing MKTXP router entry',
+                                        formatter_class=MKTXPHelpFormatter)
+        required_args_group = edit_parser.add_argument_group('Required Arguments')
+        self._add_entry_name(required_args_group, registered_only = True, help = "Name of entry to edit")  
+
+
+        # Delete command
+        delete_parser = subparsers.add_parser(MKTXPCommands.DELETE,
+                                        description = 'Deletes an existing MKTXP router entry',
+                                        formatter_class=MKTXPHelpFormatter)
+        required_args_group = delete_parser.add_argument_group('Required Arguments')
+        self._add_entry_name(required_args_group, registered_only = True, help = "Name of entry to delete")
+
+        # Start command
+        start_parser = subparsers.add_parser(MKTXPCommands.START,
+                                        description = 'Starts exporting Miktorik Router Metrics',
+                                        formatter_class=MKTXPHelpFormatter)
+
+
+    # Options checking
+    def _check_args(self, args, parser):
+        ''' Validation of supplied CLI arguments
+        '''
+        # check if there is a cmd to execute
+        self._check_cmd_args(args, parser)
+
+        if args['sub_cmd'] in (MKTXPCommands.EDIT, MKTXPCommands.DELETE):
+            # Registered Entry name could be a partial match, need to expand
+            args['entry_name'] = UniquePartialMatchList(config_handler.registered_entries()).find(args['entry_name'])
+
+        if args['sub_cmd'] == MKTXPCommands.ADD:
+            if args['entry_name'] in (config_handler.registered_entries()):
+                print('"{0}": entry name already exists'.format(args['entry_name']))
+                parser.exit()
+
+    def _check_cmd_args(self, args, parser):
+        ''' Validation of supplied CLI commands
+        '''
+        # base command check
+        if 'sub_cmd' not in args or not args['sub_cmd']:
+            # If no command was specified, check for the default one
+            cmd = self._default_command
+            if cmd:
+                args['sub_cmd'] = cmd
+            else:
+                # no appropriate default either
+                parser.print_help()
+                parser.exit()
+
+
+    @property
+    def _default_command(self):
+        ''' If no command was specified, print INFO by default
+        '''
+        return MKTXPCommands.START
+
+
+    # Internal helpers
+    @staticmethod
+    def _is_valid_dir_path(parser, path_arg):
+        ''' Checks if path_arg is a valid dir path
+        '''
+        path_arg = FSHelper.full_path(path_arg)
+        if not (os.path.exists(path_arg) and os.path.isdir(path_arg)):
+            parser.error('"{}" does not seem to be an existing directory path'.format(path_arg))
+        else:
+            return path_arg
+
+    @staticmethod
+    def _is_valid_file_path(parser, path_arg):
+        ''' Checks if path_arg is a valid file path
+        '''
+        path_arg = FSHelper.full_path(path_arg)
+        if not (os.path.exists(path_arg) and os.path.isfile(path_arg)):
+            parser.error('"{}" does not seem to be an existing file path'.format(path_arg))
+        else:
+            return path_arg
+
+    @staticmethod
+    def _add_entry_name(parser, registered_only = False, help = 'MKTXP Entry name'):
+        parser.add_argument('-en', '--entry-name', dest = 'entry_name',
+            type = str,
+            metavar = config_handler.registered_entries() if registered_only else None,
+            required = True,
+            choices = UniquePartialMatchList(config_handler.registered_entries())if registered_only else None,
+            help = help)
+
+    @staticmethod
+    def _add_entry_groups(parser):
+        required_args_group = parser.add_argument_group('Required Arguments')
+        MKTXPOptionsParser._add_entry_name(required_args_group)
+
+
+class MKTXPHelpFormatter(HelpFormatter):
+    ''' Custom formatter for ArgumentParser
+        Disables double metavar display, showing only for long-named options
+    '''
+    def _format_action_invocation(self, action):
+        if not action.option_strings:
+            metavar, = self._metavar_formatter(action, action.dest)(1)
+            return metavar
+        else:
+            parts = []
+            # if the Optional doesn't take a value, format is:
+            #    -s, --long
+            if action.nargs == 0:
+                parts.extend(action.option_strings)
+
+            # if the Optional takes a value, format is:
+            #    -s ARGS, --long ARGS
+            # change to
+            #    -s, --long ARGS
+            else:
+                default = action.dest.upper()
+                args_string = self._format_args(action, default)
+                for option_string in action.option_strings:
+                    #parts.append('%s %s' % (option_string, args_string))
+                    parts.append('%s' % option_string)
+                parts[-1] += ' %s'%args_string
+            return ', '.join(parts)
+
+

mktxp/collectors/base_collector.py

@@ -0,0 +1,58 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from prometheus_client.core import GaugeMetricFamily, CounterMetricFamily, InfoMetricFamily
+from mktxp.cli.config.config import MKTXPConfigKeys
+
+     
+class BaseCollector:
+    ''' Base Collector methods
+        For use by custom collectors
+    '''
+    @staticmethod
+    def info_collector(name, decription, router_records, metric_labels=[]):
+        BaseCollector._add_id_labels(metric_labels)
+        collector = InfoMetricFamily(f'mktxp_{name}', decription)
+
+        for router_record in router_records:
+            label_values = {label: router_record.get(label) if router_record.get(label) else '' for label in metric_labels}
+            collector.add_metric(metric_labels, label_values)
+        return collector
+
+    @staticmethod
+    def counter_collector(name, decription, router_records, metric_key, metric_labels=[]):
+        BaseCollector._add_id_labels(metric_labels)
+        collector = CounterMetricFamily(f'mktxp_{name}', decription, labels=metric_labels)
+
+        for router_record in router_records:           
+            label_values = [router_record.get(label) for label in metric_labels]
+            collector.add_metric(label_values, router_record.get(metric_key, 0))
+        return collector
+
+    @staticmethod
+    def gauge_collector(name, decription, router_records, metric_key, metric_labels=[]):
+        BaseCollector._add_id_labels(metric_labels)
+        collector = GaugeMetricFamily(f'mktxp_{name}', decription, labels=metric_labels)
+
+        for router_record in router_records:       
+            label_values = [router_record.get(label) for label in metric_labels]
+            collector.add_metric(label_values, router_record.get(metric_key, 0))
+        return collector
+
+
+    # Helpers
+    @staticmethod
+    def _add_id_labels(metric_labels):
+        metric_labels.append(MKTXPConfigKeys.ROUTERBOARD_NAME)
+        metric_labels.append(MKTXPConfigKeys.ROUTERBOARD_ADDRESS)

mktxp/collectors/capsman_collector.py

@@ -0,0 +1,33 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+from mktxp.collectors.base_collector import BaseCollector
+from mktxp.router_metric import RouterMetric
+
+
+class CapsmanCollector(BaseCollector):
+    ''' CAPsMAN Metrics collector
+    '''    
+    @staticmethod
+    def collect(router_metric):
+        resource_labels = ['uptime', 'version', 'free-memory', 'total-memory', 
+                           'cpu', 'cpu-count', 'cpu-frequency', 'cpu-load', 
+                           'free-hdd-space', 'total-hdd-space', 
+                           'architecture-name', 'board-name']
+        resource_records = router_metric.resource_records(resource_labels)
+        if not resource_records:
+            return
+
+        resource_metrics = BaseCollector.info_collector('resource', 'resource', resource_records, resource_labels)
+        yield resource_metrics
+

mktxp/collectors/dhcp_collector.py

@@ -0,0 +1,46 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+from mktxp.cli.config.config import MKTXPConfigKeys
+from mktxp.collectors.base_collector import BaseCollector
+from mktxp.router_metric import RouterMetric
+
+
+class DHCPCollector(BaseCollector):
+    ''' DHCP Metrics collector
+    '''    
+    @staticmethod
+    def collect(router_metric):
+        dhcp_lease_labels = ['active_address', 'mac_address', 'host_name', 'comment', 'server', 'expires_after']
+        dhcp_lease_records = router_metric.dhcp_lease_records(dhcp_lease_labels)
+        if not dhcp_lease_records:
+            return
+
+        # calculate number of leases per DHCP server
+        dhcp_lease_servers = {}
+        for dhcp_lease_record in dhcp_lease_records:
+            dhcp_lease_servers[dhcp_lease_record['server']] = dhcp_lease_servers.get(dhcp_lease_record['server'], 0) + 1
+
+        # compile leases-per-server records
+        dhcp_lease_servers_records = [{ MKTXPConfigKeys.ROUTERBOARD_NAME: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
+                                        MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
+                                        'server': key, 'count': value} for key, value in dhcp_lease_servers.items()]
+        
+        # yield lease-per-server metrics
+        dhcp_lease_server_metrics = BaseCollector.gauge_collector('dhcp_lease_active_count', 'Number of active leases per DHCP server', dhcp_lease_servers_records, 'count', ['server'])
+        yield dhcp_lease_server_metrics
+
+        # active lease metrics
+        if router_metric.router_entry.dhcp_lease:
+            dhcp_lease_metrics = BaseCollector.info_collector('dhcp_lease', 'DHCP Active Leases', dhcp_lease_records, dhcp_lease_labels)
+            yield dhcp_lease_metrics

mktxp/collectors/health_collector.py

@@ -0,0 +1,33 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+from mktxp.collectors.base_collector import BaseCollector
+from mktxp.router_metric import RouterMetric
+
+
+class HealthCollector(BaseCollector):
+    ''' System Health Metrics collector
+    '''    
+    @staticmethod
+    def collect(router_metric):
+        health_labels = ['voltage', 'temperature']
+        health_records = router_metric.health_records(health_labels)
+        if not health_records:
+            return
+
+        voltage_metrics = BaseCollector.gauge_collector('routerboard_voltage', 'Supplied routerboard voltage', health_records, 'voltage')
+        yield voltage_metrics
+
+        temperature_metrics = BaseCollector.gauge_collector('routerboard_temperature', ' Routerboard current temperature', health_records, 'temperature')
+        yield temperature_metrics
+

mktxp/collectors/identity_collector.py

@@ -0,0 +1,30 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+from mktxp.collectors.base_collector import BaseCollector
+from mktxp.router_metric import RouterMetric
+
+
+class IdentityCollector(BaseCollector):
+    ''' System Identity Metrics collector
+    '''     
+    @staticmethod
+    def collect(router_metric):
+        identity_labels = ['name']
+        identity_records = router_metric.identity_records(identity_labels)
+        if not identity_records:
+            return
+
+        identity_metrics = BaseCollector.info_collector('identity', 'System identity', identity_records, identity_labels)
+        yield identity_metrics
+

mktxp/collectors/interface_collector.py

@@ -0,0 +1,52 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+from mktxp.collectors.base_collector import BaseCollector
+from mktxp.router_metric import RouterMetric
+
+
+class InterfaceCollector(BaseCollector):
+    ''' Router Interface Metrics collector
+    '''        
+    @staticmethod
+    def collect(router_metric):
+        interface_traffic_labels = ['name', 'comment', 'rx_byte', 'tx_byte', 'rx_packet', 'tx_packet', 'rx_error', 'tx_error', 'rx_drop', 'tx_drop']
+        interface_traffic_records = router_metric.interface_traffic_records(interface_traffic_labels)
+        if not interface_traffic_records:
+            return
+
+        rx_byte_metric = BaseCollector.counter_collector('rx_byte', 'Number of received bytes', interface_traffic_records, 'rx_byte', ['name'])
+        yield rx_byte_metric
+
+        tx_byte_metric = BaseCollector.counter_collector('tx_byte', 'Number of transmitted bytes', interface_traffic_records, 'tx_byte', ['name'])
+        yield tx_byte_metric
+
+        rx_packet_metric = BaseCollector.counter_collector('rx_packet', 'Number of packets received', interface_traffic_records, 'rx_packet', ['name'])
+        yield rx_packet_metric
+
+        tx_packet_metric = BaseCollector.counter_collector('tx_packet', 'Number of transmitted packets', interface_traffic_records, 'tx_packet', ['name'])
+        yield tx_packet_metric
+
+        rx_error_metric = BaseCollector.counter_collector('rx_error', 'Number of packets received with an error', interface_traffic_records, 'rx_error', ['name'])
+        yield rx_error_metric
+
+        tx_error_metric = BaseCollector.counter_collector('tx_error', 'Number of packets transmitted with an error', interface_traffic_records, 'tx_error', ['name'])
+        yield tx_error_metric
+
+        rx_drop_metric = BaseCollector.counter_collector('rx_drop', 'Number of received packets being dropped', interface_traffic_records, 'rx_drop', ['name'])
+        yield rx_drop_metric
+
+        tx_drop_metric = BaseCollector.counter_collector('tx_drop', 'Number of transmitted packets being dropped', interface_traffic_records, 'tx_drop', ['name'])
+        yield tx_drop_metric
+
+

mktxp/collectors/monitor_collector.py

@@ -0,0 +1,77 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+import re
+from mktxp.collectors.base_collector import BaseCollector
+from mktxp.router_metric import RouterMetric
+
+
+class MonitorCollector(BaseCollector):
+    ''' Ethernet Interface Monitor Metrics collector
+    '''    
+    @staticmethod
+    def collect(router_metric):
+        monitor_labels = ['status', 'rate', 'full_duplex', 'name']
+        monitor_records = router_metric.interface_monitor_records(monitor_labels)
+        if not monitor_records:
+            return
+
+        # translate records to appropriate values
+        for monitor_record in monitor_records:
+            for monitor_label in monitor_labels:
+                value = monitor_record.get(monitor_label, None)    
+                if value:            
+                    monitor_record[monitor_label] = MonitorCollector._translated_values(monitor_label, value)
+
+        monitor_status_metrics = BaseCollector.gauge_collector('status', 'Current interface link status', monitor_records, 'status', ['name'])
+        yield monitor_status_metrics
+
+        # limit records according to the relevant metrics
+        rate_records = [monitor_record for monitor_record in monitor_records if monitor_record.get('rate', None)]
+        monitor_rates_metrics = BaseCollector.gauge_collector('rate', 'Actual interface connection data rate', rate_records, 'rate', ['name'])
+        yield monitor_rates_metrics
+
+        full_duplex_records = [monitor_record for monitor_record in monitor_records if monitor_record.get('full_duplex', None)]
+        monitor_rates_metrics = BaseCollector.gauge_collector('full_duplex', 'Full duplex data transmission', full_duplex_records, 'full_duplex', ['name'])
+        yield monitor_rates_metrics
+
+
+    # Helpers
+    @staticmethod
+    def _translated_values(monitor_label, value):
+        return {
+                'status': lambda value: '1' if value=='link-ok' else '0',
+                'rate': lambda value: MonitorCollector._rates(value),
+                'full_duplex': lambda value: '1' if value=='true' else '0',
+                'name': lambda value: value
+                }[monitor_label](value)
+
+    @staticmethod
+    def _rates(rate_option):
+        # according mikrotik docs, an interface rate should be one of these
+        rate_value =  {
+                '10Mbps': '10',
+                '100Mbps': '100',
+                '1Gbps': '1000',
+                '2.5Gbps': '2500',
+                '5Gbps': '5000',
+                '10Gbps': '10000',
+                '40Gbps': '40000'
+                }.get(rate_option, None)
+        if rate_value:
+            return rate_value
+        
+        # ...or just calculate if it's not
+        rate = lambda rate_option: 1000 if rate_option.find('Mbps') < 0 else 1
+        return(int(float(re.sub('[^.\-\d]', '', rate_option)) * rate(rate_option)))
+

mktxp/collectors/pool_collector.py

@@ -0,0 +1,45 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+from mktxp.cli.config.config import MKTXPConfigKeys
+from mktxp.collectors.base_collector import BaseCollector
+from mktxp.router_metric import RouterMetric
+
+
+class PoolCollector(BaseCollector):
+    ''' IP Pool Metrics collector
+    '''    
+    @staticmethod
+    def collect(router_metric):
+
+        # initialize all pool counts, including those currently not used
+        pool_records = router_metric.pool_records(['name'])
+        if not pool_records:
+            return
+
+        pool_used_labels = ['pool']
+        pool_used_counts = {pool_record['name']: 0 for pool_record in pool_records}
+
+        # for pools in usage, calculate the current numbers
+        pool_used_records = router_metric.pool_used_records(pool_used_labels)
+        for pool_used_record in pool_used_records:
+            pool_used_counts[pool_used_record['pool']] = pool_used_counts.get(pool_used_record['pool'], 0) + 1
+
+       # compile used-per-pool records
+        used_per_pool_records = [{ MKTXPConfigKeys.ROUTERBOARD_NAME: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
+                                   MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
+                                   'pool': key, 'count': value} for key, value in pool_used_counts.items()]
+        
+        # yield used-per-pool metrics
+        used_per_pool_metrics = BaseCollector.gauge_collector('ip_pool_used', 'Number of used addresses per IP pool', used_per_pool_records, 'count', ['pool'])
+        yield used_per_pool_metrics

mktxp/collectors/resource_collector.py

@@ -0,0 +1,77 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+import re
+from datetime import timedelta
+from mktxp.collectors.base_collector import BaseCollector
+from mktxp.router_metric import RouterMetric
+
+
+class SystemResourceCollector(BaseCollector):
+    ''' System Resource Metrics collector
+    '''        
+    @staticmethod
+    def collect(router_metric):
+        resource_labels = ['uptime', 'version', 'free_memory', 'total_memory', 
+                           'cpu', 'cpu_count', 'cpu_frequency', 'cpu_load', 
+                           'free_hdd_space', 'total_hdd_space', 
+                           'architecture_name', 'board_name']
+        resource_records = router_metric.system_resource_records(resource_labels)
+        if not resource_records:
+            return
+
+        # translate records to appropriate values
+        translated_fields = ['uptime']        
+        for resource_record in resource_records:
+            for translated_field in translated_fields:
+                value = resource_record.get(translated_field, None)    
+                if value:            
+                    resource_record[translated_field] = SystemResourceCollector._translated_values(translated_field, value)
+
+        uptime_metrics = BaseCollector.gauge_collector('uptime', 'Time interval since boot-up', resource_records, 'uptime', ['version', 'board_name', 'cpu', 'architecture_name'])
+        yield uptime_metrics
+
+        free_memory_metrics = BaseCollector.gauge_collector('free_memory', 'Unused amount of RAM', resource_records, 'free_memory', ['version', 'board_name', 'cpu', 'architecture_name'])
+        yield free_memory_metrics
+
+        total_memory_metrics = BaseCollector.gauge_collector('total_memory', 'Amount of installed RAM', resource_records, 'total_memory', ['version', 'board_name', 'cpu', 'architecture_name'])
+        yield total_memory_metrics
+
+        free_hdd_metrics = BaseCollector.gauge_collector('free_hdd_space', 'Free space on hard drive or NAND', resource_records, 'free_hdd_space', ['version', 'board_name', 'cpu', 'architecture_name'])
+        yield free_hdd_metrics
+
+        total_hdd_metrics = BaseCollector.gauge_collector('total_hdd_space', 'Size of the hard drive or NAND', resource_records, 'total_hdd_space', ['version', 'board_name', 'cpu', 'architecture_name'])
+        yield total_hdd_metrics
+
+        cpu_load_metrics = BaseCollector.gauge_collector('cpu_load', 'Percentage of used CPU resources', resource_records, 'cpu_load', ['version', 'board_name', 'cpu', 'architecture_name'])
+        yield cpu_load_metrics
+
+        cpu_count_metrics = BaseCollector.gauge_collector('cpu_count', 'Number of CPUs present on the system', resource_records, 'cpu_count', ['version', 'board_name', 'cpu', 'architecture_name'])
+        yield cpu_count_metrics
+
+        cpu_frequency_metrics = BaseCollector.gauge_collector('cpu_frequency', 'Current CPU frequency', resource_records, 'cpu_frequency', ['version', 'board_name', 'cpu', 'architecture_name'])
+        yield cpu_frequency_metrics
+
+
+    # Helpers
+    @staticmethod
+    def _translated_values(translated_field, value):
+        return {
+                'uptime': lambda value: SystemResourceCollector._parse_uptime(value)
+                }[translated_field](value)
+
+    @staticmethod
+    def _parse_uptime(time):
+        time_dict = re.match(r'((?P<weeks>\d+)w)?((?P<days>\d+)d)?((?P<hours>\d+)h)?((?P<minutes>\d+)m)?((?P<seconds>\d+)s)?', time).groupdict()
+        return timedelta(**{key: int(value) for key, value in time_dict.items() if value}).total_seconds()
+

mktxp/collectors/route_collector.py

@@ -0,0 +1,54 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+from mktxp.cli.config.config import MKTXPConfigKeys
+from mktxp.collectors.base_collector import BaseCollector
+from mktxp.router_metric import RouterMetric
+
+
+class RouteCollector(BaseCollector):
+    ''' IP Route Metrics collector
+    '''        
+    @staticmethod
+    def collect(router_metric):
+        route_labels = ['connect', 'dynamic', 'static', 'bgp', 'ospf']
+        route_records = router_metric.route_records(route_labels)
+        if not route_records:
+            return
+        
+        # compile total routes records
+        total_routes = len(route_records)
+        total_routes_records = [{ MKTXPConfigKeys.ROUTERBOARD_NAME: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
+                                  MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
+                                  'count': total_routes
+                                }]
+        total_routes_metrics = BaseCollector.gauge_collector('total_routes', 'Overall number of routes in RIB', total_routes_records, 'count')
+        yield total_routes_metrics
+
+
+        # init routes per protocol (with 0)
+        routes_per_protocol = {route_label: 0 for route_label in route_labels}
+        for route_record in route_records:
+            for route_label in route_labels:
+                if route_record.get(route_label):
+                    routes_per_protocol[route_label] += 1 
+
+        # compile route-per-protocol records
+        route_per_protocol_records = [{ MKTXPConfigKeys.ROUTERBOARD_NAME: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_NAME],
+                                        MKTXPConfigKeys.ROUTERBOARD_ADDRESS: router_metric.router_id[MKTXPConfigKeys.ROUTERBOARD_ADDRESS],
+                                        'protocol': key, 'count': value} for key, value in routes_per_protocol.items()]
+        
+        # yield route-per-protocol metrics
+        route_per_protocol_metrics = BaseCollector.gauge_collector('routes_protocol_count', 'Number of routes per protocol in RIB', route_per_protocol_records, 'count', ['protocol'])
+        yield route_per_protocol_metrics
+

mktxp/collectors/wlan_collector.py

@@ -0,0 +1,38 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+from mktxp.collectors.base_collector import BaseCollector
+from mktxp.router_metric import RouterMetric
+
+
+class WLANCollector(BaseCollector):
+    ''' Wireless Metrics collector
+    '''    
+    @staticmethod
+    def collect(router_metric):
+        monitor_labels = ['channel', 'noise_floor', 'overall_tx_ccq']
+        monitor_records = router_metric.interface_monitor_records(monitor_labels, 'wireless')
+        if not monitor_records:
+            return
+
+        # sanitize records for relevant labels
+        noise_floor_records = [monitor_record for monitor_record in monitor_records if monitor_record.get('noise_floor')]
+        tx_ccq_records = [monitor_record for monitor_record in monitor_records if monitor_record.get('overall_tx_ccq')]
+
+        if noise_floor_records:
+            noise_floor_metrics = BaseCollector.gauge_collector('noise_floor', 'Noise floor threshold', noise_floor_records, 'noise_floor', ['channel'])
+            yield noise_floor_metrics
+
+        if tx_ccq_records:
+            overall_tx_ccq_metrics = BaseCollector.gauge_collector('overall_tx_ccq', ' Client Connection Quality for transmitting', tx_ccq_records, 'overall_tx_ccq', ['channel'])
+            yield overall_tx_ccq_metrics

mktxp/collectors_handler.py

@@ -0,0 +1,57 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+from mktxp.collectors.dhcp_collector import DHCPCollector
+from mktxp.collectors.interface_collector import InterfaceCollector
+from mktxp.collectors.health_collector import HealthCollector
+from mktxp.collectors.identity_collector import IdentityCollector
+from mktxp.collectors.monitor_collector import MonitorCollector
+from mktxp.collectors.pool_collector import PoolCollector
+from mktxp.collectors.resource_collector import SystemResourceCollector
+from mktxp.collectors.route_collector import RouteCollector
+from mktxp.collectors.wlan_collector import WLANCollector
+
+class CollectorsHandler:
+    ''' MKTXP Collectors Handler
+    '''
+    def __init__(self, metrics_handler):
+        self.metrics_handler = metrics_handler
+
+    def collect(self):
+        for router_metric in self.metrics_handler.router_metrics:
+            yield from IdentityCollector.collect(router_metric)
+            yield from SystemResourceCollector.collect(router_metric)
+            yield from HealthCollector.collect(router_metric)
+
+            if router_metric.router_entry.dhcp:
+                yield from DHCPCollector.collect(router_metric)
+
+            if router_metric.router_entry.pool:
+                yield from PoolCollector.collect(router_metric)
+            
+            if router_metric.router_entry.interface:
+                yield from InterfaceCollector.collect(router_metric)
+            
+            if router_metric.router_entry.monitor:
+                yield from MonitorCollector.collect(router_metric)
+            
+            if router_metric.router_entry.route:
+                yield from RouteCollector.collect(router_metric)
+        
+            if router_metric.router_entry.wireless:
+                yield from WLANCollector.collect(router_metric)
+
+
+
+
+

mktxp/metrics_handler.py

@@ -0,0 +1,29 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from mktxp.cli.config.config import config_handler
+from mktxp.router_metric import RouterMetric
+
+
+class RouterMetricsHandler:
+    ''' Handles RouterOS entries defined in MKTXP config 
+    '''         
+    def __init__(self):
+        self.router_metrics = []
+        for router_name in config_handler.registered_entries():
+            entry = config_handler.entry(router_name)
+            if entry.enabled:
+                self.router_metrics.append(RouterMetric(router_name))
+
+

mktxp/router_connection.py

@@ -0,0 +1,72 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+import ssl
+import socket
+from routeros_api import RouterOsApiPool
+
+
+class RouterAPIConnectionError(Exception):
+    pass
+
+
+class RouterAPIConnection:
+    ''' Base wrapper interface for the routeros_api library
+    '''            
+    def __init__(self, router_name, router_entry):
+        self.router_name = router_name
+        self.router_entry  = router_entry
+        
+        ctx = None
+        if not self.router_entry.ssl_certificate:
+            ctx = ssl.create_default_context()
+            ctx.set_ciphers('ADH:@SECLEVEL=0')       
+
+        self.connection = RouterOsApiPool(
+                host = self.router_entry.hostname,
+                username = self.router_entry.username,
+                password = self.router_entry.password,
+                port = self.router_entry.port,
+                plaintext_login = True,
+                use_ssl = True,
+                ssl_context = ctx)
+        
+        self.connection.socket_timeout = 2
+        self.api = None
+
+    def is_connected(self):
+        if not (self.connection and self.connection.connected and self.api):
+            return False
+        try:
+            self.api.get_resource('/system/identity').get()
+            return True
+        except (socket.error, socket.timeout, Exception) as ex:
+            print(f'Connection to router {self.router_name}@{self.router_entry.hostname} has been lost: {ex}')
+            self.api = None
+            return False
+
+    def connect(self):
+        if self.is_connected():
+            return
+        try:
+            print('Connecting to router {0}@{1}'.format(self.router_name, self.router_entry.hostname))
+            self.api = self.connection.get_api()
+            print('Connection to router {0}@{1} has been established'.format(self.router_name, self.router_entry.hostname))
+        except (socket.error, socket.timeout, Exception) as ex:
+            print('Connection to router {0}@{1} has failed: {2}'.format(self.router_name, self.router_entry.hostname, ex))
+            raise 
+
+    def router_api(self):
+        if not self.is_connected():
+            self.connect()
+        return self.api

mktxp/router_metric.py

@@ -0,0 +1,133 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from mktxp.cli.config.config import config_handler, MKTXPConfigKeys
+from mktxp.router_connection import RouterAPIConnection
+
+class RouterMetric:
+    ''' RouterOS Metrics data provider
+    '''             
+    def __init__(self, router_name):
+        self.router_name = router_name
+        self.router_entry  = config_handler.entry(router_name)
+        self.api_connection = RouterAPIConnection(router_name, self.router_entry)
+        self.router_id = {
+            MKTXPConfigKeys.ROUTERBOARD_NAME: self.router_name,
+            MKTXPConfigKeys.ROUTERBOARD_ADDRESS: self.router_entry.hostname
+            }
+
+    def identity_records(self, identity_labels = []):
+        try:
+            identity_records = self.api_connection.router_api().get_resource('/system/identity').get()
+            return self._trimmed_records(identity_records, identity_labels)
+        except Exception as exc:
+            print(f'Error getting system identity info from router{self.router_name}@{self.router_entry.hostname}: {exc}')
+            return None
+
+    def routerboard_records(self, routerboard_labels = []):
+        try:
+            routerboard_records = self.api_connection.router_api().get_resource('/system/routerboard').get()
+            return self._trimmed_records(routerboard_records, routerboard_labels)
+        except Exception as exc:
+            print(f'Error getting system routerboard info from router{self.router_name}@{self.router_entry.hostname}: {exc}')
+            return None
+
+    def health_records(self, health_labels = []):
+        try:
+            health_records = self.api_connection.router_api().get_resource('/system/health').get()
+            return self._trimmed_records(health_records, health_labels)
+        except Exception as exc:
+            print(f'Error getting system health info from router{self.router_name}@{self.router_entry.hostname}: {exc}')
+            return None
+
+    def system_resource_records(self, resource_labels = []):
+        try:
+            system_resource_records = self.api_connection.router_api().get_resource('/system/resource').get()
+            return self._trimmed_records(system_resource_records, resource_labels)
+        except Exception as exc:
+            print(f'Error getting system resource info from router{self.router_name}@{self.router_entry.hostname}: {exc}')
+            return None
+
+    def dhcp_lease_records(self, dhcp_lease_labels = []):
+        try:
+            #dhcp_lease_records = self.api_connection.router_api().get_resource('/ip/dhcp-server/lease').get(status='bound')
+            dhcp_lease_records = self.api_connection.router_api().get_resource('/ip/dhcp-server/lease').call('print', {'active':''})
+            return self._trimmed_records(dhcp_lease_records, dhcp_lease_labels)
+        except Exception as exc:
+            print(f'Error getting dhcp info from router{self.router_name}@{self.router_entry.hostname}: {exc}')
+            return None
+
+    def interface_traffic_records(self, interface_traffic_labels = []):
+        try:
+            traffic_records = self.api_connection.router_api().get_resource('/interface').get(running='yes', disabled='no')
+            return self._trimmed_records(traffic_records, interface_traffic_labels)
+        except Exception as exc:
+            print(f'Error getting interface traffic info from router{self.router_name}@{self.router_entry.hostname}: {exc}')
+            return None
+
+    def interface_monitor_records(self, interface_monitor_labels = [], kind = 'ethernet'):
+        try:
+            interfaces = self.api_connection.router_api().get_resource(f'/interface/{kind}').get()
+            interface_names = [interface['name'] for interface in interfaces]
+
+            interface_monitor = lambda int_num : self.api_connection.router_api().get_resource(f'/interface/{kind}').call('monitor', {'once':'', 'numbers':f'{int_num}'})
+            interface_monitor_records = [interface_monitor(int_num)[0] for int_num in range(len(interface_names))]
+            return self._trimmed_records(interface_monitor_records, interface_monitor_labels)
+        except Exception as exc:
+            print(f'Error getting {kind} interface monitor info from router{self.router_name}@{self.router_entry.hostname}: {exc}')
+            return None
+
+    def pool_records(self, pool_labels = []):
+        try:
+            pool_records = self.api_connection.router_api().get_resource('/ip/pool').get()
+            return self._trimmed_records(pool_records, pool_labels)
+        except Exception as exc:
+            print(f'Error getting pool info from router{self.router_name}@{self.router_entry.hostname}: {exc}')
+            return None
+
+    def pool_used_records(self, pool_used_labels = []):
+        try:
+            pool_used_records = self.api_connection.router_api().get_resource('/ip/pool/used').get()
+            return self._trimmed_records(pool_used_records, pool_used_labels)
+        except Exception as exc:
+            print(f'Error getting pool used info from router{self.router_name}@{self.router_entry.hostname}: {exc}')
+            return None
+
+    def route_records(self, route_labels = []):
+        try:
+            route_records = self.api_connection.router_api().get_resource('/ip/route').get(active='yes')
+            return self._trimmed_records(route_records, route_labels)
+        except Exception as exc:
+            print(f'Error getting pool active routes info from router{self.router_name}@{self.router_entry.hostname}: {exc}')
+            return None
+
+
+    # Helpers
+    def _trimmed_records(self, router_records, metric_labels):
+        if len(metric_labels) == 0 and len(router_records) > 0:
+            metric_labels = router_records[0].keys()
+        metric_labels = set(metric_labels)      
+
+        labeled_records = []
+        dash2_ = lambda x : x.replace('-', '_')
+        for router_record in router_records:
+            translated_record = {dash2_(key): value for (key, value) in router_record.items() if dash2_(key) in metric_labels}
+            for key, value in self.router_id.items():
+                translated_record[key] = value
+            labeled_records.append(translated_record)
+        return labeled_records
+
+
+
+

mktxp/utils/utils.py

@@ -0,0 +1,273 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+import os, sys, shlex, tempfile, shutil, re
+import subprocess, hashlib
+import keyring, getpass
+from collections import Iterable
+from contextlib import contextmanager
+
+''' Utilities / Helpers
+'''
+
+@contextmanager
+def temp_dir(quiet = True):
+    ''' Temp dir context manager
+    '''
+    tmp_dir = tempfile.mkdtemp()
+    try:
+        yield tmp_dir
+    finally:
+        # remove tmp dir
+        try:
+            shutil.rmtree(tmp_dir)
+        except OSError as e:
+            if not quiet:
+                print ('Error while removing a tmp dir: {}'.format(e.args[0]))
+
+class CmdProcessingError(Exception):
+    pass
+
+def run_cmd(cmd, shell = False):
+    ''' Runs shell commands in a separate process
+    '''
+    if not shell:
+        cmd = shlex.split(cmd)
+    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell = shell)
+    output = proc.communicate()[0].decode('utf-8')
+    if proc.returncode != 0:
+        raise CmdProcessingError(output)
+    return output
+
+def get_last_digit_from_shell_cmd(cmd):
+    try:
+        cmd_output = run_cmd(cmd, shell = True)
+    except CmdProcessingError as e:
+        if not quiet:
+            print ('Error while running cmd: {}'.format(e.args[0]))
+        return -1
+    else:
+        return get_last_digit(cmd_output)
+
+def get_last_digit(str_to_search):
+    p = re.compile('(\d*\.?\d+)')
+    match = p.search(str_to_search)
+    if match:
+        return float(match.group())
+    else:
+        return -1
+
+def encfs_version():
+    cmd = 'encfs --version'
+    return get_last_digit_from_shell_cmd(cmd)
+
+
+class FSHelper:
+    ''' File System ops helper
+    '''
+    @staticmethod
+    def full_path(path, check_parent_path = False):
+        ''' Full path
+        '''
+        if path:
+            path = os.path.expanduser(path)
+            path = os.path.expandvars(path)
+            path = os.path.abspath(path)
+            path = os.path.realpath(path)
+
+        # for files, check that the parent dir exists
+        if check_parent_path:
+            if not os.access(os.path.dirname(path), os.W_OK):
+                print('Non-valid folder path:\n\t "{}"'.format(os.path.dirname(path)))
+                sys.exit(1)
+
+        return path if path else None
+
+    @staticmethod
+    def mountpoint(path):
+        ''' The mount point portion of a path
+        '''
+        path = FSHelper.full_path(path)
+        while path != os.path.sep:
+            if os.path.ismount(path):
+                return path
+            path = os.path.realpath(os.path.join(path, os.pardir))
+        return path if path != os.path.sep else None
+
+    @staticmethod
+    def move_FS_entry(orig_path, target_path,
+                      check_unique = True,
+                      quiet = False, stop = False):
+        ''' Moves FS entry
+        '''
+        succeeded = False
+        try:
+            if check_unique and os.path.exists(target_path):
+                raise OSError('\nTarget path entry already exists')
+            shutil.move(orig_path, target_path)
+            succeeded = True
+        except OSError as e:
+            if not quiet:
+                print(str(e))
+                print('Failed to move entry:\n\t{0}\n\t{1}'.format(orig_path, target_path))
+                print('Exiting...') if stop else print('Skipping...')
+            if stop:
+                sys.exit(1)
+        return succeeded
+
+    @staticmethod
+    def file_md5(fpath, block_size=0, hex=False):
+        ''' Calculates MD5 hash for a file at fpath
+        '''
+        md5 = hashlib.md5()
+        if block_size == 0:
+            block_size = 128 * md5.block_size
+        with open(fpath,'rb') as f:
+            for chunk in iter(lambda: f.read(block_size), b''):
+                md5.update(chunk)
+        return md5.hexdigest() if hex else md5.digest()
+
+
+class UniqueDirNamesChecker:
+    ''' Unique file names Helper
+    '''
+    def __init__(self, src_dir, unique_fnames = None):
+        self._uname_gen = unique_fnames() if unique_fnames else self.unique_fnames()
+
+        # init the generator function with file names from given source directory
+        src_dir = FSHelper.full_path(src_dir)
+        fnames = [fname for fname in os.listdir(src_dir)]
+
+        for fname in fnames:
+            next(self._uname_gen)
+            self._uname_gen.send(fname)
+
+    def unique_name(self, fname):
+        ''' Returns unique file name
+        '''
+        next(self._uname_gen)
+        return self._uname_gen.send(fname)
+
+    @staticmethod
+    def unique_fnames():
+        ''' default unique file names generator method,
+            via appending a simple numbering pattern
+        '''
+        unique_names = {}
+        while True:
+            fname = yield
+            while True:
+                if fname in unique_names:
+                    unique_names[fname] += 1
+                    name_base, name_ext = os.path.splitext(fname)
+                    fname = '{0}_{1}{2}'.format(name_base, unique_names[fname], name_ext)
+                else:
+                    unique_names[fname] = 0
+                    yield fname
+                    break
+
+
+class PasswordHandler:
+    ''' Password Helper
+    '''
+    @staticmethod
+    def get_pwd_input(confirm = False):
+        ''' Gets password from command line
+        '''
+        pwd = getpass.getpass('Enter password:')
+        if pwd and confirm:
+            pwd_confirm = getpass.getpass('Confirm password:')
+            if pwd != pwd_confirm:
+                print ("Passwords do not match")
+                return None
+        return pwd
+
+    @classmethod
+    def get_pwd(cls, pwd_entry_name = None, confirm = False):
+        ''' Gets password from an OS-specific keyring or command line
+        '''
+        pwd = None
+        new_pwd = False
+
+        if pwd_entry_name:
+            pwd = keyring.get_password(pwd_entry_name, getpass.getuser())
+
+        if not pwd:
+            pwd = cls.get_pwd_input(confirm = confirm)
+            new_pwd = True
+
+        return pwd, new_pwd
+
+    @staticmethod
+    def store_pwd(pwd, pwd_entry_name):
+        ''' Store password into an OS-specific keyring
+        '''
+        if not (pwd and pwd_entry_name):
+            return
+        keyring.set_password(pwd_entry_name, getpass.getuser(), pwd)
+
+    @staticmethod
+    def delete_pwd(pwd_entry_name):
+        ''' Deletes password from an OS-specific keyring
+        '''
+        if pwd_entry_name:
+            pwd = keyring.get_password(pwd_entry_name, getpass.getuser())
+            if pwd:
+                keyring.delete_password(pwd_entry_name, getpass.getuser())
+
+
+class UniquePartialMatchList(list):
+    ''' Enables matching elements by unique "shortcuts"
+        e.g:
+            >> 'Another' in UniquePartialMatchList(['A long string', 'Another longs string'])
+            >> True
+            >>'long' in UniquePartialMatchList(['A long string', 'Another longs string'])
+            >> False
+            >> l.find('Another')
+            >> 'Another longs string'
+    '''
+    def _matched_items(self, partialMatch):
+        ''' Generator expression of <matched items>, where <matched item> is
+            a tuple of (<matched_element>, <is_exact_match>)
+        '''
+        def _contains_or_equal(item):
+            if isinstance(item, Iterable):
+                return (partialMatch in item)
+            else:
+                return (partialMatch == item)
+        return ((item, (partialMatch == item)) for item in self if _contains_or_equal(item))
+
+    def find(self, partialMatch):
+        ''' Returns the element in which <partialMatch> can be found
+            <partialMatch> is found if it either:
+                equals to an element or is contained by exactly one element
+        '''
+        matched_cnt, unique_match = 0, None
+        matched_items = self._matched_items(partialMatch)
+        for match, exact_match in matched_items:
+            if exact_match:
+                # found exact match
+                return match
+            else:
+                # found a partial match
+                if not unique_match:
+                    unique_match = match
+                matched_cnt += 1
+        return unique_match if matched_cnt == 1 else None
+
+    def __contains__(self, partialMatch):
+        ''' Check if <partialMatch> is contained by an element in the list,
+            where <contained> is defined either as:
+                either "equals to element" or "contained by exactly one element"
+        '''
+        return True if self.find(partialMatch) else False