DS refactor, fixes/optimizations

mktxp/datasource/base_ds.py

@@ -0,0 +1,39 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+class BaseDSProcessor:
+    ''' Base Metrics DataSource processing
+    '''             
+
+    @staticmethod
+    def trimmed_records(router_entry, *, router_records = [], metric_labels = [], add_router_id = True, translation_table = {}):
+        if len(metric_labels) == 0 and len(router_records) > 0:
+            metric_labels = router_records[0].keys()
+        metric_labels = set(metric_labels)      
+
+        labeled_records = []
+        dash2_ = lambda x : x.replace('-', '_')
+        for router_record in router_records:
+            translated_record = {dash2_(key): value for (key, value) in router_record.items() if dash2_(key) in metric_labels}
+
+            if add_router_id:
+                for key, value in router_entry.router_id.items():
+                    translated_record[key] = value
+            
+            # translate fields if needed
+            for key, func in translation_table.items():
+                translated_record[key] = func(translated_record.get(key))
+            labeled_records.append(translated_record)
+            
+        return labeled_records

mktxp/datasource/capsman_ds.py

@@ -0,0 +1,41 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from mktxp.datasource.base_ds import BaseDSProcessor
+
+
+class CapsmanCapsMetricsDataSource:
+    ''' Caps Metrics data provider
+    '''             
+    @staticmethod
+    def metric_records(router_entry, *, metric_labels = []):
+        try:
+            remote_caps_records = router_entry.api_connection.router_api().get_resource('/caps-man/remote-cap').get()
+            return BaseDSProcessor.trimmed_records(router_entry, router_records = remote_caps_records, metric_labels = metric_labels)
+        except Exception as exc:
+            print(f'Error getting caps-man remote caps info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}')
+            return None
+
+
+class CapsmanRegistrationsMetricsDataSource:
+    ''' Capsman Registrations Metrics data provider
+    '''             
+    @staticmethod
+    def metric_records(router_entry, *, metric_labels = [],  add_router_id = True):
+        try:
+            registration_table_records = router_entry.api_connection.router_api().get_resource('/caps-man/registration-table').get()
+            return BaseDSProcessor.trimmed_records(router_entry, router_records = registration_table_records, metric_labels = metric_labels, add_router_id = add_router_id)
+        except Exception as exc:
+            print(f'Error getting caps-man registration table info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}')
+            return None

mktxp/datasource/dhcp_ds.py

@@ -0,0 +1,43 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from mktxp.datasource.base_ds import BaseDSProcessor
+
+
+class DHCPMetricsDataSource:
+    ''' DHCP Metrics data provider
+    '''             
+    @staticmethod
+    def metric_records(router_entry, *, metric_labels = [], add_router_id = True):
+        try:
+            #dhcp_lease_records = router_entry.api_connection.router_api().get_resource('/ip/dhcp-server/lease').get(status='bound')
+            dhcp_lease_records = router_entry.api_connection.router_api().get_resource('/ip/dhcp-server/lease').call('print', {'active':''})
+
+            # translation rules
+            translation_table = {}
+            if 'comment' in metric_labels:
+                translation_table['comment'] = lambda c: c if c else ''           
+            if 'host_name' in metric_labels:
+                translation_table['host_name'] = lambda c: c if c else ''           
+            if 'expires_after' in metric_labels:
+                translation_table['expires_after'] = lambda c: c if c else ''        
+            if 'active_address' in metric_labels:
+                translation_table['active_address'] = lambda c: c if c else ''        
+
+            return BaseDSProcessor.trimmed_records(router_entry, router_records = dhcp_lease_records, metric_labels = metric_labels, add_router_id = add_router_id, translation_table = translation_table)
+        except Exception as exc:
+            print(f'Error getting dhcp info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}')
+            return None
+
+

mktxp/datasource/firewall_ds.py

@@ -0,0 +1,39 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from mktxp.datasource.base_ds import BaseDSProcessor
+
+
+class FirewallMetricsDataSource:
+    ''' Firewall Metrics data provider
+    '''             
+    @staticmethod
+    def metric_records(router_entry, *, metric_labels = [], raw = False, matching_only = True):
+        try:
+            filter_path = '/ip/firewall/filter' if not raw else '/ip/firewall/raw'
+            firewall_records = router_entry.api_connection.router_api().get_resource(filter_path).call('print', {'stats':'', 'all':''})
+            if matching_only:
+                firewall_records = [record for record in firewall_records if int(record.get('bytes', '0')) > 0]
+
+            # translation rules
+            translation_table = {}
+            if 'comment' in metric_labels:
+                translation_table['comment'] = lambda c: c if c else ''           
+
+            return BaseDSProcessor.trimmed_records(router_entry, router_records = firewall_records, metric_labels = metric_labels, translation_table = translation_table)
+        except Exception as exc:
+            print(f'Error getting firewall filters info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}')
+            return None
+
+

mktxp/datasource/health_ds.py

@@ -0,0 +1,28 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from mktxp.datasource.base_ds import BaseDSProcessor
+
+
+class HealthMetricsDataSource:
+    ''' Health Metrics data provider
+    '''             
+    @staticmethod
+    def metric_records(router_entry, *, metric_labels = []):
+        try:
+            health_records = router_entry.api_connection.router_api().get_resource('/system/health').get()
+            return BaseDSProcessor.trimmed_records(router_entry, router_records = health_records, metric_labels = metric_labels)
+        except Exception as exc:
+            print(f'Error getting system health info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}')
+            return None

mktxp/datasource/identity_ds.py

@@ -0,0 +1,28 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from mktxp.datasource.base_ds import BaseDSProcessor
+
+
+class IdentityMetricsDataSource:
+    ''' Identity Metrics data provider
+    '''             
+    @staticmethod
+    def metric_records(router_entry, *, metric_labels = []):
+        try:
+            identity_records = router_entry.api_connection.router_api().get_resource('/system/identity').get()
+            return BaseDSProcessor.trimmed_records(router_entry, router_records = identity_records, metric_labels = metric_labels)
+        except Exception as exc:
+            print(f'Error getting system identity info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}')
+            return None

mktxp/datasource/interface_ds.py

@@ -0,0 +1,59 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from mktxp.datasource.base_ds import BaseDSProcessor
+
+
+class InterfaceTrafficMetricsDataSource:
+    ''' Interface Traffic Metrics data provider
+    '''             
+    @staticmethod
+    def metric_records(router_entry, *, metric_labels = []):
+        try:
+            traffic_records = router_entry.api_connection.router_api().get_resource('/interface').get(running='yes', disabled='no')
+            return BaseDSProcessor.trimmed_records(router_entry, router_records = traffic_records, metric_labels = metric_labels)
+        except Exception as exc:
+            print(f'Error getting interface traffic info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}')
+            return None
+
+
+class InterfaceMonitorMetricsDataSource:
+    ''' Interface Monitor Metrics data provider
+    '''             
+    @staticmethod
+    def metric_records(router_entry, *, metric_labels = [], kind = 'ethernet', include_comments = False, running_only = True):
+        try:
+            interfaces = router_entry.api_connection.router_api().get_resource(f'/interface/{kind}').get()
+            interface_names = [(interface['name'], interface.get('comment'), interface.get('running')) for interface in interfaces]
+
+            interface_monitor_records = []
+            for int_num, interface_name in enumerate(interface_names):
+                interface_monitor_record = {}
+                if not running_only or interface_name[2] == 'true':
+                    interface_monitor_record = router_entry.api_connection.router_api().get_resource(f'/interface/{kind}').call('monitor', {'once':'', 'numbers':f'{int_num}'})[0]
+                else:
+                    # unless explicitly requested, no need to do a monitor call for not running interfaces                    
+                    interface_monitor_record = {'name': interface_name[0], 'status': 'no-link'}
+
+                if include_comments and interface_name[1]:
+                    # combines names with comments
+                    interface_monitor_record['name'] = interface_name[1] if router_entry.config_entry.use_comments_over_names else \
+                                                                                                        f"{interface_name[0]} ({interface_name[1]})"                                
+                interface_monitor_records.append(interface_monitor_record)
+                
+            return BaseDSProcessor.trimmed_records(router_entry, router_records = interface_monitor_records, metric_labels = metric_labels)
+        except Exception as exc:
+            print(f'Error getting {kind} interface monitor info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}')
+            return None
+

mktxp/datasource/mktxp_ds.py

@@ -0,0 +1,29 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from mktxp.datasource.base_ds import BaseDSProcessor
+
+
+class MKTXPMetricsDataSource:
+    ''' MKTXP Metrics data provider
+    '''             
+    @staticmethod
+    def metric_records(router_entry):
+        mktxp_records = []
+        for key in router_entry.time_spent.keys():
+            mktxp_records.append({'name': key, 'duration': router_entry.time_spent[key]})           
+
+        # translation rules            
+        translation_table = {'duration': lambda d: d*1000}
+        return BaseDSProcessor.trimmed_records(router_entry, router_records = mktxp_records, translation_table = translation_table)

mktxp/datasource/pool_ds.py

@@ -0,0 +1,41 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from mktxp.datasource.base_ds import BaseDSProcessor
+
+
+class PoolMetricsDataSource:
+    ''' Pool Metrics data provider
+    '''             
+    @staticmethod
+    def metric_records(router_entry, *, metric_labels = []):
+        try:
+            pool_records = router_entry.api_connection.router_api().get_resource('/ip/pool').get()
+            return BaseDSProcessor.trimmed_records(router_entry, router_records = pool_records, metric_labels = metric_labels)
+        except Exception as exc:
+            print(f'Error getting pool info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}')
+            return None
+
+
+class PoolUsedMetricsDataSource:
+    ''' Pool/Used Metrics data provider
+    '''             
+    @staticmethod
+    def metric_records(router_entry, *, metric_labels = []):
+        try:
+            pool_used_records = router_entry.api_connection.router_api().get_resource('/ip/pool/used').get()
+            return BaseDSProcessor.trimmed_records(router_entry, router_records = pool_used_records, metric_labels = metric_labels)
+        except Exception as exc:
+            print(f'Error getting pool used info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}')
+            return None

mktxp/datasource/route_ds.py

@@ -0,0 +1,28 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from mktxp.datasource.base_ds import BaseDSProcessor
+
+
+class RouteMetricsDataSource:
+    ''' Routes Metrics data provider
+    '''             
+    @staticmethod
+    def metric_records(router_entry, *, metric_labels = []):
+        try:
+            route_records = router_entry.api_connection.router_api().get_resource('/ip/route').get(active='yes')
+            return BaseDSProcessor.trimmed_records(router_entry, router_records = route_records, metric_labels = metric_labels)
+        except Exception as exc:
+            print(f'Error getting routes info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}')
+            return None

mktxp/datasource/routerboard_ds.py

@@ -0,0 +1,29 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from mktxp.datasource.base_ds import BaseDSProcessor
+
+
+class RouterboardMetricsDataSource:
+    ''' Routerboard Metrics data provider
+    '''             
+    @staticmethod
+    def metric_records(router_entry, *, metric_labels = []):
+        try:
+            routerboard_records = router_entry.api_connection.router_api().get_resource('/system/routerboard').get()
+            return BaseDSProcessor.trimmed_records(router_entry, router_records = routerboard_records, metric_labels = metric_labels)
+        except Exception as exc:
+            print(f'Error getting system routerboard info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}')
+            return None
+

mktxp/datasource/system_resource_ds.py

@@ -0,0 +1,28 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from mktxp.datasource.base_ds import BaseDSProcessor
+
+
+class SystemResourceMetricsDataSource:
+    ''' System Resource Metrics data provider
+    '''             
+    @staticmethod    
+    def metric_records(router_entry, *, metric_labels = []):
+        try:
+            system_resource_records = router_entry.api_connection.router_api().get_resource('/system/resource').get()
+            return BaseDSProcessor.trimmed_records(router_entry, router_records = system_resource_records, metric_labels = metric_labels)
+        except Exception as exc:
+            print(f'Error getting system resource info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}')
+            return None

mktxp/datasource/wireless_ds.py

@@ -0,0 +1,30 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from mktxp.datasource.base_ds import BaseDSProcessor
+
+
+class WirelessMetricsDataSource:
+    ''' Wireless Metrics data provider
+    '''             
+    @staticmethod
+    def metric_records(router_entry, *, metric_labels = [], add_router_id = True):
+        try:
+            registration_table_records = router_entry.api_connection.router_api().get_resource('/interface/wireless/registration-table').get()
+            return BaseDSProcessor.trimmed_records(router_entry, router_records = registration_table_records, metric_labels = metric_labels, add_router_id = add_router_id)
+        except Exception as exc:
+            print(f'Error getting wireless registration table info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}')
+            return None
+
+