From 0a500f12c46d9aac9aa17178a7f62dfb2fa99c6a Mon Sep 17 00:00:00 2001 From: Leon Morten Richter Date: Sun, 13 Nov 2022 12:17:07 +0100 Subject: [PATCH 1/4] add IPv6 firewall records --- mktxp/collector/firewall_collector.py | 29 ++++++--- mktxp/datasource/firewall_ds.py | 88 +++++++++++++++++++-------- 2 files changed, 84 insertions(+), 33 deletions(-) diff --git a/mktxp/collector/firewall_collector.py b/mktxp/collector/firewall_collector.py index 50b7180..19e9bbd 100644 --- a/mktxp/collector/firewall_collector.py +++ b/mktxp/collector/firewall_collector.py @@ -25,21 +25,36 @@ class FirewallCollector(BaseCollector): if not router_entry.config_entry.firewall: return - # initialize all pool counts, including those currently not used + # Initialize all pool counts, including those currently not used + # These are the same for both IPv4 and IPv6 firewall_labels = ['chain', 'action', 'bytes', 'comment', 'log'] - firewall_filter_records = FirewallMetricsDataSource.metric_records(router_entry, metric_labels = firewall_labels) + # ~*~*~*~*~*~ IPv4 ~*~*~*~*~*~ + firewall_filter_records = FirewallMetricsDataSource.metric_records_ipv4(router_entry, metric_labels = firewall_labels) if firewall_filter_records: - metris_records = [FirewallCollector.metric_record(router_entry, record) for record in firewall_filter_records] - firewall_filter_metrics = BaseCollector.counter_collector('firewall_filter', 'Total amount of bytes matched by firewall rules', metris_records, 'bytes', ['name', 'log']) + metrics_records = [FirewallCollector.metric_record(router_entry, record) for record in firewall_filter_records] + firewall_filter_metrics = BaseCollector.counter_collector('firewall_filter', 'Total amount of bytes matched by firewall rules', metrics_records, 'bytes', ['name', 'log']) yield firewall_filter_metrics - firewall_raw_records = FirewallMetricsDataSource.metric_records(router_entry, metric_labels = firewall_labels, raw = True) + firewall_raw_records = FirewallMetricsDataSource.metric_records_ipv4(router_entry, metric_labels = firewall_labels, raw = True) if firewall_raw_records: - metris_records = [FirewallCollector.metric_record(router_entry, record) for record in firewall_raw_records] - firewall_raw_metrics = BaseCollector.counter_collector('firewall_raw', 'Total amount of bytes matched by raw firewall rules', metris_records, 'bytes', ['name', 'log']) + metrics_records = [FirewallCollector.metric_record(router_entry, record) for record in firewall_raw_records] + firewall_raw_metrics = BaseCollector.counter_collector('firewall_raw', 'Total amount of bytes matched by raw firewall rules', metrics_records, 'bytes', ['name', 'log']) yield firewall_raw_metrics + # ~*~*~*~*~*~ IPv6 ~*~*~*~*~*~ + firewall_filter_records_ipv6 = FirewallMetricsDataSource.metric_records_ipv6(router_entry, metric_labels = firewall_labels) + if firewall_filter_records_ipv6: + metrics_records_ipv6 = [FirewallCollector.metric_record(router_entry, record) for record in firewall_filter_records_ipv6] + firewall_filter_metrics_ipv6 = BaseCollector.counter_collector('firewall_filter_ipv6', 'Total amount of bytes matched by firewall rules (IPv6)', metrics_records_ipv6, 'bytes', ['name', 'log']) + yield firewall_filter_metrics_ipv6 + + firewall_raw_records_ipv6 = FirewallMetricsDataSource.metric_records_ipv4(router_entry, metric_labels = firewall_labels, raw = True) + if firewall_raw_records_ipv6: + metrics_records_ipv6 = [FirewallCollector.metric_record(router_entry, record) for record in firewall_raw_records_ipv6] + firewall_raw_metrics_ipv6 = BaseCollector.counter_collector('firewall_raw_ipv6', 'Total amount of bytes matched by raw firewall rules (IPv6)', metrics_records_ipv6, 'bytes', ['name', 'log']) + yield firewall_raw_metrics_ipv6 + # Helpers @staticmethod def metric_record(router_entry, firewall_record): diff --git a/mktxp/datasource/firewall_ds.py b/mktxp/datasource/firewall_ds.py index 4ac456f..f4a948c 100644 --- a/mktxp/datasource/firewall_ds.py +++ b/mktxp/datasource/firewall_ds.py @@ -1,43 +1,79 @@ # coding=utf8 -## Copyright (c) 2020 Arseniy Kuznetsov -## -## This program is free software; you can redistribute it and/or -## modify it under the terms of the GNU General Public License -## as published by the Free Software Foundation; either version 2 -## of the License, or (at your option) any later version. -## -## This program is distributed in the hope that it will be useful, -## but WITHOUT ANY WARRANTY; without even the implied warranty of -## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -## GNU General Public License for more details. +# Copyright (c) 2020 Arseniy Kuznetsov +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. from mktxp.datasource.base_ds import BaseDSProcessor +from mktxp.flow.router_entry import RouterEntry + +TRANSLATION_TABLE = { + 'comment': lambda value: value if value else '', + 'log': lambda value: '1' if value == 'true' else '0' +} class FirewallMetricsDataSource: ''' Firewall Metrics data provider - ''' + This datasource supports both IPv4 and IPv6 + ''' @staticmethod - def metric_records(router_entry, *, metric_labels = None, raw = False, matching_only = True): + def _get_records(router_entry: RouterEntry, filter_path: str, args: dict, matching_only: bool = False): + """ + Get firewall records from a Mikrotik ROS device. + :param router_entry: The ROS API entry used to connect to the API + :param filter_path: The path to query the records for (e.g. /ip/firewall/filter) + :param args: A dictionary of arguments to pass to the print function used for export. + Looks like: '{'stats': '', 'all': ''}' + """ + firewall_records = router_entry.api_connection.router_api().get_resource(filter_path).call('print', args) + if matching_only: + firewall_records = [record for record in firewall_records if int(record.get('bytes', '0')) > 0] + return firewall_records + + @staticmethod + def metric_records_ipv4(router_entry, *, metric_labels=None, raw=False, matching_only=True): if metric_labels is None: - metric_labels = [] + metric_labels = [] try: filter_path = '/ip/firewall/filter' if not raw else '/ip/firewall/raw' - firewall_records = router_entry.api_connection.router_api().get_resource(filter_path).call('print', {'stats':'', 'all':''}) - if matching_only: - firewall_records = [record for record in firewall_records if int(record.get('bytes', '0')) > 0] + firewall_records = FirewallMetricsDataSource._get_records( + router_entry, + filter_path, + {'stats': '', 'all': ''}, + matching_only=matching_only + ) - # translation rules - translation_table = {} - if 'comment' in metric_labels: - translation_table['comment'] = lambda value: value if value else '' - if 'log' in metric_labels: - translation_table['log'] = lambda value: '1' if value == 'true' else '0' - - return BaseDSProcessor.trimmed_records(router_entry, router_records = firewall_records, metric_labels = metric_labels, translation_table = translation_table) + return BaseDSProcessor.trimmed_records(router_entry, router_records=firewall_records, metric_labels=metric_labels, translation_table=TRANSLATION_TABLE) except Exception as exc: - print(f'Error getting firewall filters info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}') + print( + f'Error getting firewall filters info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}' + ) return None + @staticmethod + def metric_records_ipv6(router_entry, metric_labels=None, raw=False, matching_only=True): + metric_labels = metric_labels or [] + try: + filter_path = '/ipv6/firewall/filter' if not raw else '/ip/firewall/raw' + firewall_records = FirewallMetricsDataSource._get_records( + router_entry, + filter_path, + {'stats': ''}, + matching_only=matching_only + ) + return BaseDSProcessor.trimmed_records(router_entry, router_records=firewall_records, metric_labels=metric_labels, translation_table=TRANSLATION_TABLE) + except Exception as exc: + print( + f'Error getting IPv6 firewall filters info from router{router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}' + ) + return None From 70e5a64469f542f30d81260ec876b2f6dc5a79dc Mon Sep 17 00:00:00 2001 From: Leon Morten Richter Date: Sun, 13 Nov 2022 13:36:58 +0100 Subject: [PATCH 2/4] add IPv6 reachable addresses --- mktxp/cli/config/config.py | 5 +-- mktxp/cli/config/mktxp.conf | 1 + mktxp/collector/ipv6_neighbor_collector.py | 40 ++++++++++++++++++++++ mktxp/datasource/ipv6_neighbor_ds.py | 23 +++++++++++++ mktxp/flow/collector_registry.py | 3 ++ mktxp/flow/router_entry.py | 1 + 6 files changed, 71 insertions(+), 2 deletions(-) create mode 100644 mktxp/collector/ipv6_neighbor_collector.py create mode 100644 mktxp/datasource/ipv6_neighbor_ds.py diff --git a/mktxp/cli/config/config.py b/mktxp/cli/config/config.py index dfa1ddf..e316807 100755 --- a/mktxp/cli/config/config.py +++ b/mktxp/cli/config/config.py @@ -50,6 +50,7 @@ class MKTXPConfigKeys: FE_CAPSMAN_CLIENTS_KEY = 'capsman_clients' FE_POE_KEY = 'poe' FE_PUBLIC_IP_KEY = 'public_ip' + FE_IPV6_NEIGHBOR_KEY = 'ipv6_neighbor' FE_NETWATCH_KEY = 'netwatch' MKTXP_SOCKET_TIMEOUT = 'socket_timeout' @@ -86,7 +87,7 @@ class MKTXPConfigKeys: BOOLEAN_KEYS_YES = {FE_DHCP_KEY, FE_DHCP_LEASE_KEY, FE_DHCP_POOL_KEY, FE_IP_CONNECTIONS_KEY, FE_INTERFACE_KEY, FE_FIREWALL_KEY, FE_MONITOR_KEY, FE_ROUTE_KEY, MKTXP_USE_COMMENTS_OVER_NAMES, FE_WIRELESS_KEY, FE_WIRELESS_CLIENTS_KEY, FE_CAPSMAN_KEY, FE_CAPSMAN_CLIENTS_KEY, FE_POE_KEY, - FE_NETWATCH_KEY, FE_PUBLIC_IP_KEY} + FE_NETWATCH_KEY, FE_PUBLIC_IP_KEY, FE_IPV6_NEIGHBOR_KEY} SYSTEM_BOOLEAN_KEYS_YES = {MKTXP_BANDWIDTH_KEY} SYSTEM_BOOLEAN_KEYS_NO = {MKTXP_VERBOSE_MODE} @@ -106,7 +107,7 @@ class ConfigEntry: MKTXPConfigKeys.FE_DHCP_KEY, MKTXPConfigKeys.FE_DHCP_LEASE_KEY, MKTXPConfigKeys.FE_DHCP_POOL_KEY, MKTXPConfigKeys.FE_INTERFACE_KEY, MKTXPConfigKeys.FE_FIREWALL_KEY, MKTXPConfigKeys.FE_MONITOR_KEY, MKTXPConfigKeys.FE_ROUTE_KEY, MKTXPConfigKeys.FE_WIRELESS_KEY, MKTXPConfigKeys.FE_WIRELESS_CLIENTS_KEY, MKTXPConfigKeys.FE_IP_CONNECTIONS_KEY, MKTXPConfigKeys.FE_CAPSMAN_KEY, MKTXPConfigKeys.FE_CAPSMAN_CLIENTS_KEY, MKTXPConfigKeys.FE_POE_KEY, MKTXPConfigKeys.FE_NETWATCH_KEY, MKTXPConfigKeys.MKTXP_USE_COMMENTS_OVER_NAMES, - MKTXPConfigKeys.FE_PUBLIC_IP_KEY, + MKTXPConfigKeys.FE_PUBLIC_IP_KEY, MKTXPConfigKeys.FE_IPV6_NEIGHBOR_KEY ]) MKTXPSystemEntry = namedtuple('MKTXPSystemEntry', [MKTXPConfigKeys.PORT_KEY, MKTXPConfigKeys.MKTXP_SOCKET_TIMEOUT, MKTXPConfigKeys.MKTXP_INITIAL_DELAY, MKTXPConfigKeys.MKTXP_MAX_DELAY, diff --git a/mktxp/cli/config/mktxp.conf b/mktxp/cli/config/mktxp.conf index f8501d3..1f768d6 100644 --- a/mktxp/cli/config/mktxp.conf +++ b/mktxp/cli/config/mktxp.conf @@ -33,6 +33,7 @@ monitor = True # Interface monitor metrics poe = True # POE metrics public_ip = True # Public IP metrics + ipv6_neighbor = False # Reachable IPv6 Neighbors route = True # Routes metrics wireless = True # WLAN general metrics wireless_clients = True # WLAN clients metrics diff --git a/mktxp/collector/ipv6_neighbor_collector.py b/mktxp/collector/ipv6_neighbor_collector.py new file mode 100644 index 0000000..3d1d729 --- /dev/null +++ b/mktxp/collector/ipv6_neighbor_collector.py @@ -0,0 +1,40 @@ +# coding=utf8 +# Copyright (c) 2020 Arseniy Kuznetsov +## +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +## +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + + +from mktxp.collector.base_collector import BaseCollector +from mktxp.datasource.ipv6_neighbor_ds import IPv6NeighborDataSource + + +class IPv6NeighborCollector(BaseCollector): + '''IPv6 Neighbor Collector''' + @staticmethod + def collect(router_entry): + if not router_entry.config_entry.ipv6_neighbor: + return + + metric_labels = ['address', 'interface', 'mac_address', 'status'] + + records = IPv6NeighborDataSource.metric_records( + router_entry, + metric_labels=metric_labels + ) + + metrics = BaseCollector.gauge_collector( + 'ipv6_neighbor_info', + 'Reachable IPv6 neighbors', + records, + 'ipv6_neighbor', + metric_labels=metric_labels + ) + yield metrics diff --git a/mktxp/datasource/ipv6_neighbor_ds.py b/mktxp/datasource/ipv6_neighbor_ds.py new file mode 100644 index 0000000..1e764d0 --- /dev/null +++ b/mktxp/datasource/ipv6_neighbor_ds.py @@ -0,0 +1,23 @@ +# coding=utf8 +# Copyright (c) 2020 Arseniy Kuznetsov +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + + +from mktxp.datasource.base_ds import BaseDSProcessor + + +class IPv6NeighborDataSource: + + def metric_records(router_entry, metric_labels): + metric_labels = metric_labels or [] + records = router_entry.api_connection.router_api().get_resource('/ipv6/neighbor').get(status='reachable') + return BaseDSProcessor.trimmed_records(router_entry, router_records=records, metric_labels=metric_labels, ) diff --git a/mktxp/flow/collector_registry.py b/mktxp/flow/collector_registry.py index 41e9bdc..8f5ae75 100644 --- a/mktxp/flow/collector_registry.py +++ b/mktxp/flow/collector_registry.py @@ -19,6 +19,7 @@ from mktxp.collector.interface_collector import InterfaceCollector from mktxp.collector.health_collector import HealthCollector from mktxp.collector.identity_collector import IdentityCollector from mktxp.collector.public_ip_collector import PublicIPAddressCollector +from mktxp.collector.ipv6_neighbor_collector import IPv6NeighborCollector from mktxp.collector.monitor_collector import MonitorCollector from mktxp.collector.poe_collector import POECollector from mktxp.collector.netwatch_collector import NetwatchCollector @@ -46,6 +47,8 @@ class CollectorRegistry: self.register('HealthCollector', HealthCollector.collect) self.register('PublicIPAddressCollector', PublicIPAddressCollector.collect) + self.register('IPv6NeighborCollector', IPv6NeighborCollector.collect) + self.register('DHCPCollector', DHCPCollector.collect) self.register('IPConnectionCollector', IPConnectionCollector.collect) self.register('PoolCollector', PoolCollector.collect) diff --git a/mktxp/flow/router_entry.py b/mktxp/flow/router_entry.py index ac85741..051f048 100644 --- a/mktxp/flow/router_entry.py +++ b/mktxp/flow/router_entry.py @@ -31,6 +31,7 @@ class RouterEntry: 'SystemResourceCollector': 0, 'HealthCollector': 0, 'PublicIPAddressCollector': 0, + 'IPv6NeighborCollector': 0, 'DHCPCollector': 0, 'PoolCollector': 0, 'IPConnectionCollector': 0, From 2122c17ce3bd8528f06c3b1f4e130dde87bbe78f Mon Sep 17 00:00:00 2001 From: Leon Morten Richter Date: Sun, 13 Nov 2022 14:44:07 +0100 Subject: [PATCH 3/4] fix typo --- mktxp/collector/firewall_collector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mktxp/collector/firewall_collector.py b/mktxp/collector/firewall_collector.py index 19e9bbd..6d9951f 100644 --- a/mktxp/collector/firewall_collector.py +++ b/mktxp/collector/firewall_collector.py @@ -49,7 +49,7 @@ class FirewallCollector(BaseCollector): firewall_filter_metrics_ipv6 = BaseCollector.counter_collector('firewall_filter_ipv6', 'Total amount of bytes matched by firewall rules (IPv6)', metrics_records_ipv6, 'bytes', ['name', 'log']) yield firewall_filter_metrics_ipv6 - firewall_raw_records_ipv6 = FirewallMetricsDataSource.metric_records_ipv4(router_entry, metric_labels = firewall_labels, raw = True) + firewall_raw_records_ipv6 = FirewallMetricsDataSource.metric_records_ipv6(router_entry, metric_labels = firewall_labels, raw = True) if firewall_raw_records_ipv6: metrics_records_ipv6 = [FirewallCollector.metric_record(router_entry, record) for record in firewall_raw_records_ipv6] firewall_raw_metrics_ipv6 = BaseCollector.counter_collector('firewall_raw_ipv6', 'Total amount of bytes matched by raw firewall rules (IPv6)', metrics_records_ipv6, 'bytes', ['name', 'log']) From 4888597c96b92f61c1e1291104f1e442d8a207ad Mon Sep 17 00:00:00 2001 From: Leon Morten Richter Date: Sun, 13 Nov 2022 14:48:45 +0100 Subject: [PATCH 4/4] fix correct filter path for IPv6 firewall --- mktxp/datasource/firewall_ds.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mktxp/datasource/firewall_ds.py b/mktxp/datasource/firewall_ds.py index f4a948c..e38d8c1 100644 --- a/mktxp/datasource/firewall_ds.py +++ b/mktxp/datasource/firewall_ds.py @@ -63,7 +63,7 @@ class FirewallMetricsDataSource: def metric_records_ipv6(router_entry, metric_labels=None, raw=False, matching_only=True): metric_labels = metric_labels or [] try: - filter_path = '/ipv6/firewall/filter' if not raw else '/ip/firewall/raw' + filter_path = '/ipv6/firewall/filter' if not raw else '/ipv6/firewall/raw' firewall_records = FirewallMetricsDataSource._get_records( router_entry, filter_path,