From fb543562205963c429ab14381a70da2779d83a2f Mon Sep 17 00:00:00 2001 From: Leon Morten Richter Date: Sun, 28 May 2023 14:11:12 +0200 Subject: [PATCH 1/3] teach mktxp to check for updates --- main.py | 6 +++ mktxp/cli/config/config.py | 6 ++- mktxp/cli/config/mktxp.conf | 4 +- mktxp/collector/resource_collector.py | 11 +++++ mktxp/utils/utils.py | 68 ++++++++++++++++++++++++++- 5 files changed, 90 insertions(+), 5 deletions(-) create mode 100644 main.py diff --git a/main.py b/main.py new file mode 100644 index 0000000..a1b11db --- /dev/null +++ b/main.py @@ -0,0 +1,6 @@ + +from mktxp.cli.dispatch import main + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/mktxp/cli/config/config.py b/mktxp/cli/config/config.py index a759452..f7dd791 100755 --- a/mktxp/cli/config/config.py +++ b/mktxp/cli/config/config.py @@ -88,6 +88,8 @@ class MKTXPConfigKeys: FE_QUEUE_KEY = 'queue' FE_REMOTE_DHCP_ENTRY = 'remote_dhcp_entry' + FE_CHECK_FOR_UPDATES = 'check_for_updates' + MKTXP_SOCKET_TIMEOUT = 'socket_timeout' MKTXP_INITIAL_DELAY = 'initial_delay_on_failure' MKTXP_MAX_DELAY = 'max_delay_on_failure' @@ -127,7 +129,7 @@ class MKTXPConfigKeys: DEFAULT_MKTXP_TOTAL_MAX_SCRAPE_DURATION = 30 - BOOLEAN_KEYS_NO = {ENABLED_KEY, SSL_KEY, NO_SSL_CERTIFICATE, + BOOLEAN_KEYS_NO = {ENABLED_KEY, SSL_KEY, NO_SSL_CERTIFICATE, FE_CHECK_FOR_UPDATES, SSL_CERTIFICATE_VERIFY, FE_IPV6_FIREWALL_KEY, FE_IPV6_NEIGHBOR_KEY, FE_CONNECTION_STATS_KEY} # Feature keys enabled by default @@ -157,7 +159,7 @@ class ConfigEntry: MKTXPConfigKeys.FE_FIREWALL_KEY, MKTXPConfigKeys.FE_MONITOR_KEY, MKTXPConfigKeys.FE_ROUTE_KEY, MKTXPConfigKeys.FE_WIRELESS_KEY, MKTXPConfigKeys.FE_WIRELESS_CLIENTS_KEY, MKTXPConfigKeys.FE_IP_CONNECTIONS_KEY, MKTXPConfigKeys.FE_CONNECTION_STATS_KEY, MKTXPConfigKeys.FE_CAPSMAN_KEY, MKTXPConfigKeys.FE_CAPSMAN_CLIENTS_KEY, MKTXPConfigKeys.FE_POE_KEY, MKTXPConfigKeys.FE_NETWATCH_KEY, MKTXPConfigKeys.MKTXP_USE_COMMENTS_OVER_NAMES, MKTXPConfigKeys.FE_PUBLIC_IP_KEY, MKTXPConfigKeys.FE_IPV6_FIREWALL_KEY, MKTXPConfigKeys.FE_IPV6_NEIGHBOR_KEY, - MKTXPConfigKeys.FE_USER_KEY, MKTXPConfigKeys.FE_QUEUE_KEY, MKTXPConfigKeys.FE_REMOTE_DHCP_ENTRY + MKTXPConfigKeys.FE_USER_KEY, MKTXPConfigKeys.FE_QUEUE_KEY, MKTXPConfigKeys.FE_REMOTE_DHCP_ENTRY, MKTXPConfigKeys.FE_CHECK_FOR_UPDATES ]) MKTXPSystemEntry = namedtuple('MKTXPSystemEntry', [MKTXPConfigKeys.PORT_KEY, MKTXPConfigKeys.MKTXP_SOCKET_TIMEOUT, MKTXPConfigKeys.MKTXP_INITIAL_DELAY, MKTXPConfigKeys.MKTXP_MAX_DELAY, diff --git a/mktxp/cli/config/mktxp.conf b/mktxp/cli/config/mktxp.conf index 161816b..8c0829b 100644 --- a/mktxp/cli/config/mktxp.conf +++ b/mktxp/cli/config/mktxp.conf @@ -53,4 +53,6 @@ remote_dhcp_entry = None # An MKTXP entry for remote DHCP info resolution (capsman/wireless) - use_comments_over_names = True # when available, forces using comments over the interfaces names + use_comments_over_names = True # when available, forces using comments over the interfaces names + + check_for_updates = False # check for available ROS updates diff --git a/mktxp/collector/resource_collector.py b/mktxp/collector/resource_collector.py index e37fb6b..15b3d2d 100644 --- a/mktxp/collector/resource_collector.py +++ b/mktxp/collector/resource_collector.py @@ -15,6 +15,7 @@ from mktxp.collector.base_collector import BaseCollector from mktxp.flow.processor.output import BaseOutputProcessor from mktxp.datasource.system_resource_ds import SystemResourceMetricsDataSource +from mktxp.utils.utils import check_for_updates class SystemResourceCollector(BaseCollector): @@ -61,6 +62,16 @@ class SystemResourceCollector(BaseCollector): cpu_frequency_metrics = BaseCollector.gauge_collector('system_cpu_frequency', 'Current CPU frequency', resource_records, 'cpu_frequency', ['version', 'board_name', 'cpu', 'architecture_name']) yield cpu_frequency_metrics + # Check for updates + if router_entry.config_entry.check_for_updates: + for record in resource_records: + cur_version, newest_version = check_for_updates(record['version']) + record['newest_version'] = str(newest_version) + record['update_available'] = cur_version < newest_version + + update_available_metrics = BaseCollector.gauge_collector('system_update_available', 'Is there a newer version available', resource_records, 'update_available', ['newest_version',]) + yield update_available_metrics + # Helpers @staticmethod diff --git a/mktxp/utils/utils.py b/mktxp/utils/utils.py index 3144610..5512135 100755 --- a/mktxp/utils/utils.py +++ b/mktxp/utils/utils.py @@ -11,14 +11,16 @@ ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ## GNU General Public License for more details. - import os, sys, shlex, tempfile, shutil, re -import subprocess, hashlib +import subprocess, hashlib, urllib from timeit import default_timer from collections.abc import Iterable +import xml.etree.ElementTree as ET from contextlib import contextmanager from multiprocessing import Process, Event from datetime import timedelta +from pkg_resources import packaging + ''' Utilities / Helpers @@ -270,3 +272,65 @@ class Benchmark: self.time = default_timer() - self.start +# mapping of channels to RSS feeds +CHANNEL_RSS_FEED_MAPPING = { + 'development': 'https://mikrotik.com/development.rss', + 'long-term': 'https://mikrotik.com/bugfix.rss', + 'stable': 'https://mikrotik.com/current.rss', + 'testing': 'https://mikrotik.com/candidate.rss', +} + + +def get_available_updates(channel): + """Check the RSS feed for available updates for a given update channel. + This method fetches the RSS feed and yields all version from the parsed XML. + Version numbers are parsed into version.Version instances (part of setuptools).""" + rss_feed = CHANNEL_RSS_FEED_MAPPING[channel] + + with urllib.request.urlopen(rss_feed) as response: + result = response.read() + root = ET.fromstring(result) + channel = root[0] + + for child in channel: + # iterate over all updates + if child.tag == 'item': + title, _, _, _, _, _ = child + # extract and parse the version number from title + version_text = re.findall(r'[\d+\.]+', title.text)[0] + version_number = packaging.version.parse(version_text) + yield version_number + +def parse_ros_version(string): + """Parse the version returned from the /system/resource command. + Returns a tuple: (, ). + + >>> parse_ros_version('1.2.3 (stable)') + 1.2.3, stable + """ + version, channel = re.findall(r'([\d\.]+).*?([\w]+)', string)[0] + return packaging.version.parse(version), channel + +def check_for_updates(cur_version): + """Try to check if there is a newer version available. + If anything goes wrong, it returns the same version. + Returns a tuple: (, )""" + error = False + try: + cur_version, channel = parse_ros_version(cur_version) + available_versions = get_available_updates(channel) + newest_version = sorted(available_versions)[-1] + except KeyError: + print(f'unknown update channel {channel}') + error = True + except urllib.error.HTTPError as err: + print(f'update feed returned: {str(err)}') + error = True + except Exception as err: + print(f'could not check for updates, because: {str(err)}') + error = True + + if error: + return cur_version, cur_version + + return cur_version, newest_version From 7d63fcba8547fc674f9987cf43d1646e14d16897 Mon Sep 17 00:00:00 2001 From: Leon Morten Richter Date: Tue, 30 May 2023 11:07:12 +0200 Subject: [PATCH 2/3] onyl fetch available ROS version once per hour --- mktxp/utils/utils.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/mktxp/utils/utils.py b/mktxp/utils/utils.py index 5512135..ce9d9d3 100755 --- a/mktxp/utils/utils.py +++ b/mktxp/utils/utils.py @@ -11,8 +11,10 @@ ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ## GNU General Public License for more details. +from functools import lru_cache import os, sys, shlex, tempfile, shutil, re import subprocess, hashlib, urllib +import time from timeit import default_timer from collections.abc import Iterable import xml.etree.ElementTree as ET @@ -281,12 +283,21 @@ CHANNEL_RSS_FEED_MAPPING = { } -def get_available_updates(channel): +def get_ttl_hash(seconds=3600): + """Return the same value withing `seconds` time period""" + return round(time.time() / seconds) + + +@lru_cache(maxsize=5) +def get_available_updates(channel, ttl_hash=get_ttl_hash()): """Check the RSS feed for available updates for a given update channel. - This method fetches the RSS feed and yields all version from the parsed XML. + This method fetches the RSS feed and returns all version from the parsed XML. Version numbers are parsed into version.Version instances (part of setuptools).""" + del ttl_hash rss_feed = CHANNEL_RSS_FEED_MAPPING[channel] + print(f'Fetching available ROS releases from {rss_feed}') + versions = [] with urllib.request.urlopen(rss_feed) as response: result = response.read() root = ET.fromstring(result) @@ -299,7 +310,9 @@ def get_available_updates(channel): # extract and parse the version number from title version_text = re.findall(r'[\d+\.]+', title.text)[0] version_number = packaging.version.parse(version_text) - yield version_number + versions.append(version_number) + return versions + def parse_ros_version(string): """Parse the version returned from the /system/resource command. From 171e56ff20cf3400335087ffd004f6be690ded60 Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Fri, 30 Jun 2023 10:05:10 +0100 Subject: [PATCH 3/3] Delete main.py per previous question / comment, deleting this file for now,so it does not block the PR --- main.py | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 main.py diff --git a/main.py b/main.py deleted file mode 100644 index a1b11db..0000000 --- a/main.py +++ /dev/null @@ -1,6 +0,0 @@ - -from mktxp.cli.dispatch import main - - -if __name__ == '__main__': - main() \ No newline at end of file