From 9a088ee26d7a12d8ab62f2b4de899e2d3ada43dd Mon Sep 17 00:00:00 2001 From: Ritiek Malhotra Date: Sun, 12 Apr 2020 17:09:15 +0530 Subject: [PATCH] A bit of thread refactoring --- spotdl/command_line/arguments.py | 25 ++++++++-------- spotdl/command_line/helpers.py | 49 +++++++++++++++----------------- spotdl/config.py | 1 - spotdl/track.py | 29 ++++++++----------- spotdl/util.py | 22 ++++++++++++++ 5 files changed, 69 insertions(+), 57 deletions(-) diff --git a/spotdl/command_line/arguments.py b/spotdl/command_line/arguments.py index 92c2aaa..a8952c3 100644 --- a/spotdl/command_line/arguments.py +++ b/spotdl/command_line/arguments.py @@ -68,31 +68,37 @@ def get_arguments(argv=None, to_merge=True): group.add_argument( "-l", "--list", - help="download tracks from a file" + help="download tracks from a file (WARNING: this file will be modified!)" ) group.add_argument( "-p", "--playlist", - help="load tracks from playlist URL into .txt", + help="load tracks from playlist URL into .txt or " + "if `--write-to=` has been passed", ) group.add_argument( - "-b", "--album", help="load tracks from album URL into .txt" + "-b" + "--album", + help="load tracks from album URL into .txt or if " + "`--write-to=` has been passed" ) group.add_argument( "-ab", "--all-albums", - help="load all tracks from artist URL into .txt", + help="load all tracks from artist URL into .txt " + "or if `--write-to=` has been passed" ) group.add_argument( "-u", "--username", - help="load tracks from user's playlist into .txt", + help="load tracks from user's playlist into .txt " + "or if `--write-to=` has been passed" ) parser.add_argument( "--write-m3u", help="generate an .m3u playlist file with youtube links given " - "a text file containing tracks", + "a text file containing tracks", action="store_true", ) parser.add_argument( @@ -102,13 +108,6 @@ def get_arguments(argv=None, to_merge=True): help="choose the track to download manually from a list of matching tracks", action="store_true", ) - parser.add_argument( - "-nr", - "--no-remove-original", - default=config["no-remove-original"], - help="do not remove the original file after conversion", - action="store_true", - ) parser.add_argument( "-nm", "--no-metadata", diff --git a/spotdl/command_line/helpers.py b/spotdl/command_line/helpers.py index a90e8a8..884aeb0 100644 --- a/spotdl/command_line/helpers.py +++ b/spotdl/command_line/helpers.py @@ -1,7 +1,10 @@ from spotdl.metadata.providers import ProviderSpotify from spotdl.metadata.providers import ProviderYouTube from spotdl.metadata.embedders import EmbedderDefault -from spotdl.encode.encoders import EncoderFFmpeg, EncoderAvconv + +from spotdl.encode.encoders import EncoderFFmpeg +from spotdl.encode.encoders import EncoderAvconv + from spotdl.lyrics.providers import LyricWikia from spotdl.lyrics.providers import Genius @@ -11,7 +14,6 @@ import spotdl.util import os import urllib.request -import threading def search_metadata(track, lyrics=True, search_format="{artist} - {track-name}"): @@ -46,7 +48,7 @@ def download_track(track, arguments): def download_track_from_metadata(metadata, arguments): track = Track(metadata, cache_albumart=(not arguments.no_metadata)) - # log.info(log_fmt) + print(metadata["name"]) stream = metadata["streams"].get( quality=arguments.quality, @@ -114,45 +116,41 @@ def download_tracks_from_file(path, arguments): # Remove duplicates and empty elements # Also strip whitespaces from elements (if any) - spotdl.util.remove_duplicates(tracks, condition=lambda x: x, operation=str.strip) + spotdl.util.remove_duplicates( + tracks, + condition=lambda x: x, + operation=str.strip + ) # Overwrite file with open(path, "w") as fout: fout.writelines(tracks) - next_track_metadata = threading.Thread(target=lambda: None) - next_track_metadata.start() tracks_count = len(tracks) current_iteration = 1 - def mutable_assignment(mutable_resource, track): - mutable_resource["next_track"] = search_metadata( - track, - search_format=arguments.search_format - ) - + next_track = tracks.pop(0) metadata = { "current_track": None, - "next_track": None, + "next_track": spotdl.util.ThreadWithReturnValue( + target=search_metadata, + args=(next_track, True, arguments.search_format) + ) } + metadata["next_track"].start() while tracks_count > 0: - current_track = tracks.pop(0) tracks_count -= 1 - metadata["current_track"] = metadata["next_track"] + metadata["current_track"] = metadata["next_track"].join() metadata["next_track"] = None try: - if metadata["current_track"] is None: - metadata["current_track"] = search_metadata( - current_track, - search_format=arguments.search_format - ) if tracks_count > 0: - next_track = tracks[0] - next_track_metadata = threading.Thread( - target=mutable_assignment, - args=(metadata, next_track) + current_track = next_track + next_track = tracks.pop(0) + metadata["next_track"] = spotdl.util.ThreadWithReturnValue( + target=search_metadata, + args=(next_track, True, arguments.search_format) ) - next_track_metadata.start() + metadata["next_track"].start() log_fmt=(str(current_iteration) + ". {artist} - {track-name}") # log.info(log_fmt) @@ -161,7 +159,6 @@ def download_tracks_from_file(path, arguments): arguments ) current_iteration += 1 - next_track_metadata.join() except (urllib.request.URLError, TypeError, IOError) as e: # log.exception(e.args[0]) # log.warning("Failed. Will retry after other songs\n") diff --git a/spotdl/config.py b/spotdl/config.py index 8944deb..7ae609b 100644 --- a/spotdl/config.py +++ b/spotdl/config.py @@ -6,7 +6,6 @@ import spotdl.util DEFAULT_CONFIGURATION = { "spotify-downloader": { - "no-remove-original": False, "manual": False, "no-metadata": False, "no-fallback-metadata": False, diff --git a/spotdl/track.py b/spotdl/track.py index e89a378..6bd6dc8 100644 --- a/spotdl/track.py +++ b/spotdl/track.py @@ -2,11 +2,12 @@ import tqdm import urllib.request import subprocess -import threading from spotdl.encode.encoders import EncoderFFmpeg from spotdl.metadata.embedders import EmbedderDefault +import spotdl.util + CHUNK_SIZE= 16 * 1024 class Track: @@ -14,25 +15,18 @@ class Track: self.metadata = metadata self._chunksize = CHUNK_SIZE - self._cache_resources = { - "albumart": {"content": None, "threadinstance": None } - } if cache_albumart: self._albumart_thread = self._cache_albumart() - def _fetch_response_content_threaded(self, mutable_resource, url): - content = urllib.request.urlopen(url).read() - mutable_resource["content"] = content + self._cache_albumart = cache_albumart def _cache_albumart(self): - # A hack to get a thread's return value - albumart_thread = threading.Thread( - target=self._fetch_response_content_threaded, - args=(self._cache_resources["albumart"], - self.metadata["album"]["images"][0]["url"]), + albumart_thread = spotdl.util.ThreadWithReturnValue( + target=lambda url: urllib.request.urlopen(url).read(), + args=(self.metadata["album"]["images"][0]["url"],) ) albumart_thread.start() - self._cache_resources["albumart"]["threadinstance"] = albumart_thread + return albumart_thread def _calculate_total_chunks(self, filesize): return (filesize // self._chunksize) + 1 @@ -79,14 +73,15 @@ class Track: process.wait() def apply_metadata(self, input_path, encoding=None, embedder=EmbedderDefault()): - albumart = self._cache_resources["albumart"] - if albumart["threadinstance"]: - albumart["threadinstance"].join() + if self._cache_albumart: + albumart = self._albumart_thread.join() + else: + albumart = None embedder.apply_metadata( input_path, self.metadata, - cached_albumart=albumart["content"], + cached_albumart=albumart, encoding=encoding, ) diff --git a/spotdl/util.py b/spotdl/util.py index 66baaf0..1af53d0 100644 --- a/spotdl/util.py +++ b/spotdl/util.py @@ -4,6 +4,8 @@ import sys import math import urllib.request +import threading + try: import winreg @@ -18,6 +20,26 @@ except ImportError: sys.exit(5) +# This has been referred from +# https://stackoverflow.com/a/6894023/6554943 +# It's because threaded functions do not return by default +# Whereas this will return the value when `join` method +# is called. +class ThreadWithReturnValue(threading.Thread): + def __init__(self, target=lambda: None, args=()): + super().__init__(target=target, args=args) + self._return = None + def run(self): + if self._target is not None: + self._return = self._target( + *self._args, + **self._kwargs + ) + def join(self, *args): + super().join(*args) + return self._return + + def merge(base, overrider): """ Override default dict with config dict. """ merger = base.copy()