diff --git a/CHANGES.md b/CHANGES.md index 232cea5..157f028 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] ### Added - Added `--no-remove-original-file` ([@NightMachinary](https://github.com/NightMachinary)) (#580) +- Added leading Zeros in `track_number` for correct sorting ([@Dsujan](https://github.com/Dsujan)) (#592) ### Fixed - diff --git a/setup.py b/setup.py index e36db14..ec2150c 100755 --- a/setup.py +++ b/setup.py @@ -10,11 +10,7 @@ setup( name="spotdl", # Tests are included automatically: # https://docs.python.org/3.6/distutils/sourcedist.html#specifying-the-files-to-distribute - packages=[ - "spotdl", - "spotdl.lyrics", - "spotdl.lyrics.providers", - ], + packages=["spotdl", "spotdl.lyrics", "spotdl.lyrics.providers"], version=spotdl.__version__, install_requires=[ "pathlib >= 1.0.1", diff --git a/spotdl/convert.py b/spotdl/convert.py index e84bc7a..70292ed 100644 --- a/spotdl/convert.py +++ b/spotdl/convert.py @@ -16,7 +16,14 @@ https://trac.ffmpeg.org/wiki/Encode/AAC """ -def song(input_song, output_song, folder, avconv=False, trim_silence=False, delete_original=True): +def song( + input_song, + output_song, + folder, + avconv=False, + trim_silence=False, + delete_original=True, +): """ Do the audio format conversion. """ if avconv and trim_silence: raise ValueError("avconv does not support trim_silence") @@ -28,7 +35,9 @@ def song(input_song, output_song, folder, avconv=False, trim_silence=False, dele else: return 0 - convert = Converter(input_song, output_song, folder, delete_original=delete_original) + convert = Converter( + input_song, output_song, folder, delete_original=delete_original + ) if avconv: exit_code, command = convert.with_avconv() else: @@ -97,7 +106,9 @@ class Converter: return code, command def with_ffmpeg(self, trim_silence=False): - ffmpeg_pre = "ffmpeg -y -nostdin " # -nostdin is necessary for spotdl to be able to run in the backgroung. + ffmpeg_pre = ( + "ffmpeg -y -nostdin " + ) # -nostdin is necessary for spotdl to be able to run in the backgroung. if not log.level == 10: ffmpeg_pre += "-hide_banner -nostats -v panic " diff --git a/spotdl/downloader.py b/spotdl/downloader.py index bd0f1ab..beae9f2 100644 --- a/spotdl/downloader.py +++ b/spotdl/downloader.py @@ -96,6 +96,7 @@ class Downloader: self.raw_song = raw_song self.number = number self.content, self.meta_tags = youtube_tools.match_video_and_metadata(raw_song) + self.total_songs = int(self.meta_tags["total_tracks"]) def download_single(self): """ Logic behind downloading a song. """ @@ -158,7 +159,10 @@ class Downloader: def refine_songname(self, songname): if self.meta_tags is not None: refined_songname = internals.format_string( - const.args.file_format, self.meta_tags, slugification=True + const.args.file_format, + self.meta_tags, + slugification=True, + total_songs=self.total_songs, ) log.debug( 'Refining songname from "{0}" to "{1}"'.format( diff --git a/spotdl/handle.py b/spotdl/handle.py index 57926f1..7012ac5 100644 --- a/spotdl/handle.py +++ b/spotdl/handle.py @@ -37,7 +37,7 @@ default_conf = { "write-successful": None, "log-level": "INFO", "spotify_client_id": "4fe3fecfe5334023a1472516cc99d805", - "spotify_client_secret": "0f02b7c483c04257984695007a4a8d5c" + "spotify_client_secret": "0f02b7c483c04257984695007a4a8d5c", } } @@ -280,13 +280,13 @@ def get_arguments(raw_args=None, to_group=True, to_merge=True): "-sci", "--spotify-client-id", default=config["spotify_client_id"], - help=argparse.SUPPRESS + help=argparse.SUPPRESS, ) parser.add_argument( "-scs", "--spotify-client-secret", default=config["spotify_client_secret"], - help=argparse.SUPPRESS + help=argparse.SUPPRESS, ) parser.add_argument( "-c", "--config", default=None, help="path to custom config.yml file" @@ -320,11 +320,12 @@ def get_arguments(raw_args=None, to_group=True, to_merge=True): if parsed.avconv and parsed.trim_silence: parser.error("--trim-silence can only be used with FFmpeg") - if parsed.write_to and not (parsed.playlist \ - or parsed.album \ - or parsed.all_albums \ - or parsed.username): - parser.error("--write-to can only be used with --playlist, --album, --all-albums, or --username") + if parsed.write_to and not ( + parsed.playlist or parsed.album or parsed.all_albums or parsed.username + ): + parser.error( + "--write-to can only be used with --playlist, --album, --all-albums, or --username" + ) parsed.log_level = log_leveller(parsed.log_level) diff --git a/spotdl/internals.py b/spotdl/internals.py index 64a3acc..8fbe120 100644 --- a/spotdl/internals.py +++ b/spotdl/internals.py @@ -1,8 +1,10 @@ from logzero import logger as log import os import sys +import math import urllib.request + from spotdl import const try: @@ -73,7 +75,9 @@ def is_youtube(raw_song): return status -def format_string(string_format, tags, slugification=False, force_spaces=False): +def format_string( + string_format, tags, slugification=False, force_spaces=False, total_songs=0 +): """ Generate a string of the format '[artist] - [song]' for the given spotify song. """ format_tags = dict(formats) format_tags[0] = tags["name"] @@ -93,9 +97,16 @@ def format_string(string_format, tags, slugification=False, force_spaces=False): k: sanitize_title(str(v), ok="'-_()[]{}") if slugification else str(v) for k, v in format_tags.items() } + # calculating total digits presnet in total_songs to prepare a zfill. + total_digits = 0 if total_songs == 0 else int(math.log10(total_songs)) + 1 for x in formats: format_tag = "{" + formats[x] + "}" + # Making consistent track number by prepending zero + # on it according to number of digits in total songs + if format_tag == "{track_number}": + format_tags_sanitized[x] = format_tags_sanitized[x].zfill(total_digits) + string_format = string_format.replace(format_tag, format_tags_sanitized[x]) if const.args.no_spaces and not force_spaces: diff --git a/spotdl/lyrics/providers/tests/test_genius.py b/spotdl/lyrics/providers/tests/test_genius.py index e033a1c..ae39d94 100644 --- a/spotdl/lyrics/providers/tests/test_genius.py +++ b/spotdl/lyrics/providers/tests/test_genius.py @@ -18,7 +18,6 @@ class TestGenius: assert track.base_url == "https://genius.com" def test_get_lyrics(self, track, monkeypatch): - def mocked_urlopen(url, timeout=None): class DummyHTTPResponse: def read(self): @@ -30,7 +29,6 @@ class TestGenius: assert track.get_lyrics() == "amazing lyrics!" def test_lyrics_not_found_error(self, track, monkeypatch): - def mocked_urlopen(url, timeout=None): raise urllib.request.HTTPError("", "", "", "", "") diff --git a/spotdl/lyrics/providers/tests/test_lyricwikia_wrapper.py b/spotdl/lyrics/providers/tests/test_lyricwikia_wrapper.py index 474a101..92142fa 100644 --- a/spotdl/lyrics/providers/tests/test_lyricwikia_wrapper.py +++ b/spotdl/lyrics/providers/tests/test_lyricwikia_wrapper.py @@ -15,17 +15,21 @@ class TestLyricWikia: # `LyricWikia` class uses the 3rd party method `lyricwikia.get_lyrics` # internally and there is no need to test a 3rd party library as they # have their own implementation of tests. - monkeypatch.setattr("lyricwikia.get_lyrics", lambda a, b, c, d: "awesome lyrics!") + monkeypatch.setattr( + "lyricwikia.get_lyrics", lambda a, b, c, d: "awesome lyrics!" + ) track = LyricWikia("Lyricwikia", "Lyricwikia") assert track.get_lyrics() == "awesome lyrics!" def test_lyrics_not_found_error(self, monkeypatch): - def lyricwikia_lyrics_not_found(msg): raise lyricwikia.LyricsNotFound(msg) # Wrap `lyricwikia.LyricsNotFound` with `exceptions.LyricsNotFound` error. - monkeypatch.setattr("lyricwikia.get_lyrics", lambda a, b, c, d: lyricwikia_lyrics_not_found("Nope, no lyrics.")) + monkeypatch.setattr( + "lyricwikia.get_lyrics", + lambda a, b, c, d: lyricwikia_lyrics_not_found("Nope, no lyrics."), + ) track = LyricWikia("Lyricwikia", "Lyricwikia") with pytest.raises(exceptions.LyricsNotFound): track.get_lyrics() diff --git a/spotdl/metadata.py b/spotdl/metadata.py index aeecd90..1623797 100644 --- a/spotdl/metadata.py +++ b/spotdl/metadata.py @@ -80,7 +80,9 @@ class EmbedMetadata: audiofile["TYER"] = TYER(encoding=3, text=meta_tags["year"]) if meta_tags["publisher"]: audiofile["TPUB"] = TPUB(encoding=3, text=meta_tags["publisher"]) - audiofile["COMM"] = COMM(encoding=3, text=meta_tags["external_urls"][self.provider]) + audiofile["COMM"] = COMM( + encoding=3, text=meta_tags["external_urls"][self.provider] + ) if meta_tags["lyrics"]: audiofile["USLT"] = USLT( encoding=3, desc=u"Lyrics", text=meta_tags["lyrics"] diff --git a/spotdl/patcher.py b/spotdl/patcher.py index d52bacf..16caa5b 100644 --- a/spotdl/patcher.py +++ b/spotdl/patcher.py @@ -11,24 +11,30 @@ def _getbestthumb(self): part_url = "https://i.ytimg.com/vi/%s/" % self.videoid # Thumbnail resolution sorted in descending order - thumbs = ("maxresdefault.jpg", - "sddefault.jpg", - "hqdefault.jpg", - "mqdefault.jpg", - "default.jpg") + thumbs = ( + "maxresdefault.jpg", + "sddefault.jpg", + "hqdefault.jpg", + "mqdefault.jpg", + "default.jpg", + ) for thumb in thumbs: url = part_url + thumb if self._content_available(url): return url + def _process_streams(self): - for format_index in range(len(self._ydl_info['formats'])): + for format_index in range(len(self._ydl_info["formats"])): try: - self._ydl_info['formats'][format_index]['url'] = self._ydl_info['formats'][format_index]['fragment_base_url'] + self._ydl_info["formats"][format_index]["url"] = self._ydl_info["formats"][ + format_index + ]["fragment_base_url"] except KeyError: pass return backend_youtube_dl.YtdlPafy._old_process_streams(self) + @classmethod def _content_available(cls, url): return internals.content_available(url) @@ -39,6 +45,7 @@ class PatchPafy: These patches have not been released by pafy on PyPI yet but are useful to us. """ + def patch_getbestthumb(self): # https://github.com/mps-youtube/pafy/pull/211 pafy.backend_shared.BasePafy._bestthumb = None @@ -47,7 +54,9 @@ class PatchPafy: def patch_process_streams(self): # https://github.com/mps-youtube/pafy/pull/230 - backend_youtube_dl.YtdlPafy._old_process_streams = backend_youtube_dl.YtdlPafy._process_streams + backend_youtube_dl.YtdlPafy._old_process_streams = ( + backend_youtube_dl.YtdlPafy._process_streams + ) backend_youtube_dl.YtdlPafy._process_streams = _process_streams def patch_insecure_streams(self): diff --git a/spotdl/spotdl.py b/spotdl/spotdl.py index 71ff564..569ef63 100644 --- a/spotdl/spotdl.py +++ b/spotdl/spotdl.py @@ -28,8 +28,9 @@ def match_args(): track_dl.download_single() elif const.args.list: if const.args.write_m3u: - youtube_tools.generate_m3u(track_file=const.args.list, - text_file=const.args.write_to) + youtube_tools.generate_m3u( + track_file=const.args.list, text_file=const.args.write_to + ) else: list_dl = downloader.ListDownloader( tracks_file=const.args.list, @@ -38,17 +39,21 @@ def match_args(): ) list_dl.download_list() elif const.args.playlist: - spotify_tools.write_playlist(playlist_url=const.args.playlist, - text_file=const.args.write_to) + spotify_tools.write_playlist( + playlist_url=const.args.playlist, text_file=const.args.write_to + ) elif const.args.album: - spotify_tools.write_album(album_url=const.args.album, - text_file=const.args.write_to) + spotify_tools.write_album( + album_url=const.args.album, text_file=const.args.write_to + ) elif const.args.all_albums: - spotify_tools.write_all_albums_from_artist(artist_url=const.args.all_albums, - text_file=const.args.write_to) + spotify_tools.write_all_albums_from_artist( + artist_url=const.args.all_albums, text_file=const.args.write_to + ) elif const.args.username: - spotify_tools.write_user_playlist(username=const.args.username, - text_file=const.args.write_to) + spotify_tools.write_user_playlist( + username=const.args.username, text_file=const.args.write_to + ) def main(): diff --git a/spotdl/spotify_tools.py b/spotdl/spotify_tools.py index 0f62a4f..07d974e 100644 --- a/spotdl/spotify_tools.py +++ b/spotdl/spotify_tools.py @@ -17,8 +17,6 @@ from spotdl.lyrics.exceptions import LyricsNotFound spotify = None - - def generate_token(): """ Generate the token. """ credentials = oauth2.SpotifyClientCredentials( @@ -39,6 +37,7 @@ def must_be_authorized(func, spotify=spotify): token = generate_token() spotify = spotipy.Spotify(auth=token) return func(*args, **kwargs) + return wrapper diff --git a/spotdl/youtube_tools.py b/spotdl/youtube_tools.py index ff8fd76..c597132 100644 --- a/spotdl/youtube_tools.py +++ b/spotdl/youtube_tools.py @@ -18,6 +18,7 @@ pafy.g.opener.addheaders.append(("Range", "bytes=0-")) # More info: https://github.com/mps-youtube/pafy/pull/211 if pafy.__version__ <= "0.5.4": from spotdl import patcher + pafy_patcher = patcher.PatchPafy() pafy_patcher.patch_getbestthumb() pafy_patcher.patch_process_streams() @@ -52,10 +53,13 @@ def match_video_and_metadata(track): """ Get and match track data from YouTube and Spotify. """ meta_tags = None - def fallback_metadata(meta_tags): - fallback_metadata_info = "Track not found on Spotify, falling back on YouTube metadata" - skip_fallback_metadata_warning = "Fallback condition not met, shall not embed metadata" + fallback_metadata_info = ( + "Track not found on Spotify, falling back on YouTube metadata" + ) + skip_fallback_metadata_warning = ( + "Fallback condition not met, shall not embed metadata" + ) if meta_tags is None: if const.args.no_fallback_metadata: log.warning(skip_fallback_metadata_warning) @@ -64,7 +68,6 @@ def match_video_and_metadata(track): meta_tags = generate_metadata(content) return meta_tags - if internals.is_youtube(track): log.debug("Input song is a YouTube URL") content = go_pafy(track, meta_tags=None) @@ -95,25 +98,29 @@ def match_video_and_metadata(track): def generate_metadata(content): """ Fetch a song's metadata from YouTube. """ - meta_tags = {"spotify_metadata": False, - "name": content.title, - "artists": [{"name": content.author}], - "duration": content.length, - "external_urls": {"youtube": content.watchv_url}, - "album": {"images" : [{"url": content.getbestthumb()}], - "artists": [{"name": None}],"name": None}, - "year": content.published.split("-")[0], - "release_date": content.published.split(" ")[0], - "type": "track", - "disc_number": 1, - "track_number": 1, - "total_tracks": 1, - "publisher": None, - "external_ids": {"isrc": None}, - "lyrics": None, - "copyright": None, - "genre": None, - } + meta_tags = { + "spotify_metadata": False, + "name": content.title, + "artists": [{"name": content.author}], + "duration": content.length, + "external_urls": {"youtube": content.watchv_url}, + "album": { + "images": [{"url": content.getbestthumb()}], + "artists": [{"name": None}], + "name": None, + }, + "year": content.published.split("-")[0], + "release_date": content.published.split(" ")[0], + "type": "track", + "disc_number": 1, + "track_number": 1, + "total_tracks": 1, + "publisher": None, + "external_ids": {"isrc": None}, + "lyrics": None, + "copyright": None, + "genre": None, + } return meta_tags diff --git a/test/loader.py b/test/loader.py index 79c9771..c8d3ffa 100644 --- a/test/loader.py +++ b/test/loader.py @@ -20,7 +20,7 @@ def load_defaults(): # so that we get same results even if YouTube changes the list/order of videos on their page. GIST_URL = "https://gist.githubusercontent.com/ritiek/e731338e9810e31c2f00f13c249a45f5/raw/c11a27f3b5d11a8d082976f1cdd237bd605ec2c2/search_results.html" + def monkeypatch_youtube_search_page(*args, **kwargs): fake_urlopen = urllib.request.urlopen(GIST_URL) return fake_urlopen - diff --git a/test/test_download_with_metadata.py b/test/test_download_with_metadata.py index 58a9ee7..0e81e41 100644 --- a/test/test_download_with_metadata.py +++ b/test/test_download_with_metadata.py @@ -101,16 +101,28 @@ class TestDownload: def test_m4a(self, monkeypatch, filename_fixture): expect_download = True - monkeypatch.setattr("pafy.backend_shared.BaseStream.download", self.blank_audio_generator) - monkeypatch.setattr("pafy.backend_youtube_dl.YtdlStream.download", self.blank_audio_generator) - download = youtube_tools.download_song(filename_fixture + ".m4a", pytest.content_fixture) + monkeypatch.setattr( + "pafy.backend_shared.BaseStream.download", self.blank_audio_generator + ) + monkeypatch.setattr( + "pafy.backend_youtube_dl.YtdlStream.download", self.blank_audio_generator + ) + download = youtube_tools.download_song( + filename_fixture + ".m4a", pytest.content_fixture + ) assert download == expect_download def test_webm(self, monkeypatch, filename_fixture): expect_download = True - monkeypatch.setattr("pafy.backend_shared.BaseStream.download", self.blank_audio_generator) - monkeypatch.setattr("pafy.backend_youtube_dl.YtdlStream.download", self.blank_audio_generator) - download = youtube_tools.download_song(filename_fixture + ".webm", pytest.content_fixture) + monkeypatch.setattr( + "pafy.backend_shared.BaseStream.download", self.blank_audio_generator + ) + monkeypatch.setattr( + "pafy.backend_youtube_dl.YtdlStream.download", self.blank_audio_generator + ) + download = youtube_tools.download_song( + filename_fixture + ".webm", pytest.content_fixture + ) assert download == expect_download diff --git a/test/test_patcher.py b/test/test_patcher.py index 3c9ba65..a32facb 100644 --- a/test/test_patcher.py +++ b/test/test_patcher.py @@ -6,6 +6,7 @@ import pytest pafy_patcher = patcher.PatchPafy() pafy_patcher.patch_getbestthumb() + class TestPafyContentAvailable: pass @@ -30,7 +31,6 @@ class TestMethodCalls: thumbnail = patcher._getbestthumb(content_fixture) assert thumbnail == "https://i.ytimg.com/vi/3nQNiWdeH2Q/sddefault.jpg" - def test_pafy_content_available(self): TestPafyContentAvailable._content_available = patcher._content_available assert TestPafyContentAvailable()._content_available("https://youtube.com/") diff --git a/test/test_youtube_tools.py b/test/test_youtube_tools.py index 364760e..e32423c 100644 --- a/test/test_youtube_tools.py +++ b/test/test_youtube_tools.py @@ -104,19 +104,19 @@ def content_fixture(metadata_fixture): MATCH_METADATA_NO_FALLBACK_TEST_TABLE = [ ("https://open.spotify.com/track/5nWduGwBGBn1PSqYTJUDbS", True), ("http://youtube.com/watch?v=3nQNiWdeH2Q", None), - ("Linux Talk | Working with Drives and Filesystems", None) + ("Linux Talk | Working with Drives and Filesystems", None), ] MATCH_METADATA_FALLBACK_TEST_TABLE = [ ("https://open.spotify.com/track/5nWduGwBGBn1PSqYTJUDbS", True), ("http://youtube.com/watch?v=3nQNiWdeH2Q", False), - ("Linux Talk | Working with Drives and Filesystems", False) + ("Linux Talk | Working with Drives and Filesystems", False), ] MATCH_METADATA_NO_METADATA_TEST_TABLE = [ ("https://open.spotify.com/track/5nWduGwBGBn1PSqYTJUDbS", None), ("http://youtube.com/watch?v=3nQNiWdeH2Q", None), - ("Linux Talk | Working with Drives and Filesystems", None) + ("Linux Talk | Working with Drives and Filesystems", None), ] @@ -128,21 +128,37 @@ class TestMetadataOrigin: else: assert metadata["spotify_metadata"] == metadata_type - @pytest.mark.parametrize("track, metadata_type", MATCH_METADATA_NO_FALLBACK_TEST_TABLE) - def test_match_metadata_with_no_fallback(self, track, metadata_type, content_fixture, monkeypatch): - monkeypatch.setattr(youtube_tools, "go_pafy", lambda track, meta_tags: content_fixture) + @pytest.mark.parametrize( + "track, metadata_type", MATCH_METADATA_NO_FALLBACK_TEST_TABLE + ) + def test_match_metadata_with_no_fallback( + self, track, metadata_type, content_fixture, monkeypatch + ): + monkeypatch.setattr( + youtube_tools, "go_pafy", lambda track, meta_tags: content_fixture + ) const.args.no_fallback_metadata = True self.match_metadata(track, metadata_type) @pytest.mark.parametrize("track, metadata_type", MATCH_METADATA_FALLBACK_TEST_TABLE) - def test_match_metadata_with_fallback(self, track, metadata_type, content_fixture, monkeypatch): - monkeypatch.setattr(youtube_tools, "go_pafy", lambda track, meta_tags: content_fixture) + def test_match_metadata_with_fallback( + self, track, metadata_type, content_fixture, monkeypatch + ): + monkeypatch.setattr( + youtube_tools, "go_pafy", lambda track, meta_tags: content_fixture + ) const.args.no_fallback_metadata = False self.match_metadata(track, metadata_type) - @pytest.mark.parametrize("track, metadata_type", MATCH_METADATA_NO_METADATA_TEST_TABLE) - def test_match_metadata_with_no_metadata(self, track, metadata_type, content_fixture, monkeypatch): - monkeypatch.setattr(youtube_tools, "go_pafy", lambda track, meta_tags: content_fixture) + @pytest.mark.parametrize( + "track, metadata_type", MATCH_METADATA_NO_METADATA_TEST_TABLE + ) + def test_match_metadata_with_no_metadata( + self, track, metadata_type, content_fixture, monkeypatch + ): + monkeypatch.setattr( + youtube_tools, "go_pafy", lambda track, meta_tags: content_fixture + ) const.args.no_metadata = True self.match_metadata(track, metadata_type) @@ -185,7 +201,11 @@ def test_check_exists(metadata_fixture, filename_fixture, tmpdir): def test_generate_m3u(tmpdir, monkeypatch): - monkeypatch.setattr(youtube_tools.GenerateYouTubeURL, "_fetch_response", loader.monkeypatch_youtube_search_page) + monkeypatch.setattr( + youtube_tools.GenerateYouTubeURL, + "_fetch_response", + loader.monkeypatch_youtube_search_page, + ) expect_m3u = ( "#EXTM3U\n\n" "#EXTINF:208,Janji - Heroes Tonight (feat. Johnning) [NCS Release]\n"