Merge branch 'master' into feat/keep-trackid-as-songname

.travis.yml

@@ -1,6 +1,4 @@
-dist: xenial
 language: python
-sudo: required
 python:
   - "3.4"
   - "3.5"

CHANGES.md

@@ -7,10 +7,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ## [Unreleased]
 ### Added
 - Added `--no-remove-original-file` ([@NightMachinary](https://github.com/NightMachinary)) (#580)
+- Added leading Zeros in `track_number` for correct sorting ([@Dsujan](https://github.com/Dsujan)) (#592)
 - Added `track_id` key for `--file-format` parameter ([@kadaliao](https://github.com/kadaliao)) (#568)
 
 ### Fixed
--
+- Generate list error --write-m3u ([@arthurlutz](https://github.com/arthurlutz)) (#559)
 
 ### Changed
 - Fetch lyrics from Genius and fallback to LyricWikia if not found ([@ritiek](https://github.com/ritiek)) (#585)

setup.py

@@ -10,7 +10,7 @@ setup(
     name="spotdl",
     # Tests are included automatically:
     # https://docs.python.org/3.6/distutils/sourcedist.html#specifying-the-files-to-distribute
-    packages=["spotdl"],
+    packages=["spotdl", "spotdl.lyrics", "spotdl.lyrics.providers"],
     version=spotdl.__version__,
     install_requires=[
         "pathlib >= 1.0.1",

spotdl/convert.py

@@ -16,7 +16,14 @@ https://trac.ffmpeg.org/wiki/Encode/AAC
 """
 
 
-def song(input_song, output_song, folder, avconv=False, trim_silence=False, delete_original=True):
+def song(
+    input_song,
+    output_song,
+    folder,
+    avconv=False,
+    trim_silence=False,
+    delete_original=True,
+):
     """ Do the audio format conversion. """
     if avconv and trim_silence:
         raise ValueError("avconv does not support trim_silence")
@@ -28,7 +35,9 @@ def song(input_song, output_song, folder, avconv=False, trim_silence=False, dele
     else:
         return 0
 
-    convert = Converter(input_song, output_song, folder, delete_original=delete_original)
+    convert = Converter(
+        input_song, output_song, folder, delete_original=delete_original
+    )
     if avconv:
         exit_code, command = convert.with_avconv()
     else:
@@ -97,7 +106,9 @@ class Converter:
         return code, command
 
     def with_ffmpeg(self, trim_silence=False):
-        ffmpeg_pre = "ffmpeg -y -nostdin " # -nostdin is necessary for spotdl to be able to run in the backgroung.
+        ffmpeg_pre = (
+            "ffmpeg -y -nostdin "
+        )  # -nostdin is necessary for spotdl to be able to run in the backgroung.
 
         if not log.level == 10:
             ffmpeg_pre += "-hide_banner -nostats -v panic "

spotdl/downloader.py

@@ -96,6 +96,7 @@ class Downloader:
         self.raw_song = raw_song
         self.number = number
         self.content, self.meta_tags = youtube_tools.match_video_and_metadata(raw_song)
+        self.total_songs = int(self.meta_tags["total_tracks"])
 
     def download_single(self):
         """ Logic behind downloading a song. """
@@ -158,7 +159,10 @@ class Downloader:
     def refine_songname(self, songname):
         if self.meta_tags is not None:
             refined_songname = internals.format_string(
-                const.args.file_format, self.meta_tags, slugification=True
+                const.args.file_format,
+                self.meta_tags,
+                slugification=True,
+                total_songs=self.total_songs,
             )
             log.debug(
                 'Refining songname from "{0}" to "{1}"'.format(

spotdl/handle.py

@@ -37,7 +37,7 @@ default_conf = {
         "write-successful": None,
         "log-level": "INFO",
         "spotify_client_id": "4fe3fecfe5334023a1472516cc99d805",
-        "spotify_client_secret": "0f02b7c483c04257984695007a4a8d5c"
+        "spotify_client_secret": "0f02b7c483c04257984695007a4a8d5c",
     }
 }
 
@@ -280,13 +280,13 @@ def get_arguments(raw_args=None, to_group=True, to_merge=True):
         "-sci",
         "--spotify-client-id",
         default=config["spotify_client_id"],
-        help=argparse.SUPPRESS
+        help=argparse.SUPPRESS,
     )
     parser.add_argument(
         "-scs",
         "--spotify-client-secret",
         default=config["spotify_client_secret"],
-        help=argparse.SUPPRESS
+        help=argparse.SUPPRESS,
     )
     parser.add_argument(
         "-c", "--config", default=None, help="path to custom config.yml file"
@@ -320,11 +320,12 @@ def get_arguments(raw_args=None, to_group=True, to_merge=True):
     if parsed.avconv and parsed.trim_silence:
         parser.error("--trim-silence can only be used with FFmpeg")
 
-    if parsed.write_to and not (parsed.playlist \
-            or parsed.album \
-            or parsed.all_albums \
-            or parsed.username):
-        parser.error("--write-to can only be used with --playlist, --album, --all-albums, or --username")
+    if parsed.write_to and not (
+        parsed.playlist or parsed.album or parsed.all_albums or parsed.username
+    ):
+        parser.error(
+            "--write-to can only be used with --playlist, --album, --all-albums, or --username"
+        )
 
     parsed.log_level = log_leveller(parsed.log_level)
 

spotdl/internals.py

@@ -1,8 +1,10 @@
 from logzero import logger as log
 import os
 import sys
+import math
 import urllib.request
 
+
 from spotdl import const
 
 try:
@@ -74,7 +76,9 @@ def is_youtube(raw_song):
     return status
 
 
-def format_string(string_format, tags, slugification=False, force_spaces=False):
+def format_string(
+    string_format, tags, slugification=False, force_spaces=False, total_songs=0
+):
     """ Generate a string of the format '[artist] - [song]' for the given spotify song. """
     format_tags = dict(formats)
     format_tags[0] = tags["name"]
@@ -95,9 +99,16 @@ def format_string(string_format, tags, slugification=False, force_spaces=False):
         k: sanitize_title(str(v), ok="'-_()[]{}") if slugification else str(v)
         for k, v in format_tags.items()
     }
+    # calculating total digits presnet in total_songs to prepare a zfill.
+    total_digits = 0 if total_songs == 0 else int(math.log10(total_songs)) + 1
 
     for x in formats:
         format_tag = "{" + formats[x] + "}"
+        # Making consistent track number by prepending zero
+        # on it according to number of digits in total songs
+        if format_tag == "{track_number}":
+            format_tags_sanitized[x] = format_tags_sanitized[x].zfill(total_digits)
+
         string_format = string_format.replace(format_tag, format_tags_sanitized[x])
 
     if const.args.no_spaces and not force_spaces:

spotdl/lyrics/providers/tests/test_genius.py

@@ -18,7 +18,6 @@ class TestGenius:
         assert track.base_url == "https://genius.com"
 
     def test_get_lyrics(self, track, monkeypatch):
-
         def mocked_urlopen(url, timeout=None):
             class DummyHTTPResponse:
                 def read(self):
@@ -30,7 +29,6 @@ class TestGenius:
         assert track.get_lyrics() == "amazing lyrics!"
 
     def test_lyrics_not_found_error(self, track, monkeypatch):
-
         def mocked_urlopen(url, timeout=None):
             raise urllib.request.HTTPError("", "", "", "", "")
 

spotdl/lyrics/providers/tests/test_lyricwikia_wrapper.py

@@ -15,17 +15,21 @@ class TestLyricWikia:
         # `LyricWikia` class uses the 3rd party method `lyricwikia.get_lyrics`
         # internally and there is no need to test a 3rd party library as they
         # have their own implementation of tests.
-        monkeypatch.setattr("lyricwikia.get_lyrics", lambda a, b, c, d: "awesome lyrics!")
+        monkeypatch.setattr(
+            "lyricwikia.get_lyrics", lambda a, b, c, d: "awesome lyrics!"
+        )
         track = LyricWikia("Lyricwikia", "Lyricwikia")
         assert track.get_lyrics() == "awesome lyrics!"
 
     def test_lyrics_not_found_error(self, monkeypatch):
-
         def lyricwikia_lyrics_not_found(msg):
             raise lyricwikia.LyricsNotFound(msg)
 
         # Wrap `lyricwikia.LyricsNotFound` with `exceptions.LyricsNotFound` error.
-        monkeypatch.setattr("lyricwikia.get_lyrics", lambda a, b, c, d: lyricwikia_lyrics_not_found("Nope, no lyrics."))
+        monkeypatch.setattr(
+            "lyricwikia.get_lyrics",
+            lambda a, b, c, d: lyricwikia_lyrics_not_found("Nope, no lyrics."),
+        )
         track = LyricWikia("Lyricwikia", "Lyricwikia")
         with pytest.raises(exceptions.LyricsNotFound):
             track.get_lyrics()

spotdl/metadata.py

@@ -80,7 +80,9 @@ class EmbedMetadata:
         audiofile["TYER"] = TYER(encoding=3, text=meta_tags["year"])
         if meta_tags["publisher"]:
             audiofile["TPUB"] = TPUB(encoding=3, text=meta_tags["publisher"])
-        audiofile["COMM"] = COMM(encoding=3, text=meta_tags["external_urls"][self.provider])
+        audiofile["COMM"] = COMM(
+            encoding=3, text=meta_tags["external_urls"][self.provider]
+        )
         if meta_tags["lyrics"]:
             audiofile["USLT"] = USLT(
                 encoding=3, desc=u"Lyrics", text=meta_tags["lyrics"]

spotdl/patcher.py

@@ -11,24 +11,30 @@ def _getbestthumb(self):
 
     part_url = "https://i.ytimg.com/vi/%s/" % self.videoid
     # Thumbnail resolution sorted in descending order
-    thumbs = ("maxresdefault.jpg",
-              "sddefault.jpg",
-              "hqdefault.jpg",
-              "mqdefault.jpg",
-              "default.jpg")
+    thumbs = (
+        "maxresdefault.jpg",
+        "sddefault.jpg",
+        "hqdefault.jpg",
+        "mqdefault.jpg",
+        "default.jpg",
+    )
     for thumb in thumbs:
         url = part_url + thumb
         if self._content_available(url):
             return url
 
+
 def _process_streams(self):
-    for format_index in range(len(self._ydl_info['formats'])):
+    for format_index in range(len(self._ydl_info["formats"])):
         try:
-            self._ydl_info['formats'][format_index]['url'] = self._ydl_info['formats'][format_index]['fragment_base_url']
+            self._ydl_info["formats"][format_index]["url"] = self._ydl_info["formats"][
+                format_index
+            ]["fragment_base_url"]
         except KeyError:
             pass
     return backend_youtube_dl.YtdlPafy._old_process_streams(self)
 
+
 @classmethod
 def _content_available(cls, url):
     return internals.content_available(url)
@@ -39,6 +45,7 @@ class PatchPafy:
     These patches have not been released by pafy on PyPI yet but
     are useful to us.
     """
+
     def patch_getbestthumb(self):
         # https://github.com/mps-youtube/pafy/pull/211
         pafy.backend_shared.BasePafy._bestthumb = None
@@ -47,7 +54,9 @@ class PatchPafy:
 
     def patch_process_streams(self):
         # https://github.com/mps-youtube/pafy/pull/230
-        backend_youtube_dl.YtdlPafy._old_process_streams = backend_youtube_dl.YtdlPafy._process_streams
+        backend_youtube_dl.YtdlPafy._old_process_streams = (
+            backend_youtube_dl.YtdlPafy._process_streams
+        )
         backend_youtube_dl.YtdlPafy._process_streams = _process_streams
 
     def patch_insecure_streams(self):

spotdl/spotdl.py

@@ -28,8 +28,9 @@ def match_args():
             track_dl.download_single()
     elif const.args.list:
         if const.args.write_m3u:
-            youtube_tools.generate_m3u(track_file=const.args.list,
-                                       text_file=const.args.write_to)
+            youtube_tools.generate_m3u(
+                track_file=const.args.list
+            )
         else:
             list_dl = downloader.ListDownloader(
                 tracks_file=const.args.list,
@@ -38,17 +39,21 @@ def match_args():
             )
             list_dl.download_list()
     elif const.args.playlist:
-        spotify_tools.write_playlist(playlist_url=const.args.playlist,
-                                     text_file=const.args.write_to)
+        spotify_tools.write_playlist(
+            playlist_url=const.args.playlist, text_file=const.args.write_to
+        )
     elif const.args.album:
-        spotify_tools.write_album(album_url=const.args.album,
-                                  text_file=const.args.write_to)
+        spotify_tools.write_album(
+            album_url=const.args.album, text_file=const.args.write_to
+        )
     elif const.args.all_albums:
-        spotify_tools.write_all_albums_from_artist(artist_url=const.args.all_albums,
-                                                   text_file=const.args.write_to)
+        spotify_tools.write_all_albums_from_artist(
+            artist_url=const.args.all_albums, text_file=const.args.write_to
+        )
     elif const.args.username:
-        spotify_tools.write_user_playlist(username=const.args.username,
-                                          text_file=const.args.write_to)
+        spotify_tools.write_user_playlist(
+            username=const.args.username, text_file=const.args.write_to
+        )
 
 
 def main():

spotdl/spotify_tools.py

@@ -17,8 +17,6 @@ from spotdl.lyrics.exceptions import LyricsNotFound
 spotify = None
 
 
-
-
 def generate_token():
     """ Generate the token. """
     credentials = oauth2.SpotifyClientCredentials(
@@ -39,6 +37,7 @@ def must_be_authorized(func, spotify=spotify):
             token = generate_token()
             spotify = spotipy.Spotify(auth=token)
             return func(*args, **kwargs)
+
     return wrapper
 
 

spotdl/youtube_tools.py

@@ -18,6 +18,7 @@ pafy.g.opener.addheaders.append(("Range", "bytes=0-"))
 # More info: https://github.com/mps-youtube/pafy/pull/211
 if pafy.__version__ <= "0.5.4":
     from spotdl import patcher
+
     pafy_patcher = patcher.PatchPafy()
     pafy_patcher.patch_getbestthumb()
     pafy_patcher.patch_process_streams()
@@ -52,10 +53,13 @@ def match_video_and_metadata(track):
     """ Get and match track data from YouTube and Spotify. """
     meta_tags = None
 
-
     def fallback_metadata(meta_tags):
-        fallback_metadata_info = "Track not found on Spotify, falling back on YouTube metadata"
-        skip_fallback_metadata_warning = "Fallback condition not met, shall not embed metadata"
+        fallback_metadata_info = (
+            "Track not found on Spotify, falling back on YouTube metadata"
+        )
+        skip_fallback_metadata_warning = (
+            "Fallback condition not met, shall not embed metadata"
+        )
         if meta_tags is None:
             if const.args.no_fallback_metadata:
                 log.warning(skip_fallback_metadata_warning)
@@ -64,7 +68,6 @@ def match_video_and_metadata(track):
                 meta_tags = generate_metadata(content)
         return meta_tags
 
-
     if internals.is_youtube(track):
         log.debug("Input song is a YouTube URL")
         content = go_pafy(track, meta_tags=None)
@@ -95,25 +98,29 @@ def match_video_and_metadata(track):
 
 def generate_metadata(content):
     """ Fetch a song's metadata from YouTube. """
-    meta_tags = {"spotify_metadata": False,
-                 "name": content.title,
-                 "artists": [{"name": content.author}],
-                 "duration": content.length,
-                 "external_urls": {"youtube": content.watchv_url},
-                 "album": {"images" : [{"url": content.getbestthumb()}],
-                           "artists": [{"name": None}],"name": None},
-                 "year": content.published.split("-")[0],
-                 "release_date": content.published.split(" ")[0],
-                 "type": "track",
-                 "disc_number": 1,
-                 "track_number": 1,
-                 "total_tracks": 1,
-                 "publisher": None,
-                 "external_ids": {"isrc": None},
-                 "lyrics": None,
-                 "copyright": None,
-                 "genre": None,
-                 }
+    meta_tags = {
+        "spotify_metadata": False,
+        "name": content.title,
+        "artists": [{"name": content.author}],
+        "duration": content.length,
+        "external_urls": {"youtube": content.watchv_url},
+        "album": {
+            "images": [{"url": content.getbestthumb()}],
+            "artists": [{"name": None}],
+            "name": None,
+        },
+        "year": content.published.split("-")[0],
+        "release_date": content.published.split(" ")[0],
+        "type": "track",
+        "disc_number": 1,
+        "track_number": 1,
+        "total_tracks": 1,
+        "publisher": None,
+        "external_ids": {"isrc": None},
+        "lyrics": None,
+        "copyright": None,
+        "genre": None,
+    }
 
     return meta_tags
 

test/loader.py

@@ -20,7 +20,7 @@ def load_defaults():
 # so that we get same results even if YouTube changes the list/order of videos on their page.
 GIST_URL = "https://gist.githubusercontent.com/ritiek/e731338e9810e31c2f00f13c249a45f5/raw/c11a27f3b5d11a8d082976f1cdd237bd605ec2c2/search_results.html"
 
+
 def monkeypatch_youtube_search_page(*args, **kwargs):
     fake_urlopen = urllib.request.urlopen(GIST_URL)
     return fake_urlopen
-

test/test_download_with_metadata.py

@@ -101,16 +101,28 @@ class TestDownload:
 
     def test_m4a(self, monkeypatch, filename_fixture):
         expect_download = True
-        monkeypatch.setattr("pafy.backend_shared.BaseStream.download", self.blank_audio_generator)
-        monkeypatch.setattr("pafy.backend_youtube_dl.YtdlStream.download", self.blank_audio_generator)
-        download = youtube_tools.download_song(filename_fixture + ".m4a", pytest.content_fixture)
+        monkeypatch.setattr(
+            "pafy.backend_shared.BaseStream.download", self.blank_audio_generator
+        )
+        monkeypatch.setattr(
+            "pafy.backend_youtube_dl.YtdlStream.download", self.blank_audio_generator
+        )
+        download = youtube_tools.download_song(
+            filename_fixture + ".m4a", pytest.content_fixture
+        )
         assert download == expect_download
 
     def test_webm(self, monkeypatch, filename_fixture):
         expect_download = True
-        monkeypatch.setattr("pafy.backend_shared.BaseStream.download", self.blank_audio_generator)
-        monkeypatch.setattr("pafy.backend_youtube_dl.YtdlStream.download", self.blank_audio_generator)
-        download = youtube_tools.download_song(filename_fixture + ".webm", pytest.content_fixture)
+        monkeypatch.setattr(
+            "pafy.backend_shared.BaseStream.download", self.blank_audio_generator
+        )
+        monkeypatch.setattr(
+            "pafy.backend_youtube_dl.YtdlStream.download", self.blank_audio_generator
+        )
+        download = youtube_tools.download_song(
+            filename_fixture + ".webm", pytest.content_fixture
+        )
         assert download == expect_download
 
 

test/test_patcher.py

@@ -6,6 +6,7 @@ import pytest
 pafy_patcher = patcher.PatchPafy()
 pafy_patcher.patch_getbestthumb()
 
+
 class TestPafyContentAvailable:
     pass
 
@@ -30,7 +31,6 @@ class TestMethodCalls:
         thumbnail = patcher._getbestthumb(content_fixture)
         assert thumbnail == "https://i.ytimg.com/vi/3nQNiWdeH2Q/sddefault.jpg"
 
-
     def test_pafy_content_available(self):
         TestPafyContentAvailable._content_available = patcher._content_available
         assert TestPafyContentAvailable()._content_available("https://youtube.com/")

test/test_youtube_tools.py

@@ -104,19 +104,19 @@ def content_fixture(metadata_fixture):
 MATCH_METADATA_NO_FALLBACK_TEST_TABLE = [
     ("https://open.spotify.com/track/5nWduGwBGBn1PSqYTJUDbS", True),
     ("http://youtube.com/watch?v=3nQNiWdeH2Q", None),
-    ("Linux Talk | Working with Drives and Filesystems", None)
+    ("Linux Talk | Working with Drives and Filesystems", None),
 ]
 
 MATCH_METADATA_FALLBACK_TEST_TABLE = [
     ("https://open.spotify.com/track/5nWduGwBGBn1PSqYTJUDbS", True),
     ("http://youtube.com/watch?v=3nQNiWdeH2Q", False),
-    ("Linux Talk | Working with Drives and Filesystems", False)
+    ("Linux Talk | Working with Drives and Filesystems", False),
 ]
 
 MATCH_METADATA_NO_METADATA_TEST_TABLE = [
     ("https://open.spotify.com/track/5nWduGwBGBn1PSqYTJUDbS", None),
     ("http://youtube.com/watch?v=3nQNiWdeH2Q", None),
-    ("Linux Talk | Working with Drives and Filesystems", None)
+    ("Linux Talk | Working with Drives and Filesystems", None),
 ]
 
 
@@ -128,21 +128,37 @@ class TestMetadataOrigin:
         else:
             assert metadata["spotify_metadata"] == metadata_type
 
-    @pytest.mark.parametrize("track, metadata_type", MATCH_METADATA_NO_FALLBACK_TEST_TABLE)
-    def test_match_metadata_with_no_fallback(self, track, metadata_type, content_fixture, monkeypatch):
-        monkeypatch.setattr(youtube_tools, "go_pafy", lambda track, meta_tags: content_fixture)
+    @pytest.mark.parametrize(
+        "track, metadata_type", MATCH_METADATA_NO_FALLBACK_TEST_TABLE
+    )
+    def test_match_metadata_with_no_fallback(
+        self, track, metadata_type, content_fixture, monkeypatch
+    ):
+        monkeypatch.setattr(
+            youtube_tools, "go_pafy", lambda track, meta_tags: content_fixture
+        )
         const.args.no_fallback_metadata = True
         self.match_metadata(track, metadata_type)
 
     @pytest.mark.parametrize("track, metadata_type", MATCH_METADATA_FALLBACK_TEST_TABLE)
-    def test_match_metadata_with_fallback(self, track, metadata_type, content_fixture, monkeypatch):
-        monkeypatch.setattr(youtube_tools, "go_pafy", lambda track, meta_tags: content_fixture)
+    def test_match_metadata_with_fallback(
+        self, track, metadata_type, content_fixture, monkeypatch
+    ):
+        monkeypatch.setattr(
+            youtube_tools, "go_pafy", lambda track, meta_tags: content_fixture
+        )
         const.args.no_fallback_metadata = False
         self.match_metadata(track, metadata_type)
 
-    @pytest.mark.parametrize("track, metadata_type", MATCH_METADATA_NO_METADATA_TEST_TABLE)
-    def test_match_metadata_with_no_metadata(self, track, metadata_type, content_fixture, monkeypatch):
-        monkeypatch.setattr(youtube_tools, "go_pafy", lambda track, meta_tags: content_fixture)
+    @pytest.mark.parametrize(
+        "track, metadata_type", MATCH_METADATA_NO_METADATA_TEST_TABLE
+    )
+    def test_match_metadata_with_no_metadata(
+        self, track, metadata_type, content_fixture, monkeypatch
+    ):
+        monkeypatch.setattr(
+            youtube_tools, "go_pafy", lambda track, meta_tags: content_fixture
+        )
         const.args.no_metadata = True
         self.match_metadata(track, metadata_type)
 
@@ -185,7 +201,11 @@ def test_check_exists(metadata_fixture, filename_fixture, tmpdir):
 
 
 def test_generate_m3u(tmpdir, monkeypatch):
-    monkeypatch.setattr(youtube_tools.GenerateYouTubeURL, "_fetch_response", loader.monkeypatch_youtube_search_page)
+    monkeypatch.setattr(
+        youtube_tools.GenerateYouTubeURL,
+        "_fetch_response",
+        loader.monkeypatch_youtube_search_page,
+    )
     expect_m3u = (
         "#EXTM3U\n\n"
         "#EXTINF:208,Janji - Heroes Tonight (feat. Johnning) [NCS Release]\n"