Merge pull request #558 from ritiek/pafy-prefer-secure-by-default

Pafy prefer secure HTTPS by default

CHANGES.md

@@ -5,6 +5,47 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Added
+-
+
+### Fixed
+-
+
+### Changed
+-
+
+## [1.2.2] - 2019-06-03
+### Fixed
+- Patch bug in Pafy to prefer secure HTTPS ([@ritiek](https://github.com/ritiek)) (#558)
+
+## [1.2.1] - 2019-04-28
+### Fixed
+- Patch bug in Pafy when fetching audiostreams with latest youtube-dl ([@ritiek](https://github.com/ritiek)) (#539)
+
+### Changed
+- Removed duplicate debug log entry from `internals.trim_song` ([@ritiek](https://github.com/ritiek)) (#519)
+- Fix YAMLLoadWarning ([@cyberboysumanjay](https://github.com/cyberboysumanjay)) (#517)
+
+## [1.2.0] - 2019-03-01
+### Added
+- `--write-to` parameter for setting custom file to write Spotify track URLs to ([@ritiek](https://github.com/ritiek)) (#507)
+- Set custom Spotify Client ID and Client Secret via config.yml ([@ManveerBasra](https://github.com/ManveerBasra)) (#502)
+- Use YouTube as fallback metadata if track not found on Spotify. Also added `--no-fallback-metadata`
+  to preserve old behaviour ([@ritiek](https://github.com/ritiek)) (#457)
+
+### Fixed
+- Fix already downloaded prompt when using "/" in `--file-format` to create sub-directories ([@ritiek](https://github.com/ritiek)) (#503)
+- Fix writing playlist tracks to file ([@ritiek](https://github.com/ritiek)) (#506)
+
+## [1.1.2] - 2019-02-10
+### Changed
+- Fetch all artist albums by default instead of only fetching the "album" type ([@ritiek](https://github.com/ritiek)) (#493)
+- Option `-f` (`--folder`) is used when exporting text files using `-p` (`--playlist`) for playlists or `-b` (`--album`) for albums ([@Silverfeelin](https://github.com/Silverfeelin)) (#476)
+- Use first artist from album object for album artist ([@tillhainbach](https://github.com/tillhainbach))
+
+### Fixed
+- Fix renaming files when encoder is not found ([@ritiek](https://github.com/ritiek)) (#475)
+- Add missing `import time` ([@ifduyue](https://github.com/ifduyue)) (#465)
 
 ## [1.1.1] - 2019-01-03
 ### Added

README.md

@@ -30,7 +30,7 @@ If you still need to use Python 2 - check out the (outdated)
 spotify-downloader works with all major distributions and even on low-powered devices such as a Raspberry Pi.
 
 spotify-downloader can be installed via pip with:
-```
+```console
 $ pip3 install spotdl
 ```
 
@@ -41,7 +41,7 @@ page for detailed OS-specific instructions to get it and other dependencies it r
 
 For the most basic usage, downloading tracks is as easy as
 
-```
+```console
 $ spotdl --song https://open.spotify.com/track/2DGa7iaidT5s0qnINlwMjJ
 $ spotdl --song "ncs - spectre"
 ```
@@ -49,7 +49,7 @@ $ spotdl --song "ncs - spectre"
 For downloading playlist and albums, you need to first load all the tracks into text file and then pass
 this text file to `--list` argument. Here is how you would do it for a playlist
 
-```
+```console
 $ spotdl --playlist https://open.spotify.com/user/nocopyrightsounds/playlist/7sZbq8QGyMnhKPcLJvCUFD
 INFO: Writing 62 tracks to ncs-releases.txt
 $ spotdl --list ncs-releases.txt
@@ -73,7 +73,7 @@ Check out [CONTRIBUTING.md](CONTRIBUTING.md) for more info.
 
 ## Running Tests
 
-```
+```console
 $ python3 -m pytest test
 ```
 

spotdl/__init__.py

@@ -1 +1 @@
-__version__ = "1.1.1"
+__version__ = "1.2.2"

spotdl/convert.py

@@ -83,7 +83,12 @@ class Converter:
             os.rename(self.output_file, self.input_file)
 
         log.debug(command)
-        code = subprocess.call(command)
+        try:
+            code = subprocess.call(command)
+        except FileNotFoundError:
+            if self.rename_to_temp:
+                os.rename(self.input_file, self.output_file)
+            raise
 
         if self.delete_original:
             log.debug('Removing original file: "{}"'.format(self.input_file))
@@ -134,7 +139,12 @@ class Converter:
             os.rename(self.output_file, self.input_file)
 
         log.debug(command)
-        code = subprocess.call(command)
+        try:
+            code = subprocess.call(command)
+        except FileNotFoundError:
+            if self.rename_to_temp:
+                os.rename(self.input_file, self.output_file)
+            raise
 
         if self.delete_original:
             log.debug('Removing original file: "{}"'.format(self.input_file))

spotdl/downloader.py

@@ -1,6 +1,7 @@
 import spotipy
 import urllib
 import os
+import time
 from logzero import logger as log
 
 from spotdl import const
@@ -13,16 +14,20 @@ from spotdl import youtube_tools
 
 class CheckExists:
     def __init__(self, music_file, meta_tags=None):
-        self.music_file = music_file
         self.meta_tags = meta_tags
+        basepath, filename = os.path.split(music_file)
+        filepath = os.path.join(const.args.folder, basepath)
+        os.makedirs(filepath, exist_ok=True)
+        self.filepath = filepath
+        self.filename = filename
 
     def already_exists(self, raw_song):
         """ Check if the input song already exists in the given folder. """
         log.debug(
             "Cleaning any temp files and checking "
-            'if "{}" already exists'.format(self.music_file)
+            'if "{}" already exists'.format(self.filename)
         )
-        songs = os.listdir(const.args.folder)
+        songs = os.listdir(self.filepath)
         self._remove_temp_files(songs)
 
         for song in songs:
@@ -44,17 +49,17 @@ class CheckExists:
     def _remove_temp_files(self, songs):
         for song in songs:
             if song.endswith(".temp"):
-                os.remove(os.path.join(const.args.folder, song))
+                os.remove(os.path.join(self.filepath, song))
 
     def _has_metadata(self, song):
         # check if the already downloaded song has correct metadata
         # if not, remove it and download again without prompt
         already_tagged = metadata.compare(
-            os.path.join(const.args.folder, song), self.meta_tags
+            os.path.join(self.filepath, song), self.meta_tags
         )
         log.debug("Checking if it is already tagged correctly? {}", already_tagged)
         if not already_tagged:
-            os.remove(os.path.join(const.args.folder, song))
+            os.remove(os.path.join(self.filepath, song))
             return False
 
         return True
@@ -79,7 +84,7 @@ class CheckExists:
         return True
 
     def _match_filenames(self, song):
-        if os.path.splitext(song)[0] == self.music_file:
+        if os.path.splitext(song)[0] == self.filename:
             log.debug('Found an already existing song: "{}"'.format(song))
             return True
 
@@ -130,6 +135,8 @@ class Downloader:
                     trim_silence=const.args.trim_silence,
                 )
             except FileNotFoundError:
+                encoder = "avconv" if const.args.avconv else "ffmpeg"
+                log.warning("Could not find {0}, skip encoding".format(encoder))
                 output_song = self.unconverted_filename(songname)
 
             if not const.args.no_metadata and self.meta_tags is not None:
@@ -160,16 +167,12 @@ class Downloader:
             if not refined_songname == " - ":
                 songname = refined_songname
         else:
-            if not const.args.no_metadata:
-                log.warning("Could not find metadata")
             songname = internals.sanitize_title(songname)
 
         return songname
 
     @staticmethod
     def unconverted_filename(songname):
-        encoder = "avconv" if const.args.avconv else "ffmpeg"
-        log.warning("Could not find {0}, skipping conversion".format(encoder))
         const.args.output_ext = const.args.input_ext
         output_song = songname + const.args.output_ext
         return output_song
@@ -203,12 +206,8 @@ class ListDownloader:
             try:
                 track_dl = Downloader(raw_song, number=number)
                 track_dl.download_single()
-            except spotipy.client.SpotifyException:
-                # token expires after 1 hour
-                self._regenerate_token()
-                track_dl.download_single()
-            # detect network problems
             except (urllib.request.URLError, TypeError, IOError) as e:
+                # detect network problems
                 self._cleanup(raw_song, e)
                 # TODO: remove this sleep once #397 is fixed
                 # wait 0.5 sec to avoid infinite looping
@@ -234,11 +233,6 @@ class ListDownloader:
         with open(self.write_successful_file, "a") as f:
             f.write("\n" + raw_song)
 
-    @staticmethod
-    def _regenerate_token():
-        log.debug("Token expired, generating new one and authorizing")
-        spotify_tools.refresh_token()
-
     def _cleanup(self, raw_song, exception):
         self.tracks.append(raw_song)
         # remove the downloaded song from file

spotdl/handle.py

@@ -7,6 +7,7 @@ import argparse
 import mimetypes
 import os
 
+import spotdl
 from spotdl import internals
 
 
@@ -16,11 +17,13 @@ default_conf = {
     "spotify-downloader": {
         "manual": False,
         "no-metadata": False,
+        "no-fallback-metadata": False,
         "avconv": False,
         "folder": internals.get_music_dir(),
         "overwrite": "prompt",
         "input-ext": ".m4a",
         "output-ext": ".mp3",
+        "write-to": None,
         "trim-silence": False,
         "download-only-metadata": False,
         "dry-run": False,
@@ -32,6 +35,8 @@ default_conf = {
         "skip": None,
         "write-successful": None,
         "log-level": "INFO",
+        "spotify_client_id": "4fe3fecfe5334023a1472516cc99d805",
+        "spotify_client_secret": "0f02b7c483c04257984695007a4a8d5c"
     }
 }
 
@@ -53,7 +58,7 @@ def merge(default, config):
 def get_config(config_file):
     try:
         with open(config_file, "r") as ymlfile:
-            cfg = yaml.load(ymlfile)
+            cfg = yaml.safe_load(ymlfile)
     except FileNotFoundError:
         log.info("Writing default configuration to {0}:".format(config_file))
         with open(config_file, "w") as ymlfile:
@@ -120,9 +125,6 @@ def get_arguments(raw_args=None, to_group=True, to_merge=True):
             "--username",
             help="load tracks from user's playlist into <playlist_name>.txt",
         )
-        group.add_argument(
-            "-V", "--version", help="show version and exit", action="store_true"
-        )
 
     parser.add_argument(
         "--write-m3u",
@@ -134,7 +136,7 @@ def get_arguments(raw_args=None, to_group=True, to_merge=True):
         "-m",
         "--manual",
         default=config["manual"],
-        help="choose the track to download manually from a list " "of matching tracks",
+        help="choose the track to download manually from a list of matching tracks",
         action="store_true",
     )
     parser.add_argument(
@@ -144,6 +146,13 @@ def get_arguments(raw_args=None, to_group=True, to_merge=True):
         help="do not embed metadata in tracks",
         action="store_true",
     )
+    parser.add_argument(
+        "-nf",
+        "--no-fallback-metadata",
+        default=config["no-fallback-metadata"],
+        help="do not use YouTube as fallback for metadata if track not found on Spotify",
+        action="store_true",
+    )
     parser.add_argument(
         "-a",
         "--avconv",
@@ -176,6 +185,11 @@ def get_arguments(raw_args=None, to_group=True, to_merge=True):
         default=config["output-ext"],
         help="preferred output format .mp3, .m4a (AAC), .flac, etc.",
     )
+    parser.add_argument(
+        "--write-to",
+        default=config["write-to"],
+        help="write tracks from Spotify playlist, album, etc. to this file",
+    )
     parser.add_argument(
         "-ff",
         "--file-format",
@@ -254,9 +268,27 @@ def get_arguments(raw_args=None, to_group=True, to_merge=True):
         default=config["write-successful"],
         help="path to file to write successful tracks to",
     )
+    parser.add_argument(
+        "-sci",
+        "--spotify-client-id",
+        default=config["spotify_client_id"],
+        help=argparse.SUPPRESS
+    )
+    parser.add_argument(
+        "-scs",
+        "--spotify-client-secret",
+        default=config["spotify_client_secret"],
+        help=argparse.SUPPRESS
+    )
     parser.add_argument(
         "-c", "--config", default=None, help="path to custom config.yml file"
     )
+    parser.add_argument(
+        "-V",
+        "--version",
+        action="version",
+        version="%(prog)s {}".format(spotdl.__version__),
+    )
 
     parsed = parser.parse_args(raw_args)
 
@@ -280,6 +312,12 @@ def get_arguments(raw_args=None, to_group=True, to_merge=True):
     if parsed.avconv and parsed.trim_silence:
         parser.error("--trim-silence can only be used with FFmpeg")
 
+    if parsed.write_to and not (parsed.playlist \
+            or parsed.album \
+            or parsed.all_albums \
+            or parsed.username):
+        parser.error("--write-to can only be used with --playlist, --album, --all-albums, or --username")
+
     parsed.log_level = log_leveller(parsed.log_level)
 
     return parsed

spotdl/internals.py

@@ -1,6 +1,7 @@
 from logzero import logger as log
 import os
 import sys
+import urllib.request
 
 from spotdl import const
 
@@ -50,7 +51,6 @@ def input_link(links):
 
 def trim_song(tracks_file):
     """ Remove the first song from file. """
-    log.debug("Removing downloaded song from tracks file")
     with open(tracks_file, "r") as file_in:
         data = file_in.read().splitlines(True)
     with open(tracks_file, "w") as file_out:
@@ -253,3 +253,12 @@ def remove_duplicates(tracks):
     local_set = set()
     local_set_add = local_set.add
     return [x for x in tracks if not (x in local_set or local_set_add(x))]
+
+
+def content_available(url):
+    try:
+        response = urllib.request.urlopen(url)
+    except HTTPError:
+        return False
+    else:
+        return response.getcode() < 300

spotdl/metadata.py

@@ -46,6 +46,8 @@ class EmbedMetadata:
     def __init__(self, music_file, meta_tags):
         self.music_file = music_file
         self.meta_tags = meta_tags
+        self.spotify_metadata = meta_tags["spotify_metadata"]
+        self.provider = "spotify" if meta_tags["spotify_metadata"] else "youtube"
 
     def as_mp3(self):
         """ Embed metadata to MP3 files. """
@@ -62,7 +64,7 @@ class EmbedMetadata:
         audiofile["lyricist"] = meta_tags["artists"][0]["name"]
         audiofile["arranger"] = meta_tags["artists"][0]["name"]
         audiofile["performer"] = meta_tags["artists"][0]["name"]
-        audiofile["website"] = meta_tags["external_urls"]["spotify"]
+        audiofile["website"] = meta_tags["external_urls"][self.provider]
         audiofile["length"] = str(meta_tags["duration"])
         if meta_tags["publisher"]:
             audiofile["encodedby"] = meta_tags["publisher"]
@@ -78,7 +80,7 @@ class EmbedMetadata:
         audiofile["TYER"] = TYER(encoding=3, text=meta_tags["year"])
         if meta_tags["publisher"]:
             audiofile["TPUB"] = TPUB(encoding=3, text=meta_tags["publisher"])
-        audiofile["COMM"] = COMM(encoding=3, text=meta_tags["external_urls"]["spotify"])
+        audiofile["COMM"] = COMM(encoding=3, text=meta_tags["external_urls"][self.provider])
         if meta_tags["lyrics"]:
             audiofile["USLT"] = USLT(
                 encoding=3, desc=u"Lyrics", text=meta_tags["lyrics"]
@@ -106,7 +108,7 @@ class EmbedMetadata:
         audiofile = MP4(music_file)
         self._embed_basic_metadata(audiofile, preset=M4A_TAG_PRESET)
         audiofile[M4A_TAG_PRESET["year"]] = meta_tags["year"]
-        audiofile[M4A_TAG_PRESET["comment"]] = meta_tags["external_urls"]["spotify"]
+        audiofile[M4A_TAG_PRESET["comment"]] = meta_tags["external_urls"][self.provider]
         if meta_tags["lyrics"]:
             audiofile[M4A_TAG_PRESET["lyrics"]] = meta_tags["lyrics"]
         try:
@@ -127,7 +129,7 @@ class EmbedMetadata:
         audiofile = FLAC(music_file)
         self._embed_basic_metadata(audiofile)
         audiofile["year"] = meta_tags["year"]
-        audiofile["comment"] = meta_tags["external_urls"]["spotify"]
+        audiofile["comment"] = meta_tags["external_urls"][self.provider]
         if meta_tags["lyrics"]:
             audiofile["lyrics"] = meta_tags["lyrics"]
 
@@ -146,8 +148,10 @@ class EmbedMetadata:
     def _embed_basic_metadata(self, audiofile, preset=TAG_PRESET):
         meta_tags = self.meta_tags
         audiofile[preset["artist"]] = meta_tags["artists"][0]["name"]
-        audiofile[preset["albumartist"]] = meta_tags["artists"][0]["name"]
+        if meta_tags["album"]["artists"][0]["name"]:
-        audiofile[preset["album"]] = meta_tags["album"]["name"]
+            audiofile[preset["albumartist"]] = meta_tags["album"]["artists"][0]["name"]
+        if meta_tags["album"]["name"]:
+            audiofile[preset["album"]] = meta_tags["album"]["name"]
         audiofile[preset["title"]] = meta_tags["name"]
         audiofile[preset["date"]] = meta_tags["release_date"]
         audiofile[preset["originaldate"]] = meta_tags["release_date"]

spotdl/patcher.py

@@ -0,0 +1,55 @@
+from pafy import backend_youtube_dl
+import pafy
+
+from spotdl import internals
+
+
+def _getbestthumb(self):
+    url = self._ydl_info["thumbnails"][0]["url"]
+    if url:
+        return url
+
+    part_url = "https://i.ytimg.com/vi/%s/" % self.videoid
+    # Thumbnail resolution sorted in descending order
+    thumbs = ("maxresdefault.jpg",
+              "sddefault.jpg",
+              "hqdefault.jpg",
+              "mqdefault.jpg",
+              "default.jpg")
+    for thumb in thumbs:
+        url = part_url + thumb
+        if self._content_available(url):
+            return url
+
+def _process_streams(self):
+    for format_index in range(len(self._ydl_info['formats'])):
+        try:
+            self._ydl_info['formats'][format_index]['url'] = self._ydl_info['formats'][format_index]['fragment_base_url']
+        except KeyError:
+            pass
+    return backend_youtube_dl.YtdlPafy._old_process_streams(self)
+
+@classmethod
+def _content_available(cls, url):
+    return internals.content_available(url)
+
+
+class PatchPafy:
+    """
+    These patches have not been released by pafy on PyPI yet but
+    are useful to us.
+    """
+    def patch_getbestthumb(self):
+        # https://github.com/mps-youtube/pafy/pull/211
+        pafy.backend_shared.BasePafy._bestthumb = None
+        pafy.backend_shared.BasePafy._content_available = _content_available
+        pafy.backend_shared.BasePafy.getbestthumb = _getbestthumb
+
+    def patch_process_streams(self):
+        # https://github.com/mps-youtube/pafy/pull/230
+        backend_youtube_dl.YtdlPafy._old_process_streams = backend_youtube_dl.YtdlPafy._process_streams
+        backend_youtube_dl.YtdlPafy._process_streams = _process_streams
+
+    def patch_insecure_streams(self):
+        # https://github.com/mps-youtube/pafy/pull/235
+        pafy.g.def_ydl_opts["prefer_insecure"] = False

spotdl/spotdl.py

@@ -28,7 +28,8 @@ def match_args():
             track_dl.download_single()
     elif const.args.list:
         if const.args.write_m3u:
-            youtube_tools.generate_m3u(track_file=const.args.list)
+            youtube_tools.generate_m3u(track_file=const.args.list,
+                                       text_file=const.args.write_to)
         else:
             list_dl = downloader.ListDownloader(
                 tracks_file=const.args.list,
@@ -37,22 +38,22 @@ def match_args():
             )
             list_dl.download_list()
     elif const.args.playlist:
-        spotify_tools.write_playlist(playlist_url=const.args.playlist)
+        spotify_tools.write_playlist(playlist_url=const.args.playlist,
+                                     text_file=const.args.write_to)
     elif const.args.album:
-        spotify_tools.write_album(album_url=const.args.album)
+        spotify_tools.write_album(album_url=const.args.album,
+                                  text_file=const.args.write_to)
     elif const.args.all_albums:
-        spotify_tools.write_all_albums_from_artist(artist_url=const.args.all_albums)
+        spotify_tools.write_all_albums_from_artist(artist_url=const.args.all_albums,
+                                                   text_file=const.args.write_to)
     elif const.args.username:
-        spotify_tools.write_user_playlist(username=const.args.username)
+        spotify_tools.write_user_playlist(username=const.args.username,
+                                          text_file=const.args.write_to)
 
 
 def main():
     const.args = handle.get_arguments()
 
-    if const.args.version:
-        print("spotdl {version}".format(version=__version__))
-        sys.exit()
-
     internals.filter_path(const.args.folder)
     youtube_tools.set_api_key()
 

spotdl/spotify_tools.py

@@ -7,33 +7,39 @@ from titlecase import titlecase
 from logzero import logger as log
 import pprint
 import sys
+import os
+import functools
 
+from spotdl import const
 from spotdl import internals
 
+spotify = None
+
 
 def generate_token():
-    """ Generate the token. Please respect these credentials :) """
+    """ Generate the token. """
     credentials = oauth2.SpotifyClientCredentials(
-        client_id="4fe3fecfe5334023a1472516cc99d805",
+        client_id=const.args.spotify_client_id,
-        client_secret="0f02b7c483c04257984695007a4a8d5c",
+        client_secret=const.args.spotify_client_secret,
     )
     token = credentials.get_access_token()
     return token
 
 
-def refresh_token():
+def must_be_authorized(func, spotify=spotify):
-    """ Refresh expired token"""
+    def wrapper(*args, **kwargs):
-    global spotify
+        global spotify
-    new_token = generate_token()
+        try:
-    spotify = spotipy.Spotify(auth=new_token)
+            assert spotify
-
+            return func(*args, **kwargs)
-
+        except (AssertionError, spotipy.client.SpotifyException):
-# token is mandatory when using Spotify's API
+            token = generate_token()
-# https://developer.spotify.com/news-stories/2017/01/27/removing-unauthenticated-calls-to-the-web-api/
+            spotify = spotipy.Spotify(auth=token)
-_token = generate_token()
+            return func(*args, **kwargs)
-spotify = spotipy.Spotify(auth=_token)
+    return wrapper
 
 
+@must_be_authorized
 def generate_metadata(raw_song):
     """ Fetch a song's metadata from Spotify. """
     if internals.is_spotify(raw_song):
@@ -79,6 +85,7 @@ def generate_metadata(raw_song):
     # Some sugar
     meta_tags["year"], *_ = meta_tags["release_date"].split("-")
     meta_tags["duration"] = meta_tags["duration_ms"] / 1000.0
+    meta_tags["spotify_metadata"] = True
     # Remove unwanted parameters
     del meta_tags["duration_ms"]
     del meta_tags["available_markets"]
@@ -88,6 +95,15 @@ def generate_metadata(raw_song):
     return meta_tags
 
 
+@must_be_authorized
+def write_user_playlist(username, text_file=None):
+    """ Write user playlists to text_file """
+    links = get_playlists(username=username)
+    playlist = internals.input_link(links)
+    return write_playlist(playlist, text_file)
+
+
+@must_be_authorized
 def get_playlists(username):
     """ Fetch user playlists when using the -u option. """
     playlists = spotify.user_playlists(username)
@@ -116,12 +132,7 @@ def get_playlists(username):
     return links
 
 
-def write_user_playlist(username, text_file=None):
+@must_be_authorized
-    links = get_playlists(username=username)
-    playlist = internals.input_link(links)
-    return write_playlist(playlist, text_file)
-
-
 def fetch_playlist(playlist):
     try:
         playlist_id = internals.extract_spotify_id(playlist)
@@ -141,6 +152,7 @@ def fetch_playlist(playlist):
     return results
 
 
+@must_be_authorized
 def write_playlist(playlist_url, text_file=None):
     playlist = fetch_playlist(playlist_url)
     tracks = playlist["tracks"]
@@ -149,19 +161,21 @@ def write_playlist(playlist_url, text_file=None):
     return write_tracks(tracks, text_file)
 
 
+@must_be_authorized
 def fetch_album(album):
     album_id = internals.extract_spotify_id(album)
     album = spotify.album(album_id)
     return album
 
 
-def fetch_albums_from_artist(artist_url, album_type="album"):
+@must_be_authorized
+def fetch_albums_from_artist(artist_url, album_type=None):
     """
     This funcction returns all the albums from a give artist_url using the US
     market
     :param artist_url - spotify artist url
     :param album_type - the type of album to fetch (ex: single) the default is
-                        a standard album
+                        all albums
     :param return - the album from the artist
     """
 
@@ -180,6 +194,7 @@ def fetch_albums_from_artist(artist_url, album_type="album"):
     return albums
 
 
+@must_be_authorized
 def write_all_albums_from_artist(artist_url, text_file=None):
     """
     This function gets all albums from an artist and writes it to a file in the
@@ -192,7 +207,7 @@ def write_all_albums_from_artist(artist_url, text_file=None):
     album_base_url = "https://open.spotify.com/album/"
 
     # fetching all default albums
-    albums = fetch_albums_from_artist(artist_url)
+    albums = fetch_albums_from_artist(artist_url, album_type=None)
 
     # if no file if given, the default save file is in the current working
     # directory with the name of the artist
@@ -204,14 +219,8 @@ def write_all_albums_from_artist(artist_url, text_file=None):
         log.info("Fetching album: " + album["name"])
         write_album(album_base_url + album["id"], text_file=text_file)
 
-    # fetching all single albums
-    singles = fetch_albums_from_artist(artist_url, album_type="single")
-
-    for single in singles:
-        log.info("Fetching single: " + single["name"])
-        write_album(album_base_url + single["id"], text_file=text_file)
-
 
+@must_be_authorized
 def write_album(album_url, text_file=None):
     album = fetch_album(album_url)
     tracks = spotify.album_tracks(album["id"])
@@ -220,6 +229,7 @@ def write_album(album_url, text_file=None):
     return write_tracks(tracks, text_file)
 
 
+@must_be_authorized
 def write_tracks(tracks, text_file):
     log.info(u"Writing {0} tracks to {1}".format(tracks["total"], text_file))
     track_urls = []

spotdl/youtube_tools.py

@@ -14,6 +14,15 @@ from spotdl import const
 # Read more on mps-youtube/pafy#199
 pafy.g.opener.addheaders.append(("Range", "bytes=0-"))
 
+# Implement unreleased methods on Pafy object
+# More info: https://github.com/mps-youtube/pafy/pull/211
+if pafy.__version__ <= "0.5.4":
+    from spotdl import patcher
+    pafy_patcher = patcher.PatchPafy()
+    pafy_patcher.patch_getbestthumb()
+    pafy_patcher.patch_process_streams()
+    pafy_patcher.patch_insecure_streams()
+
 
 def set_api_key():
     if const.args.youtube_api_key:
@@ -39,28 +48,76 @@ def go_pafy(raw_song, meta_tags=None):
     return track_info
 
 
-def match_video_and_metadata(track, force_pafy=True):
+def match_video_and_metadata(track):
     """ Get and match track data from YouTube and Spotify. """
     meta_tags = None
 
+
+    def fallback_metadata(meta_tags):
+        fallback_metadata_info = "Track not found on Spotify, falling back on YouTube metadata"
+        skip_fallback_metadata_warning = "Fallback condition not met, shall not embed metadata"
+        if meta_tags is None:
+            if const.args.no_fallback_metadata:
+                log.warning(skip_fallback_metadata_warning)
+            else:
+                log.info(fallback_metadata_info)
+                meta_tags = generate_metadata(content)
+        return meta_tags
+
+
     if internals.is_youtube(track):
         log.debug("Input song is a YouTube URL")
         content = go_pafy(track, meta_tags=None)
         track = slugify(content.title).replace("-", " ")
         if not const.args.no_metadata:
             meta_tags = spotify_tools.generate_metadata(track)
-    else:
+            meta_tags = fallback_metadata(meta_tags)
-        # Let it generate metadata, youtube doesn't know spotify slang
-        if not const.args.no_metadata or internals.is_spotify(track):
-            meta_tags = spotify_tools.generate_metadata(track)
 
-        if force_pafy:
+    elif internals.is_spotify(track):
-            content = go_pafy(track, meta_tags)
+        log.debug("Input song is a Spotify URL")
+        # Let it generate metadata, YouTube doesn't know Spotify slang
+        meta_tags = spotify_tools.generate_metadata(track)
+        content = go_pafy(track, meta_tags)
+        if const.args.no_metadata:
+            meta_tags = None
+
+    else:
+        log.debug("Input song is plain text based")
+        if const.args.no_metadata:
+            content = go_pafy(track, meta_tags=None)
         else:
-            content = None
+            meta_tags = spotify_tools.generate_metadata(track)
+            content = go_pafy(track, meta_tags=meta_tags)
+            meta_tags = fallback_metadata(meta_tags)
+
     return content, meta_tags
 
 
+def generate_metadata(content):
+    """ Fetch a song's metadata from YouTube. """
+    meta_tags = {"spotify_metadata": False,
+                 "name": content.title,
+                 "artists": [{"name": content.author}],
+                 "duration": content.length,
+                 "external_urls": {"youtube": content.watchv_url},
+                 "album": {"images" : [{"url": content.getbestthumb()}],
+                           "artists": [{"name": None}],"name": None},
+                 "year": content.published.split("-")[0],
+                 "release_date": content.published.split(" ")[0],
+                 "type": "track",
+                 "disc_number": 1,
+                 "track_number": 1,
+                 "total_tracks": 1,
+                 "publisher": None,
+                 "external_ids": {"isrc": None},
+                 "lyrics": None,
+                 "copyright": None,
+                 "genre": None,
+                 }
+
+    return meta_tags
+
+
 def get_youtube_title(content, number=None):
     """ Get the YouTube video's title. """
     title = content.title

test/loader.py

@@ -1,6 +1,7 @@
 from spotdl import const
 from spotdl import handle
 from spotdl import spotdl
+import urllib
 
 import pytest
 
@@ -13,3 +14,13 @@ def load_defaults():
     spotdl.log = const.logzero.setup_logger(
         formatter=const._formatter, level=const.args.log_level
     )
+
+
+# GIST_URL is the monkeypatched version of: https://www.youtube.com/results?search_query=janji+-+heroes
+# so that we get same results even if YouTube changes the list/order of videos on their page.
+GIST_URL = "https://gist.githubusercontent.com/ritiek/e731338e9810e31c2f00f13c249a45f5/raw/c11a27f3b5d11a8d082976f1cdd237bd605ec2c2/search_results.html"
+
+def monkeypatch_youtube_search_page(*args, **kwargs):
+    fake_urlopen = urllib.request.urlopen(GIST_URL)
+    return fake_urlopen
+

test/test_download_with_metadata.py

@@ -1,4 +1,3 @@
-import urllib
 import subprocess
 import os
 
@@ -39,7 +38,7 @@ def metadata_fixture():
 
 
 def test_metadata(metadata_fixture):
-    expect_number = 23
+    expect_number = 24
     assert len(metadata_fixture) == expect_number
 
 
@@ -54,16 +53,11 @@ class TestFileFormat:
         assert title == EXPECTED_SPOTIFY_TITLE.replace(" ", "_")
 
 
-def monkeypatch_youtube_search_page(*args, **kwargs):
-    fake_urlopen = urllib.request.urlopen(GIST_URL)
-    return fake_urlopen
-
-
 def test_youtube_url(metadata_fixture, monkeypatch):
     monkeypatch.setattr(
         youtube_tools.GenerateYouTubeURL,
         "_fetch_response",
-        monkeypatch_youtube_search_page,
+        loader.monkeypatch_youtube_search_page,
     )
     url = youtube_tools.generate_youtube_url(SPOTIFY_TRACK_URL, metadata_fixture)
     assert url == EXPECTED_YOUTUBE_URL
@@ -73,7 +67,7 @@ def test_youtube_title(metadata_fixture, monkeypatch):
     monkeypatch.setattr(
         youtube_tools.GenerateYouTubeURL,
         "_fetch_response",
-        monkeypatch_youtube_search_page,
+        loader.monkeypatch_youtube_search_page,
     )
     content = youtube_tools.go_pafy(SPOTIFY_TRACK_URL, metadata_fixture)
     pytest.content_fixture = content
@@ -107,22 +101,16 @@ class TestDownload:
 
     def test_m4a(self, monkeypatch, filename_fixture):
         expect_download = True
-        monkeypatch.setattr(
+        monkeypatch.setattr("pafy.backend_shared.BaseStream.download", self.blank_audio_generator)
-            "pafy.backend_shared.BaseStream.download", self.blank_audio_generator
+        monkeypatch.setattr("pafy.backend_youtube_dl.YtdlStream.download", self.blank_audio_generator)
-        )
+        download = youtube_tools.download_song(filename_fixture + ".m4a", pytest.content_fixture)
-        download = youtube_tools.download_song(
-            filename_fixture + ".m4a", pytest.content_fixture
-        )
         assert download == expect_download
 
     def test_webm(self, monkeypatch, filename_fixture):
         expect_download = True
-        monkeypatch.setattr(
+        monkeypatch.setattr("pafy.backend_shared.BaseStream.download", self.blank_audio_generator)
-            "pafy.backend_shared.BaseStream.download", self.blank_audio_generator
+        monkeypatch.setattr("pafy.backend_youtube_dl.YtdlStream.download", self.blank_audio_generator)
-        )
+        download = youtube_tools.download_song(filename_fixture + ".webm", pytest.content_fixture)
-        download = youtube_tools.download_song(
-            filename_fixture + ".webm", pytest.content_fixture
-        )
         assert download == expect_download
 
 
@@ -188,6 +176,7 @@ class TestFFmpeg:
 
 
 class TestAvconv:
+    @pytest.mark.skip(reason="avconv is no longer provided with FFmpeg")
     def test_convert_from_m4a_to_mp3(self, filename_fixture, monkeypatch):
         monkeypatch.setattr("os.remove", lambda x: None)
         expect_command = "avconv -loglevel 0 -i {0}.m4a -ab 192k {0}.mp3 -y".format(

test/test_patcher.py

@@ -0,0 +1,36 @@
+from spotdl import patcher
+import pafy
+
+import pytest
+
+pafy_patcher = patcher.PatchPafy()
+pafy_patcher.patch_getbestthumb()
+
+class TestPafyContentAvailable:
+    pass
+
+
+class TestMethodAssignment:
+    def test_pafy_getbestthumb(self):
+        pafy.backend_shared.BasePafy.getbestthumb == patcher._getbestthumb
+
+
+class TestMethodCalls:
+    @pytest.fixture(scope="module")
+    def content_fixture(self):
+        content = pafy.new("http://youtube.com/watch?v=3nQNiWdeH2Q")
+        return content
+
+    def test_pafy_getbestthumb(self, content_fixture):
+        thumbnail = patcher._getbestthumb(content_fixture)
+        assert thumbnail == "https://i.ytimg.com/vi/3nQNiWdeH2Q/maxresdefault.jpg"
+
+    def test_pafy_getbestthumb_without_ytdl(self, content_fixture):
+        content_fixture._ydl_info["thumbnails"][0]["url"] = None
+        thumbnail = patcher._getbestthumb(content_fixture)
+        assert thumbnail == "https://i.ytimg.com/vi/3nQNiWdeH2Q/maxresdefault.jpg"
+
+
+    def test_pafy_content_available(self):
+        TestPafyContentAvailable._content_available = patcher._content_available
+        assert TestPafyContentAvailable()._content_available("https://youtube.com/")

test/test_spotify_tools.py

@@ -1,7 +1,13 @@
 from spotdl import spotify_tools
+from spotdl import const
+
+import spotipy
 
 import os
 import pytest
+import loader
+
+loader.load_defaults()
 
 
 def test_generate_token():
@@ -9,11 +15,33 @@ def test_generate_token():
     assert len(token) == 83
 
 
-def test_refresh_token():
+class TestMustBeAuthorizedDecorator:
-    old_instance = spotify_tools.spotify
+    def test_spotify_instance_is_unset(self):
-    spotify_tools.refresh_token()
+        spotify_tools.spotify = None
-    new_instance = spotify_tools.spotify
+
-    assert not old_instance == new_instance
+        @spotify_tools.must_be_authorized
+        def sample_func():
+            return True
+
+        assert sample_func()
+
+    def test_spotify_instance_forces_assertion_error(self):
+        @spotify_tools.must_be_authorized
+        def sample_func():
+            raise AssertionError
+
+        with pytest.raises(AssertionError):
+            sample_func()
+
+    def test_fake_token_generator(self, monkeypatch):
+        spotify_tools.spotify = None
+        monkeypatch.setattr(spotify_tools, "generate_token", lambda: 123123)
+
+        with pytest.raises(spotipy.client.SpotifyException):
+            spotify_tools.generate_metadata("ncs - spectre")
+
+    def test_correct_token(self):
+        assert spotify_tools.generate_metadata("ncs - spectre")
 
 
 class TestGenerateMetadata:
@@ -23,7 +51,7 @@ class TestGenerateMetadata:
         return metadata
 
     def test_len(self, metadata_fixture):
-        assert len(metadata_fixture) == 23
+        assert len(metadata_fixture) == 24
 
     def test_trackname(self, metadata_fixture):
         assert metadata_fixture["name"] == "Spectre"
@@ -87,7 +115,7 @@ def test_write_playlist(tmpdir):
     assert tracks == expect_tracks
 
 
-# XXX: Mock this test off if it fails in future
+# XXX: Monkeypatch these tests if they fail in future
 class TestFetchAlbum:
     @pytest.fixture(scope="module")
     def album_fixture(self):
@@ -103,7 +131,7 @@ class TestFetchAlbum:
         assert album_fixture["tracks"]["total"] == 15
 
 
-# XXX: Mock this test off if it fails in future
+# XXX: Monkeypatch these tests if they fail in future
 class TestFetchAlbumsFromArtist:
     @pytest.fixture(scope="module")
     def albums_from_artist_fixture(self):
@@ -113,7 +141,7 @@ class TestFetchAlbumsFromArtist:
         return albums
 
     def test_len(self, albums_from_artist_fixture):
-        assert len(albums_from_artist_fixture) == 18
+        assert len(albums_from_artist_fixture) == 53
 
     def test_zeroth_album_name(self, albums_from_artist_fixture):
         assert albums_from_artist_fixture[0]["name"] == "Revolution Radio"
@@ -128,11 +156,8 @@ class TestFetchAlbumsFromArtist:
         assert albums_from_artist_fixture[0]["total_tracks"] == 12
 
 
-# XXX: Mock this test off if it fails in future
 def test_write_all_albums_from_artist(tmpdir):
-    # current number of tracks on spotify since as of 10/10/2018
+    expect_tracks = 282
-    # in US market only
-    expect_tracks = 49
     text_file = os.path.join(str(tmpdir), "test_ab.txt")
     spotify_tools.write_all_albums_from_artist(
         "https://open.spotify.com/artist/4dpARuHxo51G3z768sgnrY", text_file

test/test_youtube_tools.py

@@ -17,7 +17,6 @@ YT_API_KEY = "AIzaSyAnItl3udec-Q1d5bkjKJGL-RgrKO_vU90"
 TRACK_SEARCH = "Tony's Videos VERY SHORT VIDEO 28.10.2016"
 EXPECTED_TITLE = TRACK_SEARCH
 EXPECTED_YT_URL = "http://youtube.com/watch?v=qOOcy2-tmbk"
-EXPECTED_YT_URLS = (EXPECTED_YT_URL, "http://youtube.com/watch?v=5USR1Omo7f0")
 
 RESULT_COUNT_SEARCH = "she is still sleeping SAO"
 
@@ -71,8 +70,7 @@ class TestYouTubeURL:
     def test_only_music_category(self, metadata_fixture):
         const.args.music_videos_only = True
         url = youtube_tools.generate_youtube_url(TRACK_SEARCH, metadata_fixture)
-        # YouTube keeps changing its results
+        assert url == EXPECTED_YT_URL
-        assert url in EXPECTED_YT_URLS
 
     def test_all_categories(self, metadata_fixture):
         const.args.music_videos_only = False
@@ -99,6 +97,56 @@ def content_fixture(metadata_fixture):
     return content
 
 
+# True = Metadata must be fetched from Spotify
+# False = Metadata must be fetched from YouTube
+# None = Metadata must be `None`
+
+MATCH_METADATA_NO_FALLBACK_TEST_TABLE = [
+    ("https://open.spotify.com/track/5nWduGwBGBn1PSqYTJUDbS", True),
+    ("http://youtube.com/watch?v=3nQNiWdeH2Q", None),
+    ("Linux Talk | Working with Drives and Filesystems", None)
+]
+
+MATCH_METADATA_FALLBACK_TEST_TABLE = [
+    ("https://open.spotify.com/track/5nWduGwBGBn1PSqYTJUDbS", True),
+    ("http://youtube.com/watch?v=3nQNiWdeH2Q", False),
+    ("Linux Talk | Working with Drives and Filesystems", False)
+]
+
+MATCH_METADATA_NO_METADATA_TEST_TABLE = [
+    ("https://open.spotify.com/track/5nWduGwBGBn1PSqYTJUDbS", None),
+    ("http://youtube.com/watch?v=3nQNiWdeH2Q", None),
+    ("Linux Talk | Working with Drives and Filesystems", None)
+]
+
+
+class TestMetadataOrigin:
+    def match_metadata(self, track, metadata_type):
+        _, metadata = youtube_tools.match_video_and_metadata(track)
+        if metadata_type is None:
+            assert metadata == metadata_type
+        else:
+            assert metadata["spotify_metadata"] == metadata_type
+
+    @pytest.mark.parametrize("track, metadata_type", MATCH_METADATA_NO_FALLBACK_TEST_TABLE)
+    def test_match_metadata_with_no_fallback(self, track, metadata_type, content_fixture, monkeypatch):
+        monkeypatch.setattr(youtube_tools, "go_pafy", lambda track, meta_tags: content_fixture)
+        const.args.no_fallback_metadata = True
+        self.match_metadata(track, metadata_type)
+
+    @pytest.mark.parametrize("track, metadata_type", MATCH_METADATA_FALLBACK_TEST_TABLE)
+    def test_match_metadata_with_fallback(self, track, metadata_type, content_fixture, monkeypatch):
+        monkeypatch.setattr(youtube_tools, "go_pafy", lambda track, meta_tags: content_fixture)
+        const.args.no_fallback_metadata = False
+        self.match_metadata(track, metadata_type)
+
+    @pytest.mark.parametrize("track, metadata_type", MATCH_METADATA_NO_METADATA_TEST_TABLE)
+    def test_match_metadata_with_no_metadata(self, track, metadata_type, content_fixture, monkeypatch):
+        monkeypatch.setattr(youtube_tools, "go_pafy", lambda track, meta_tags: content_fixture)
+        const.args.no_metadata = True
+        self.match_metadata(track, metadata_type)
+
+
 @pytest.fixture(scope="module")
 def title_fixture(content_fixture):
     title = youtube_tools.get_youtube_title(content_fixture)
@@ -136,6 +184,26 @@ def test_check_exists(metadata_fixture, filename_fixture, tmpdir):
     assert check == expect_check
 
 
+def test_generate_m3u(tmpdir, monkeypatch):
+    monkeypatch.setattr(youtube_tools.GenerateYouTubeURL, "_fetch_response", loader.monkeypatch_youtube_search_page)
+    expect_m3u = (
+        "#EXTM3U\n\n"
+        "#EXTINF:208,Janji - Heroes Tonight (feat. Johnning) [NCS Release]\n"
+        "http://www.youtube.com/watch?v=3nQNiWdeH2Q\n"
+        "#EXTINF:226,Alan Walker - Spectre [NCS Release]\n"
+        "http://www.youtube.com/watch?v=AOeY-nDp7hI\n"
+    )
+    m3u_track_file = os.path.join(str(tmpdir), "m3u_test.txt")
+    with open(m3u_track_file, "w") as track_file:
+        track_file.write("\nhttps://open.spotify.com/track/3SipFlNddvL0XNZRLXvdZD")
+        track_file.write("\nhttp://www.youtube.com/watch?v=AOeY-nDp7hI")
+    youtube_tools.generate_m3u(m3u_track_file)
+    m3u_file = "{}.m3u".format(m3u_track_file.split(".")[0])
+    with open(m3u_file, "r") as m3u_in:
+        m3u = m3u_in.readlines()
+    assert "".join(m3u) == expect_m3u
+
+
 class TestDownload:
     def test_webm(self, content_fixture, filename_fixture, monkeypatch):
         # content_fixture does not have any .webm audiostream