Merge pull request #558 from ritiek/pafy-prefer-secure-by-default

Pafy prefer secure HTTPS by default

CHANGES.md

@@ -14,6 +14,18 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ### Changed
 -
 
+## [1.2.2] - 2019-06-03
+### Fixed
+- Patch bug in Pafy to prefer secure HTTPS ([@ritiek](https://github.com/ritiek)) (#558)
+
+## [1.2.1] - 2019-04-28
+### Fixed
+- Patch bug in Pafy when fetching audiostreams with latest youtube-dl ([@ritiek](https://github.com/ritiek)) (#539)
+
+### Changed
+- Removed duplicate debug log entry from `internals.trim_song` ([@ritiek](https://github.com/ritiek)) (#519)
+- Fix YAMLLoadWarning ([@cyberboysumanjay](https://github.com/cyberboysumanjay)) (#517)
+
 ## [1.2.0] - 2019-03-01
 ### Added
 - `--write-to` parameter for setting custom file to write Spotify track URLs to ([@ritiek](https://github.com/ritiek)) (#507)

README.md

@@ -30,7 +30,7 @@ If you still need to use Python 2 - check out the (outdated)
 spotify-downloader works with all major distributions and even on low-powered devices such as a Raspberry Pi.
 
 spotify-downloader can be installed via pip with:
-```
+```console
 $ pip3 install spotdl
 ```
 
@@ -41,7 +41,7 @@ page for detailed OS-specific instructions to get it and other dependencies it r
 
 For the most basic usage, downloading tracks is as easy as
 
-```
+```console
 $ spotdl --song https://open.spotify.com/track/2DGa7iaidT5s0qnINlwMjJ
 $ spotdl --song "ncs - spectre"
 ```
@@ -49,7 +49,7 @@ $ spotdl --song "ncs - spectre"
 For downloading playlist and albums, you need to first load all the tracks into text file and then pass
 this text file to `--list` argument. Here is how you would do it for a playlist
 
-```
+```console
 $ spotdl --playlist https://open.spotify.com/user/nocopyrightsounds/playlist/7sZbq8QGyMnhKPcLJvCUFD
 INFO: Writing 62 tracks to ncs-releases.txt
 $ spotdl --list ncs-releases.txt
@@ -73,7 +73,7 @@ Check out [CONTRIBUTING.md](CONTRIBUTING.md) for more info.
 
 ## Running Tests
 
-```
+```console
 $ python3 -m pytest test
 ```
 

spotdl/__init__.py

@@ -1 +1 @@
-__version__ = "1.2.0"
+__version__ = "1.2.2"

spotdl/handle.py

@@ -58,7 +58,7 @@ def merge(default, config):
 def get_config(config_file):
     try:
         with open(config_file, "r") as ymlfile:
-            cfg = yaml.load(ymlfile)
+            cfg = yaml.safe_load(ymlfile)
     except FileNotFoundError:
         log.info("Writing default configuration to {0}:".format(config_file))
         with open(config_file, "w") as ymlfile:

spotdl/internals.py

@@ -51,7 +51,6 @@ def input_link(links):
 
 def trim_song(tracks_file):
     """ Remove the first song from file. """
-    log.debug("Removing downloaded song from tracks file")
     with open(tracks_file, "r") as file_in:
         data = file_in.read().splitlines(True)
     with open(tracks_file, "w") as file_out:

spotdl/patcher.py

@@ -1,3 +1,4 @@
+from pafy import backend_youtube_dl
 import pafy
 
 from spotdl import internals
@@ -20,12 +21,35 @@ def _getbestthumb(self):
         if self._content_available(url):
             return url
 
+def _process_streams(self):
+    for format_index in range(len(self._ydl_info['formats'])):
+        try:
+            self._ydl_info['formats'][format_index]['url'] = self._ydl_info['formats'][format_index]['fragment_base_url']
+        except KeyError:
+            pass
+    return backend_youtube_dl.YtdlPafy._old_process_streams(self)
 
 @classmethod
 def _content_available(cls, url):
     return internals.content_available(url)
 
-def patch_pafy():
-    pafy.backend_shared.BasePafy._bestthumb = None
-    pafy.backend_shared.BasePafy._content_available = _content_available
-    pafy.backend_shared.BasePafy.getbestthumb = _getbestthumb
+
+class PatchPafy:
+    """
+    These patches have not been released by pafy on PyPI yet but
+    are useful to us.
+    """
+    def patch_getbestthumb(self):
+        # https://github.com/mps-youtube/pafy/pull/211
+        pafy.backend_shared.BasePafy._bestthumb = None
+        pafy.backend_shared.BasePafy._content_available = _content_available
+        pafy.backend_shared.BasePafy.getbestthumb = _getbestthumb
+
+    def patch_process_streams(self):
+        # https://github.com/mps-youtube/pafy/pull/230
+        backend_youtube_dl.YtdlPafy._old_process_streams = backend_youtube_dl.YtdlPafy._process_streams
+        backend_youtube_dl.YtdlPafy._process_streams = _process_streams
+
+    def patch_insecure_streams(self):
+        # https://github.com/mps-youtube/pafy/pull/235
+        pafy.g.def_ydl_opts["prefer_insecure"] = False

spotdl/youtube_tools.py

@@ -18,7 +18,10 @@ pafy.g.opener.addheaders.append(("Range", "bytes=0-"))
 # More info: https://github.com/mps-youtube/pafy/pull/211
 if pafy.__version__ <= "0.5.4":
     from spotdl import patcher
-    patcher.patch_pafy()
+    pafy_patcher = patcher.PatchPafy()
+    pafy_patcher.patch_getbestthumb()
+    pafy_patcher.patch_process_streams()
+    pafy_patcher.patch_insecure_streams()
 
 
 def set_api_key():

test/test_download_with_metadata.py

@@ -176,6 +176,7 @@ class TestFFmpeg:
 
 
 class TestAvconv:
+    @pytest.mark.skip(reason="avconv is no longer provided with FFmpeg")
     def test_convert_from_m4a_to_mp3(self, filename_fixture, monkeypatch):
         monkeypatch.setattr("os.remove", lambda x: None)
         expect_command = "avconv -loglevel 0 -i {0}.m4a -ab 192k {0}.mp3 -y".format(

test/test_patcher.py

@@ -3,7 +3,8 @@ import pafy
 
 import pytest
 
-patcher.patch_pafy()
+pafy_patcher = patcher.PatchPafy()
+pafy_patcher.patch_getbestthumb()
 
 class TestPafyContentAvailable:
     pass

test/test_spotify_tools.py

@@ -115,7 +115,7 @@ def test_write_playlist(tmpdir):
     assert tracks == expect_tracks
 
 
-# XXX: Mock this test off if it fails in future
+# XXX: Monkeypatch these tests if they fail in future
 class TestFetchAlbum:
     @pytest.fixture(scope="module")
     def album_fixture(self):
@@ -131,7 +131,7 @@ class TestFetchAlbum:
         assert album_fixture["tracks"]["total"] == 15
 
 
-# XXX: Mock this test off if it fails in future
+# XXX: Monkeypatch these tests if they fail in future
 class TestFetchAlbumsFromArtist:
     @pytest.fixture(scope="module")
     def albums_from_artist_fixture(self):
@@ -141,7 +141,7 @@ class TestFetchAlbumsFromArtist:
         return albums
 
     def test_len(self, albums_from_artist_fixture):
-        assert len(albums_from_artist_fixture) == 52
+        assert len(albums_from_artist_fixture) == 53
 
     def test_zeroth_album_name(self, albums_from_artist_fixture):
         assert albums_from_artist_fixture[0]["name"] == "Revolution Radio"