From 3affdc830a10adfcee549b6eab6967fabc4c7cff Mon Sep 17 00:00:00 2001 From: Nitesh Sawant Date: Mon, 15 Jan 2018 23:50:19 +0530 Subject: [PATCH 01/20] Tell the user to install `unicode-slugify` in case of ImportError --- core/internals.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/core/internals.py b/core/internals.py index ca7f6ed..800567f 100755 --- a/core/internals.py +++ b/core/internals.py @@ -1,6 +1,10 @@ -from slugify import SLUG_OK, slugify from core.const import log +try: + from slugify import SLUG_OK, slugify +except ImportError: + log.warning('Remove any other slugifies and install unicode-slugify') + import os @@ -68,11 +72,11 @@ def filter_path(path): def videotime_from_seconds(time): - if time<60: + if time < 60: return str(time) - if time<3600: - return '{}:{}'.format(str(time//60), str(time%60).zfill(2)) + if time < 3600: + return '{}:{}'.format(str(time // 60), str(time % 60).zfill(2)) - return '{}:{}:{}'.format(str(time//60), - str((time%60)//60).zfill(2), - str((time%60)%60).zfill(2)) + return '{}:{}:{}'.format(str(time // 60), + str((time % 60) // 60).zfill(2), + str((time % 60) % 60).zfill(2)) From 7cccabc14571627186838767bfddc66cf4e20233 Mon Sep 17 00:00:00 2001 From: Nitesh Sawant Date: Tue, 23 Jan 2018 00:02:06 +0530 Subject: [PATCH 02/20] Changed code as per PR comments --- .vscode/settings.json | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..615aafb --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "python.pythonPath": "/usr/bin/python3" +} \ No newline at end of file From 9ed7347f03f947139fe5d3b6423570eb1a456647 Mon Sep 17 00:00:00 2001 From: Nitesh Sawant Date: Tue, 23 Jan 2018 00:06:08 +0530 Subject: [PATCH 03/20] Changes as per PR comments --- .vscode/settings.json | 3 --- core/internals.py | 10 ++++++---- 2 files changed, 6 insertions(+), 7 deletions(-) delete mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 615aafb..0000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "python.pythonPath": "/usr/bin/python3" -} \ No newline at end of file diff --git a/core/internals.py b/core/internals.py index a379cfe..5f14781 100755 --- a/core/internals.py +++ b/core/internals.py @@ -1,15 +1,17 @@ -from slugify import SLUG_OK, slugify +import sys from core import const +log = const.log + try: from slugify import SLUG_OK, slugify except ImportError: - log.warning('Remove any other slugifies and install unicode-slugify') + log.error('Oops! `unicode-slugify` was not found.') + log.info('Please remove any other slugify library and install `unicode-slugify`') + sys.exit(5) import os -log = const.log - formats = { 0 : 'track_name', 1 : 'artist', 2 : 'album', From 988427d88184ecbfc2aa203a36b53ea17ae1acc2 Mon Sep 17 00:00:00 2001 From: Nitesh Sawant Date: Tue, 23 Jan 2018 22:03:39 +0530 Subject: [PATCH 04/20] Fix merge conflict Changed videotime_from_seconds function to fix merge conflict. --- core/internals.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/core/internals.py b/core/internals.py index 5f14781..99e29e3 100755 --- a/core/internals.py +++ b/core/internals.py @@ -107,12 +107,13 @@ def filter_path(path): os.remove(os.path.join(path, temp)) + def videotime_from_seconds(time): if time < 60: return str(time) if time < 3600: - return '{}:{}'.format(str(time // 60), str(time % 60).zfill(2)) + return '{0}:{1:02}'.format(time//60, time % 60) - return '{}:{}:{}'.format(str(time // 60), - str((time % 60) // 60).zfill(2), - str((time % 60) % 60).zfill(2)) + return '{0}:{1:02}:{2:02}'.format((time//60)//60, + (time//60) % 60, + time % 60) From 6bd2a716668ba8337e1aa1c677a8a95744984477 Mon Sep 17 00:00:00 2001 From: Nitesh Sawant Date: Sun, 4 Feb 2018 15:46:08 +0530 Subject: [PATCH 05/20] Implemented passing config.yml as command line argument --- core/handle.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/core/handle.py b/core/handle.py index a2bb9a5..f6af0ea 100644 --- a/core/handle.py +++ b/core/handle.py @@ -1,4 +1,4 @@ -from core import internals +from core import internals,const import logging import yaml @@ -53,6 +53,27 @@ def get_config(config_file): return cfg['spotify-downloader'] +def override_config(config_file, parser, raw_args=None, ): + """ """ + config_file = os.path.join(sys.path[0], 'config.yml') + config = merge(default_conf['spotify-downloader'], get_config(config_file)) + + parser.set_defaults(avconv=config['avconv']) + parser.set_defaults(download_only_metadata=config['download-only-metadata']) + parser.set_defaults(dry_run=config['dry-run']) + parser.set_defaults(file_format=config['file-format']) + parser.set_defaults(folder=os.path.relpath(config['folder'], os.getcwd())) + parser.set_defaults(input_ext=config['input-ext']) + parser.set_defaults(log_level=config['log-level']) + parser.set_defaults(manual=config['manual']) + parser.set_defaults(music_videos_only=config['music-videos-only']) + parser.set_defaults(no_metadata=config['no-metadata']) + parser.set_defaults(no_spaces=config['no-spaces']) + parser.set_defaults(output_ext=config['output-ext']) + parser.set_defaults(overwrite=config['overwrite']) + + return parser.parse_args(raw_args) + def get_arguments(raw_args=None, to_group=True, to_merge=True): parser = argparse.ArgumentParser( description='Download and convert songs from Spotify, Youtube etc.', @@ -129,8 +150,16 @@ def get_arguments(raw_args=None, to_group=True, to_merge=True): choices=_LOG_LEVELS_STR, type=str.upper, help='set log verbosity') + parser.add_argument( + '-c', '--config', default=None, + help='Replace with custom config file' + ) parsed = parser.parse_args(raw_args) parsed.log_level = log_leveller(parsed.log_level) + if parsed.config is not None and to_merge: + print("Config file passed") + parsed = override_config(parsed.config,parser) + pass return parsed From e1ffa92b9c148c115eb24aeb7cd4c8611d86d9f2 Mon Sep 17 00:00:00 2001 From: Nitesh Sawant Date: Sun, 4 Feb 2018 16:01:37 +0530 Subject: [PATCH 06/20] wrote help description for def override_config --- core/handle.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/core/handle.py b/core/handle.py index f6af0ea..508a35e 100644 --- a/core/handle.py +++ b/core/handle.py @@ -54,8 +54,8 @@ def get_config(config_file): def override_config(config_file, parser, raw_args=None, ): - """ """ - config_file = os.path.join(sys.path[0], 'config.yml') + """ Override default dict with config dict passed as comamnd line argument. """ + config_file = os.path.realpath(config_file) config = merge(default_conf['spotify-downloader'], get_config(config_file)) parser.set_defaults(avconv=config['avconv']) @@ -159,7 +159,6 @@ def get_arguments(raw_args=None, to_group=True, to_merge=True): parsed.log_level = log_leveller(parsed.log_level) if parsed.config is not None and to_merge: - print("Config file passed") parsed = override_config(parsed.config,parser) pass return parsed From 8550abd06a1052b12bf88e25bd0d99fce50714bf Mon Sep 17 00:00:00 2001 From: Nitesh Sawant Date: Sun, 4 Feb 2018 16:07:39 +0530 Subject: [PATCH 07/20] Removed unnecessary imports --- core/handle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/handle.py b/core/handle.py index 508a35e..4b7366d 100644 --- a/core/handle.py +++ b/core/handle.py @@ -1,4 +1,4 @@ -from core import internals,const +from core import internals import logging import yaml From 0e3249646fed51fe75fbecda01ba6fb0b0ff6b83 Mon Sep 17 00:00:00 2001 From: Nitesh Sawant Date: Sun, 4 Feb 2018 22:36:04 +0530 Subject: [PATCH 08/20] Made Changes as per comments on PR removed the unnecessary comma and space at the end! put the closing bracket on the previous line, as it's done with all the other parser.add_argument calls. remove the pass - it's completely unneccesary. --- README.md | 7 +++++++ core/handle.py | 11 +++++------ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 3235d0f..f3e1a21 100755 --- a/README.md +++ b/README.md @@ -140,6 +140,8 @@ optional arguments: (default: False) -ll {INFO,WARNING,ERROR,DEBUG}, --log-level {INFO,WARNING,ERROR,DEBUG} set log verbosity (default: INFO) + -c CONFIG_FILE_PATH --config CONFIG_FILE_PATH + path to custom config.yml file ``` #### Download by Name @@ -261,6 +263,11 @@ to override any default options. Also note that config options are overridden by command-line arguments. +#### Specify the Custom Config File Path + +If you want to use custom `config.yml` instead of default one, you can use `-c`/`--config` option. +E.g. `$ python3 spotdl.py -s "adele hello" -c "/home/user/customConfig.yml"` + ## [Docker Image](https://hub.docker.com/r/ritiek/spotify-downloader/) [![Docker automated build](https://img.shields.io/docker/automated/jrottenberg/ffmpeg.svg)](https://hub.docker.com/r/ritiek/spotify-downloader) [![Docker pulls](https://img.shields.io/docker/pulls/ritiek/spotify-downloader.svg)](https://hub.docker.com/r/ritiek/spotify-downloader) diff --git a/core/handle.py b/core/handle.py index 4b7366d..1d8c695 100644 --- a/core/handle.py +++ b/core/handle.py @@ -40,7 +40,6 @@ def merge(default, config): merged.update(config) return merged - def get_config(config_file): try: with open(config_file, 'r') as ymlfile: @@ -53,7 +52,7 @@ def get_config(config_file): return cfg['spotify-downloader'] -def override_config(config_file, parser, raw_args=None, ): +def override_config(config_file, parser, raw_args=None): """ Override default dict with config dict passed as comamnd line argument. """ config_file = os.path.realpath(config_file) config = merge(default_conf['spotify-downloader'], get_config(config_file)) @@ -152,13 +151,13 @@ def get_arguments(raw_args=None, to_group=True, to_merge=True): help='set log verbosity') parser.add_argument( '-c', '--config', default=None, - help='Replace with custom config file' - ) + help='Replace with custom config.yml file') parsed = parser.parse_args(raw_args) - parsed.log_level = log_leveller(parsed.log_level) if parsed.config is not None and to_merge: parsed = override_config(parsed.config,parser) - pass + + parsed.log_level = log_leveller(parsed.log_level) + return parsed From a8f261edaedd5a2ac9838e309a51772291da1dcf Mon Sep 17 00:00:00 2001 From: Nitesh Sawant Date: Mon, 5 Feb 2018 23:01:44 +0530 Subject: [PATCH 09/20] Changes as per second PR review --- README.md | 4 ++-- core/handle.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f3e1a21..b975f82 100755 --- a/README.md +++ b/README.md @@ -141,7 +141,7 @@ optional arguments: -ll {INFO,WARNING,ERROR,DEBUG}, --log-level {INFO,WARNING,ERROR,DEBUG} set log verbosity (default: INFO) -c CONFIG_FILE_PATH --config CONFIG_FILE_PATH - path to custom config.yml file + Replace with custom config.yml file (default: None) ``` #### Download by Name @@ -265,7 +265,7 @@ Also note that config options are overridden by command-line arguments. #### Specify the Custom Config File Path -If you want to use custom `config.yml` instead of default one, you can use `-c`/`--config` option. +If you want to use custom `.yml` configuration instead of the default one, you can use `-c`/`--config` option. E.g. `$ python3 spotdl.py -s "adele hello" -c "/home/user/customConfig.yml"` ## [Docker Image](https://hub.docker.com/r/ritiek/spotify-downloader/) diff --git a/core/handle.py b/core/handle.py index 1d8c695..9eaa817 100644 --- a/core/handle.py +++ b/core/handle.py @@ -40,6 +40,7 @@ def merge(default, config): merged.update(config) return merged + def get_config(config_file): try: with open(config_file, 'r') as ymlfile: @@ -73,6 +74,7 @@ def override_config(config_file, parser, raw_args=None): return parser.parse_args(raw_args) + def get_arguments(raw_args=None, to_group=True, to_merge=True): parser = argparse.ArgumentParser( description='Download and convert songs from Spotify, Youtube etc.', From c8db9ed5da83fe9f13c8eefc4d73347a89b04b3e Mon Sep 17 00:00:00 2001 From: ritiek Date: Tue, 13 Feb 2018 18:03:50 +0530 Subject: [PATCH 10/20] Improve conversion --- core/convert.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/core/convert.py b/core/convert.py index af7cffd..cb18d2f 100644 --- a/core/convert.py +++ b/core/convert.py @@ -58,16 +58,18 @@ class Converter: if input_ext == '.m4a': if output_ext == '.mp3': - ffmpeg_params = '-codec:v copy -codec:a libmp3lame -q:a 2 ' + ffmpeg_params = '-codec:v copy -codec:a libmp3lame -ar 44100 ' elif output_ext == '.webm': - ffmpeg_params = '-c:a libopus -vbr on -b:a 192k -vn ' + ffmpeg_params = '-codec:a libopus -vbr on ' elif input_ext == '.webm': if output_ext == '.mp3': - ffmpeg_params = ' -ab 192k -ar 44100 -vn ' + ffmpeg_params = '-codec:a libmp3lame -ar 44100 ' elif output_ext == '.m4a': - ffmpeg_params = '-cutoff 20000 -c:a libfdk_aac -b:a 192k -vn ' + ffmpeg_params = '-cutoff 20000 -codec:a libfdk_aac -ar 44100 ' + # add common params for any of the above combination + ffmpeg_params += '-b:a 192k -vn ' ffmpeg_pre += ' -i' command = ffmpeg_pre.split() + [self.input_file] + ffmpeg_params.split() + [self.output_file] From 666334dfd82e38fd96577d36a33e4464b69f6551 Mon Sep 17 00:00:00 2001 From: Ritiek Malhotra Date: Sat, 24 Feb 2018 14:41:49 +0530 Subject: [PATCH 11/20] Install codecov after success --- .travis.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index e78439d..11d1e2f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,6 @@ python: - "3.6" before_install: - pip install tinydownload - - pip install codecov - pip install pytest-cov addons: apt: @@ -38,4 +37,6 @@ install: - tinydownload 22684734659253158385 -o ~/bin/ffmpeg - chmod 755 ~/bin/ffmpeg script: python -m pytest test --cov=. -after_success: codecov +after_success: + - pip install codecov + - codecov From c4bb047187a9ef7f22c5299dc15254c7c022df19 Mon Sep 17 00:00:00 2001 From: Ritiek Malhotra Date: Sat, 24 Feb 2018 15:03:42 +0530 Subject: [PATCH 12/20] Test URL is now None --- test/test_without_metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_without_metadata.py b/test/test_without_metadata.py index 8027ea1..0757ac2 100644 --- a/test/test_without_metadata.py +++ b/test/test_without_metadata.py @@ -22,7 +22,7 @@ def test_metadata(): class TestYouTubeURL: def test_only_music_category(self): - expect_url = 'http://youtube.com/watch?v=P11ou3CXKZo' + expect_url = None const.args.music_videos_only = True url = youtube_tools.generate_youtube_url(raw_song, metadata) assert url == expect_url From f943080edb7706cdfbc1478954c396dd803a0a35 Mon Sep 17 00:00:00 2001 From: ritiek Date: Sun, 25 Feb 2018 03:34:59 +0530 Subject: [PATCH 13/20] Partially fix download speed throttle --- core/youtube_tools.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/youtube_tools.py b/core/youtube_tools.py index 4dda0ac..0e736e9 100644 --- a/core/youtube_tools.py +++ b/core/youtube_tools.py @@ -9,6 +9,9 @@ import pprint log = const.log # Please respect this YouTube token :) pafy.set_api_key('AIzaSyAnItl3udec-Q1d5bkjKJGL-RgrKO_vU90') +# Fix download speed throttle on short duration tracks +# Read more on mps-youtube/pafy#199 +pafy.g.opener.addheaders.append(('Range', 'bytes=0-')) def go_pafy(raw_song, meta_tags=None): From 4ad77de97f0570d21a8a8107f86c9356332ff945 Mon Sep 17 00:00:00 2001 From: Vishnunarayan K I <31964688+vn-ki@users.noreply.github.com> Date: Fri, 9 Mar 2018 13:17:46 +0530 Subject: [PATCH 14/20] Filter out items other than videos in search (#249) --- core/youtube_tools.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/youtube_tools.py b/core/youtube_tools.py index 0e736e9..91c9e4f 100644 --- a/core/youtube_tools.py +++ b/core/youtube_tools.py @@ -82,6 +82,8 @@ def generate_youtube_url(raw_song, meta_tags, tries_remaining=5): log.debug('query: {0}'.format(query)) data = pafy.call_gdata('search', query) + data['items'] = list(filter(lambda x: x['id'].get('videoId') is not None, + data['items'])) query_results = {'part': 'contentDetails,snippet,statistics', 'maxResults': 50, 'id': ','.join(i['id']['videoId'] for i in data['items'])} From 46f313777b5632f43ce26f6cfcdfb48c854c18fd Mon Sep 17 00:00:00 2001 From: Ritiek Malhotra Date: Fri, 9 Mar 2018 16:10:59 +0530 Subject: [PATCH 15/20] Update music only URL --- test/test_without_metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_without_metadata.py b/test/test_without_metadata.py index 0757ac2..45f7e0d 100644 --- a/test/test_without_metadata.py +++ b/test/test_without_metadata.py @@ -22,7 +22,7 @@ def test_metadata(): class TestYouTubeURL: def test_only_music_category(self): - expect_url = None + expect_url = 'http://youtube.com/watch?v=5USR1Omo7f0' const.args.music_videos_only = True url = youtube_tools.generate_youtube_url(raw_song, metadata) assert url == expect_url From b968b5d206e4f32f6d7d49b2b9aa6be97e7f8d17 Mon Sep 17 00:00:00 2001 From: Ritiek Malhotra Date: Fri, 9 Mar 2018 20:40:15 +0530 Subject: [PATCH 16/20] Scrape YouTube by default and optionally use YouTube API to perform searches (#250) * YouTube scraping * Cleanup GenerateYouTubeURL class * Some minor improvements * Add test to fetch title with and without api key --- .gitignore | 1 + README.md | 11 +- core/handle.py | 35 +++-- core/internals.py | 15 +- core/youtube_tools.py | 273 +++++++++++++++++++++++----------- spotdl.py | 1 + test/test_without_metadata.py | 31 +++- 7 files changed, 254 insertions(+), 113 deletions(-) diff --git a/.gitignore b/.gitignore index 39259af..7e56bc1 100755 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,5 @@ Music/ *.pyc __pycache__/ .cache/ +.pytest_cache/ .python-version diff --git a/README.md b/README.md index b975f82..f34d293 100755 --- a/README.md +++ b/README.md @@ -141,7 +141,7 @@ optional arguments: -ll {INFO,WARNING,ERROR,DEBUG}, --log-level {INFO,WARNING,ERROR,DEBUG} set log verbosity (default: INFO) -c CONFIG_FILE_PATH --config CONFIG_FILE_PATH - Replace with custom config.yml file (default: None) + Replace with custom config.yml file (default: None) ``` #### Download by Name @@ -263,11 +263,16 @@ to override any default options. Also note that config options are overridden by command-line arguments. -#### Specify the Custom Config File Path - If you want to use custom `.yml` configuration instead of the default one, you can use `-c`/`--config` option. E.g. `$ python3 spotdl.py -s "adele hello" -c "/home/user/customConfig.yml"` +## Set YouTube API Key + +By default this tool will scrape YouTube to fetch for matching video tracks. +However, you can optionally use YouTube API for faster response time. +To do this, [generate your API key](https://developers.google.com/youtube/registering_an_application) +and then set it in your `config.yml`. + ## [Docker Image](https://hub.docker.com/r/ritiek/spotify-downloader/) [![Docker automated build](https://img.shields.io/docker/automated/jrottenberg/ffmpeg.svg)](https://hub.docker.com/r/ritiek/spotify-downloader) [![Docker pulls](https://img.shields.io/docker/pulls/ritiek/spotify-downloader.svg)](https://hub.docker.com/r/ritiek/spotify-downloader) diff --git a/core/handle.py b/core/handle.py index 9eaa817..19e1bec 100644 --- a/core/handle.py +++ b/core/handle.py @@ -23,6 +23,7 @@ default_conf = { 'spotify-downloader': 'music-videos-only' : False, 'no-spaces' : False, 'file-format' : '{artist} - {track_name}', + 'youtube-api-key' : None, 'log-level' : 'INFO' } } @@ -40,7 +41,7 @@ def merge(default, config): merged.update(config) return merged - + def get_config(config_file): try: with open(config_file, 'r') as ymlfile: @@ -57,21 +58,22 @@ def override_config(config_file, parser, raw_args=None): """ Override default dict with config dict passed as comamnd line argument. """ config_file = os.path.realpath(config_file) config = merge(default_conf['spotify-downloader'], get_config(config_file)) - + + parser.set_defaults(manual=config['manual']) + parser.set_defaults(no_metadata=config['no-metadata']) parser.set_defaults(avconv=config['avconv']) + parser.set_defaults(folder=os.path.relpath(config['folder'], os.getcwd())) + parser.set_defaults(overwrite=config['overwrite']) + parser.set_defaults(input_ext=config['input-ext']) + parser.set_defaults(output_ext=config['output-ext']) parser.set_defaults(download_only_metadata=config['download-only-metadata']) parser.set_defaults(dry_run=config['dry-run']) - parser.set_defaults(file_format=config['file-format']) - parser.set_defaults(folder=os.path.relpath(config['folder'], os.getcwd())) - parser.set_defaults(input_ext=config['input-ext']) - parser.set_defaults(log_level=config['log-level']) - parser.set_defaults(manual=config['manual']) parser.set_defaults(music_videos_only=config['music-videos-only']) - parser.set_defaults(no_metadata=config['no-metadata']) parser.set_defaults(no_spaces=config['no-spaces']) - parser.set_defaults(output_ext=config['output-ext']) - parser.set_defaults(overwrite=config['overwrite']) - + parser.set_defaults(file_format=config['file-format']) + parser.set_defaults(no_spaces=config['youtube-api-key']) + parser.set_defaults(log_level=config['log-level']) + return parser.parse_args(raw_args) @@ -151,15 +153,18 @@ def get_arguments(raw_args=None, to_group=True, to_merge=True): choices=_LOG_LEVELS_STR, type=str.upper, help='set log verbosity') + parser.add_argument( + '-yk', '--youtube-api-key', default=config['youtube-api-key'], + help=argparse.SUPPRESS) parser.add_argument( '-c', '--config', default=None, - help='Replace with custom config.yml file') + help='Replace with custom config.yml file') parsed = parser.parse_args(raw_args) if parsed.config is not None and to_merge: - parsed = override_config(parsed.config,parser) - + parsed = override_config(parsed.config, parser) + parsed.log_level = log_leveller(parsed.log_level) - + return parsed diff --git a/core/internals.py b/core/internals.py index 3c3e1f3..8f8f6d9 100755 --- a/core/internals.py +++ b/core/internals.py @@ -120,6 +120,19 @@ def videotime_from_seconds(time): return '{0}:{1:02}:{2:02}'.format((time//60)//60, (time//60) % 60, time % 60) +def get_sec(time_str): + v = time_str.split(':', 3) + v.reverse() + sec = 0 + if len(v) > 0: # seconds + sec += int(v[0]) + if len(v) > 1: # minutes + sec += int(v[1]) * 60 + if len(v) > 2: # hours + sec += int(v[2]) * 3600 + return sec + + def get_splits(url): if '/' in url: if url.endswith('/'): @@ -127,4 +140,4 @@ def get_splits(url): splits = url.split('/') else: splits = url.split(':') - return splits \ No newline at end of file + return splits diff --git a/core/youtube_tools.py b/core/youtube_tools.py index 91c9e4f..8d6b01e 100644 --- a/core/youtube_tools.py +++ b/core/youtube_tools.py @@ -1,3 +1,5 @@ +from bs4 import BeautifulSoup +import urllib import pafy from core import internals @@ -7,13 +9,21 @@ import os import pprint log = const.log -# Please respect this YouTube token :) -pafy.set_api_key('AIzaSyAnItl3udec-Q1d5bkjKJGL-RgrKO_vU90') + # Fix download speed throttle on short duration tracks # Read more on mps-youtube/pafy#199 pafy.g.opener.addheaders.append(('Range', 'bytes=0-')) +def set_api_key(): + if const.args.youtube_api_key: + key = const.args.youtube_api_key + else: + # Please respect this YouTube token :) + key = 'AIzaSyAnItl3udec-Q1d5bkjKJGL-RgrKO_vU90' + pafy.set_api_key(key) + + def go_pafy(raw_song, meta_tags=None): """ Parse track from YouTube. """ if internals.is_youtube(raw_song): @@ -58,92 +68,175 @@ def download_song(file_name, content): return False -def generate_youtube_url(raw_song, meta_tags, tries_remaining=5): - """ Search for the song on YouTube and generate a URL to its video. """ - # prevents an infinite loop but allows for a few retries - if tries_remaining == 0: - log.debug('No tries left. I quit.') - return - - query = { 'part' : 'snippet', - 'maxResults' : 50, - 'type' : 'video' } - - if const.args.music_videos_only: - query['videoCategoryId'] = '10' - - if not meta_tags: - song = raw_song - query['q'] = song - else: - song = '{0} - {1}'.format(meta_tags['artists'][0]['name'], - meta_tags['name']) - query['q'] = song - log.debug('query: {0}'.format(query)) - - data = pafy.call_gdata('search', query) - data['items'] = list(filter(lambda x: x['id'].get('videoId') is not None, - data['items'])) - query_results = {'part': 'contentDetails,snippet,statistics', - 'maxResults': 50, - 'id': ','.join(i['id']['videoId'] for i in data['items'])} - log.debug('query_results: {0}'.format(query_results)) - - vdata = pafy.call_gdata('videos', query_results) - - videos = [] - for x in vdata['items']: - duration_s = pafy.playlist.parseISO8591(x['contentDetails']['duration']) - youtubedetails = {'link': x['id'], 'title': x['snippet']['title'], - 'videotime':internals.videotime_from_seconds(duration_s), - 'seconds': duration_s} - videos.append(youtubedetails) - if not meta_tags: - break - - if not videos: - return None - - if const.args.manual: - log.info(song) - log.info('0. Skip downloading this song.\n') - # fetch all video links on first page on YouTube - for i, v in enumerate(videos): - log.info(u'{0}. {1} {2} {3}'.format(i+1, v['title'], v['videotime'], - "http://youtube.com/watch?v="+v['link'])) - # let user select the song to download - result = internals.input_link(videos) - if not result: - return None - else: - if not meta_tags: - # if the metadata could not be acquired, take the first result - # from Youtube because the proper song length is unknown - result = videos[0] - log.debug('Since no metadata found on Spotify, going with the first result') - else: - # filter out videos that do not have a similar length to the Spotify song - duration_tolerance = 10 - max_duration_tolerance = 20 - possible_videos_by_duration = list() - - ''' - start with a reasonable duration_tolerance, and increment duration_tolerance - until one of the Youtube results falls within the correct duration or - the duration_tolerance has reached the max_duration_tolerance - ''' - while len(possible_videos_by_duration) == 0: - possible_videos_by_duration = list(filter(lambda x: abs(x['seconds'] - meta_tags['duration']) <= duration_tolerance, videos)) - duration_tolerance += 1 - if duration_tolerance > max_duration_tolerance: - log.error("{0} by {1} was not found.\n".format(meta_tags['name'], meta_tags['artists'][0]['name'])) - return None - - result = possible_videos_by_duration[0] - - if result: - url = "http://youtube.com/watch?v=" + result['link'] - else: - url = None - +def generate_search_url(song): + """ Generate YouTube search URL for the given song. """ + # urllib.request.quote() encodes URL with special characters + song = urllib.request.quote(song) + # Special YouTube URL filter to search only for videos + url = 'https://www.youtube.com/results?sp=EgIQAQ%253D%253D&q={0}'.format(song) return url + + +def is_video(result): + # ensure result is not a channel + not_video = result.find('channel') is not None or \ + 'yt-lockup-channel' in result.parent.attrs['class'] or \ + 'yt-lockup-channel' in result.attrs['class'] + + # ensure result is not a mix/playlist + not_video = not_video or \ + 'yt-lockup-playlist' in result.parent.attrs['class'] + + # ensure video result is not an advertisement + not_video = not_video or \ + result.find('googleads') is not None + + video = not not_video + return video + + +def generate_youtube_url(raw_song, meta_tags): + url_fetch = GenerateYouTubeURL(raw_song, meta_tags) + if const.args.youtube_api_key: + url = url_fetch.api() + else: + url = url_fetch.scrape() + return url + + +class GenerateYouTubeURL: + def __init__(self, raw_song, meta_tags): + self.raw_song = raw_song + self.meta_tags = meta_tags + + def _best_match(self, videos): + """ Select the best matching video from a list of videos. """ + if const.args.manual: + log.info(self.raw_song) + log.info('0. Skip downloading this song.\n') + # fetch all video links on first page on YouTube + for i, v in enumerate(videos): + log.info(u'{0}. {1} {2} {3}'.format(i+1, v['title'], v['videotime'], + "http://youtube.com/watch?v="+v['link'])) + # let user select the song to download + result = internals.input_link(videos) + if result is None: + return None + else: + if not self.meta_tags: + # if the metadata could not be acquired, take the first result + # from Youtube because the proper song length is unknown + result = videos[0] + log.debug('Since no metadata found on Spotify, going with the first result') + else: + # filter out videos that do not have a similar length to the Spotify song + duration_tolerance = 10 + max_duration_tolerance = 20 + possible_videos_by_duration = list() + + ''' + start with a reasonable duration_tolerance, and increment duration_tolerance + until one of the Youtube results falls within the correct duration or + the duration_tolerance has reached the max_duration_tolerance + ''' + while len(possible_videos_by_duration) == 0: + possible_videos_by_duration = list(filter(lambda x: abs(x['seconds'] - self.meta_tags['duration']) <= duration_tolerance, videos)) + duration_tolerance += 1 + if duration_tolerance > max_duration_tolerance: + log.error("{0} by {1} was not found.\n".format(self.meta_tags['name'], self.meta_tags['artists'][0]['name'])) + return None + + result = possible_videos_by_duration[0] + + if result: + url = "http://youtube.com/watch?v=" + result['link'] + else: + url = None + + return url + + def scrape(self, tries_remaining=5): + """ Search and scrape YouTube to return a list of matching videos. """ + + # prevents an infinite loop but allows for a few retries + if tries_remaining == 0: + log.debug('No tries left. I quit.') + return + + if self.meta_tags is None: + song = self.raw_song + search_url = generate_search_url(song) + else: + song = internals.generate_songname(const.args.file_format, + self.meta_tags) + search_url = generate_search_url(song) + log.debug('Opening URL: {0}'.format(search_url)) + + item = urllib.request.urlopen(search_url).read() + items_parse = BeautifulSoup(item, "html.parser") + + videos = [] + for x in items_parse.find_all('div', {'class': 'yt-lockup-dismissable yt-uix-tile'}): + + if not is_video(x): + continue + + y = x.find('div', class_='yt-lockup-content') + link = y.find('a')['href'][-11:] + title = y.find('a')['title'] + + try: + videotime = x.find('span', class_="video-time").get_text() + except AttributeError: + log.debug('Could not find video duration on YouTube, retrying..') + return generate_youtube_url(self.raw_song, self.meta_tags, tries_remaining - 1) + + youtubedetails = {'link': link, 'title': title, 'videotime': videotime, + 'seconds': internals.get_sec(videotime)} + videos.append(youtubedetails) + if self.meta_tags is None: + break + + return self._best_match(videos) + + + def api(self): + """ Use YouTube API to search and return a list of matching videos. """ + + query = { 'part' : 'snippet', + 'maxResults' : 50, + 'type' : 'video' } + + if const.args.music_videos_only: + query['videoCategoryId'] = '10' + + if not self.meta_tags: + song = self.raw_song + query['q'] = song + else: + song = '{0} - {1}'.format(self.meta_tags['artists'][0]['name'], + self.meta_tags['name']) + query['q'] = song + log.debug('query: {0}'.format(query)) + + data = pafy.call_gdata('search', query) + data['items'] = list(filter(lambda x: x['id'].get('videoId') is not None, + data['items'])) + query_results = {'part': 'contentDetails,snippet,statistics', + 'maxResults': 50, + 'id': ','.join(i['id']['videoId'] for i in data['items'])} + log.debug('query_results: {0}'.format(query_results)) + + vdata = pafy.call_gdata('videos', query_results) + + videos = [] + for x in vdata['items']: + duration_s = pafy.playlist.parseISO8591(x['contentDetails']['duration']) + youtubedetails = {'link': x['id'], 'title': x['snippet']['title'], + 'videotime':internals.videotime_from_seconds(duration_s), + 'seconds': duration_s} + videos.append(youtubedetails) + if not self.meta_tags: + break + + return self._best_match(videos) diff --git a/spotdl.py b/spotdl.py index f175680..6f5380d 100755 --- a/spotdl.py +++ b/spotdl.py @@ -170,6 +170,7 @@ def download_single(raw_song, number=None): if __name__ == '__main__': const.args = handle.get_arguments() internals.filter_path(const.args.folder) + youtube_tools.set_api_key() const.log = const.logzero.setup_logger(formatter=const.formatter, level=const.args.log_level) diff --git a/test/test_without_metadata.py b/test/test_without_metadata.py index 45f7e0d..225f36f 100644 --- a/test/test_without_metadata.py +++ b/test/test_without_metadata.py @@ -13,6 +13,22 @@ loader.load_defaults() raw_song = "Tony's Videos VERY SHORT VIDEO 28.10.2016" +class TestYouTubeAPIKeys: + def test_custom(self): + expect_key = 'some_api_key' + const.args.youtube_api_key = expect_key + youtube_tools.set_api_key() + key = youtube_tools.pafy.g.api_key + assert key == expect_key + + def test_default(self): + expect_key = 'AIzaSyAnItl3udec-Q1d5bkjKJGL-RgrKO_vU90' + const.args.youtube_api_key = None + youtube_tools.set_api_key() + key = youtube_tools.pafy.g.api_key + assert key == expect_key + + def test_metadata(): expect_metadata = None global metadata @@ -22,10 +38,12 @@ def test_metadata(): class TestYouTubeURL: def test_only_music_category(self): - expect_url = 'http://youtube.com/watch?v=5USR1Omo7f0' + # YouTube keeps changing its results + expect_urls = ('http://youtube.com/watch?v=qOOcy2-tmbk', + 'http://youtube.com/watch?v=5USR1Omo7f0') const.args.music_videos_only = True url = youtube_tools.generate_youtube_url(raw_song, metadata) - assert url == expect_url + assert url in expect_urls def test_all_categories(self): expect_url = 'http://youtube.com/watch?v=qOOcy2-tmbk' @@ -49,16 +67,21 @@ class TestYouTubeURL: class TestYouTubeTitle: - def test_single_download(self): + def test_single_download_with_youtube_api(self): global content global title expect_title = "Tony's Videos VERY SHORT VIDEO 28.10.2016" + key = 'AIzaSyAnItl3udec-Q1d5bkjKJGL-RgrKO_vU90' + const.args.youtube_api_key = key + youtube_tools.set_api_key() content = youtube_tools.go_pafy(raw_song, metadata) title = youtube_tools.get_youtube_title(content) assert title == expect_title - def test_download_from_list(self): + def test_download_from_list_without_youtube_api(self): expect_title = "1. Tony's Videos VERY SHORT VIDEO 28.10.2016" + const.args.youtube_api_key = None + youtube_tools.set_api_key() content = youtube_tools.go_pafy(raw_song, metadata) title = youtube_tools.get_youtube_title(content, 1) assert title == expect_title From 2cc9a4a9d347af1f1af18284101a4f497c1c73a6 Mon Sep 17 00:00:00 2001 From: Ritiek Malhotra Date: Fri, 9 Mar 2018 20:52:46 +0530 Subject: [PATCH 17/20] Update YouTube API key to not conflict with users before #250 --- core/youtube_tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/youtube_tools.py b/core/youtube_tools.py index 8d6b01e..395d060 100644 --- a/core/youtube_tools.py +++ b/core/youtube_tools.py @@ -20,7 +20,7 @@ def set_api_key(): key = const.args.youtube_api_key else: # Please respect this YouTube token :) - key = 'AIzaSyAnItl3udec-Q1d5bkjKJGL-RgrKO_vU90' + key = 'AIzaSyC6cEeKlxtOPybk9sEe5ksFN5sB-7wzYp0' pafy.set_api_key(key) From c5846d615ea08708c411ed55c363f72ed3978436 Mon Sep 17 00:00:00 2001 From: Ritiek Malhotra Date: Fri, 9 Mar 2018 20:57:19 +0530 Subject: [PATCH 18/20] Update test for new API key --- test/test_without_metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_without_metadata.py b/test/test_without_metadata.py index 225f36f..dc7bf33 100644 --- a/test/test_without_metadata.py +++ b/test/test_without_metadata.py @@ -22,7 +22,7 @@ class TestYouTubeAPIKeys: assert key == expect_key def test_default(self): - expect_key = 'AIzaSyAnItl3udec-Q1d5bkjKJGL-RgrKO_vU90' + expect_key = 'AIzaSyC6cEeKlxtOPybk9sEe5ksFN5sB-7wzYp0' const.args.youtube_api_key = None youtube_tools.set_api_key() key = youtube_tools.pafy.g.api_key From a571ca2a38de688ea5f7d6fdc3536ee475eae1a2 Mon Sep 17 00:00:00 2001 From: Ritiek Malhotra Date: Sat, 10 Mar 2018 10:52:19 +0530 Subject: [PATCH 19/20] Add BeautifulSoup4 as a dependency --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 3927114..c69e4d0 100755 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,7 @@ youtube_dl >= 2017.5.1 pafy >= 0.5.3.1 spotipy >= 2.4.4 mutagen >= 1.37 +beautifulsoup4 >= 4.6.0 unicode-slugify >= 0.1.3 titlecase >= 0.10.0 logzero >= 1.3.1 From 7f7c3d6f582fbc8e4fed6ccc1b106879adaf1c89 Mon Sep 17 00:00:00 2001 From: Linus Groh Date: Sat, 10 Mar 2018 12:27:16 +0100 Subject: [PATCH 20/20] Update youtube_tools.py --- core/youtube_tools.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/core/youtube_tools.py b/core/youtube_tools.py index 395d060..a2d4c52 100644 --- a/core/youtube_tools.py +++ b/core/youtube_tools.py @@ -132,13 +132,11 @@ class GenerateYouTubeURL: # filter out videos that do not have a similar length to the Spotify song duration_tolerance = 10 max_duration_tolerance = 20 - possible_videos_by_duration = list() + possible_videos_by_duration = [] - ''' - start with a reasonable duration_tolerance, and increment duration_tolerance - until one of the Youtube results falls within the correct duration or - the duration_tolerance has reached the max_duration_tolerance - ''' + # start with a reasonable duration_tolerance, and increment duration_tolerance + # until one of the Youtube results falls within the correct duration or + # the duration_tolerance has reached the max_duration_tolerance while len(possible_videos_by_duration) == 0: possible_videos_by_duration = list(filter(lambda x: abs(x['seconds'] - self.meta_tags['duration']) <= duration_tolerance, videos)) duration_tolerance += 1 @@ -149,7 +147,7 @@ class GenerateYouTubeURL: result = possible_videos_by_duration[0] if result: - url = "http://youtube.com/watch?v=" + result['link'] + url = "http://youtube.com/watch?v={0}".format(result['link']) else: url = None