Switch to youtube API (#191)

* Switch to youtube API

* Fix test failures

* Minor fix

README.md

@@ -137,6 +137,8 @@ optional arguments:
                         prefered output extension .mp3 or .m4a (AAC) (default:
                         .mp3)
   -d, --dry-run         Show only track title and YouTube URL (default: False)
+  -mo, --music-videos-only
+                        Search only for music on Youtube (default: False)
   -ll {INFO,WARNING,ERROR,DEBUG}, --log-level {INFO,WARNING,ERROR,DEBUG}
                         set log verbosity (default: INFO)
 ```

core/internals.py

@@ -76,6 +76,10 @@ def get_arguments():
         '-d', '--dry-run', default=False,
         help='Show only track title and YouTube URL',
         action='store_true')
+    parser.add_argument(
+        '-mo', '--music-videos-only', default=False,
+        help='Search only for music on Youtube',
+        action='store_true')
     parser.add_argument(
         '-ll', '--log-level', default='INFO',
         choices=_LOG_LEVELS_STR,
@@ -122,18 +126,6 @@ def generate_token():
     return token
 
 
-def generate_search_url(song, viewsort=False):
-    """ Generate YouTube search URL for the given song. """
-    # urllib.request.quote() encodes URL with special characters
-    song = quote(song)
-    if viewsort:
-        url = u"https://www.youtube.com/results?q={0}".format(song)
-    else:
-        url = u"https://www.youtube.com/results?sp=EgIQAQ%253D%253D&q={0}".format(song)
-
-    return url
-
-
 def filter_path(path):
     if not os.path.exists(path):
         os.makedirs(path)
@@ -142,14 +134,12 @@ def filter_path(path):
             os.remove(os.path.join(path, temp))
 
 
-def get_sec(time_str):
-    v = time_str.split(':', 3)
-    v.reverse()
-    sec = 0
-    if len(v) > 0:  # seconds
-        sec += int(v[0])
-    if len(v) > 1:  # minutes
-        sec += int(v[1]) * 60
-    if len(v) > 2:  # hours
-        sec += int(v[2]) * 3600
-    return sec
+def videotime_from_seconds(time):
+    if time<60:
+        return str(time)
+    if time<3600:
+        return '{}:{}'.format(str(time//60), str(time%60).zfill(2))
+
+    return '{}:{}:{}'.format(str(time//60),
+                             str((time%60)//60).zfill(2),
+                             str((time%60)%60).zfill(2))

spotdl.py

@@ -87,35 +87,35 @@ def generate_youtube_url(raw_song, meta_tags, tries_remaining=5):
         log.debug('No tries left. I quit.')
         return
 
+    query = {'part': 'snippet',
+             'maxResults': 50,
+             'type': 'video'}
+
+    if args.music_videos_only:
+        query['videoCategoryId'] = '10'
+
     if meta_tags is None:
         song = raw_song
-        search_url = internals.generate_search_url(song, viewsort=False)
+        query['q'] = song
     else:
         song = generate_songname(meta_tags)
-        search_url = internals.generate_search_url(song, viewsort=True)
-    log.debug('Opening URL: {0}'.format(search_url))
+        query['q'] = song
+    log.debug('Query: {0}'.format(query))
 
-    item = urllib.request.urlopen(search_url).read()
-    items_parse = BeautifulSoup(item, "html.parser")
+    data = pafy.call_gdata('search', query)
+    query2 = {'part': 'contentDetails,snippet,statistics',
+              'maxResults': 50,
+              'id': ','.join(i['id']['videoId'] for i in data['items'])}
+    log.debug('Query2: {0}'.format(query2))
+
+    vdata = pafy.call_gdata('videos', query2)
 
     videos = []
-    for x in items_parse.find_all('div', {'class': 'yt-lockup-dismissable yt-uix-tile'}):
-
-        if not is_video(x):
-            continue
-
-        y = x.find('div', class_='yt-lockup-content')
-        link = y.find('a')['href']
-        title = y.find('a')['title']
-
-        try:
-            videotime = x.find('span', class_="video-time").get_text()
-        except AttributeError:
-            log.debug('Could not find video duration on YouTube, retrying..')
-            return generate_youtube_url(raw_song, meta_tags, tries_remaining - 1)
-
-        youtubedetails = {'link': link, 'title': title, 'videotime': videotime,
-                          'seconds': internals.get_sec(videotime)}
+    for x in vdata['items']:
+        duration_s = pafy.playlist.parseISO8591(x['contentDetails']['duration'])
+        youtubedetails = {'link': x['id'], 'title': x['snippet']['title'],
+                          'videotime':internals.videotime_from_seconds(duration_s),
+                          'seconds': duration_s}
         videos.append(youtubedetails)
         if meta_tags is None:
             break
@@ -131,7 +131,7 @@ def generate_youtube_url(raw_song, meta_tags, tries_remaining=5):
         # fetch all video links on first page on YouTube
         for i, v in enumerate(videos):
             log.info(u'{0}. {1} {2} {3}'.format(i+1, v['title'], v['videotime'],
-                  "http://youtube.com"+v['link']))
+                  "http://youtube.com/watch?v="+v['link']))
         # let user select the song to download
         result = internals.input_link(videos)
         if result is None:
@@ -163,11 +163,11 @@ def generate_youtube_url(raw_song, meta_tags, tries_remaining=5):
             result = possible_videos_by_duration[0]
 
     if result:
-        full_link = u'http://youtube.com{0}'.format(result['link'])
+        url = "http://youtube.com/watch?v=" + result['link']
     else:
-        full_link = None
+        url = None
 
-    return full_link
+    return  url
 
 
 def go_pafy(raw_song, meta_tags=None):

test/test_simple.py

@@ -14,6 +14,7 @@ class TestArgs:
     folder = 'test'
     log_level = logger.logging.DEBUG
     overwrite = 'skip'
+    music_videos_only = False
 
 test_args = TestArgs()
 setattr(spotdl, "args", test_args)

test/test_spotify.py

@@ -14,6 +14,7 @@ class TestArgs:
     folder = 'test'
     log_level = 'DEBUG'
     overwrite = 'skip'
+    music_videos_only = False
 
 test_args = TestArgs()
 setattr(spotdl, "args", test_args)