Merge pull request #380 from ritiek/duplicates-in-file

Overwrite track file with unique tracks

spotdl/internals.py

@@ -158,6 +158,23 @@ def get_splits(url):
     return splits
 
 
+def get_unique_tracks(text_file):
+    """
+    Returns a list of unique tracks given a path to a
+    file containing tracks.
+    """
+
+    with open(text_file, 'r') as listed:
+        # Read tracks into a list and remove any duplicates
+        lines = listed.read().splitlines()
+
+    # Remove blank and strip whitespaces from lines (if any)
+    lines = [line.strip() for line in lines if line.strip()]
+    lines = remove_duplicates(lines)
+
+    return lines
+
+
 # a hacky way to user's localized music directory
 # (thanks @linusg, issue #203)
 def get_music_dir():

spotdl/spotdl.py

@@ -65,15 +65,13 @@ def check_exists(music_file, raw_song, meta_tags):
 
 def download_list(text_file):
     """ Download all songs from the list. """
-    with open(text_file, 'r') as listed:
-        # read tracks into a list and remove any duplicates
-        lines = listed.read().splitlines()
-        lines = internals.remove_duplicates(lines)
-    # ignore blank lines in text_file (if any)
-    try:
-        lines.remove('')
-    except ValueError:
-        pass
+
+    log.info('Checking and removing any duplicate tracks')
+    lines = internals.get_unique_tracks(text_file)
+
+    # override file with unique tracks
+    with open(text_file, 'w') as listed:
+        listed.write('\n'.join(lines))
 
     log.info(u'Preparing to download {} songs'.format(len(lines)))
     downloaded_songs = []

test/test_internals.py

@@ -80,3 +80,39 @@ class TestGetSeconds:
             internals.get_sec('10*05')
         with pytest.raises(ValueError):
             internals.get_sec('02,28,46')
+
+
+duplicate_tracks_test_table = [
+    (('https://open.spotify.com/track/2DGa7iaidT5s0qnINlwMjJ',
+      'https://open.spotify.com/track/2DGa7iaidT5s0qnINlwMjJ'),
+     ('https://open.spotify.com/track/2DGa7iaidT5s0qnINlwMjJ',)),
+
+    (('https://open.spotify.com/track/2DGa7iaidT5s0qnINlwMjJ',
+      '',
+      'https://open.spotify.com/track/3SipFlNddvL0XNZRLXvdZD'),
+     ('https://open.spotify.com/track/2DGa7iaidT5s0qnINlwMjJ',
+      'https://open.spotify.com/track/3SipFlNddvL0XNZRLXvdZD')),
+
+    (('ncs fade',
+      'https://open.spotify.com/track/2DGa7iaidT5s0qnINlwMjJ',
+      '',
+      'ncs fade'),
+     ('ncs fade',
+      'https://open.spotify.com/track/2DGa7iaidT5s0qnINlwMjJ')),
+
+    (('ncs spectre ',
+      '  https://open.spotify.com/track/2DGa7iaidT5s0qnINlwMjJ',
+      ''),
+     ('ncs spectre',
+      'https://open.spotify.com/track/2DGa7iaidT5s0qnINlwMjJ'))
+]
+
+
+@pytest.mark.parametrize("duplicates, expected", duplicate_tracks_test_table)
+def test_get_unique_tracks(tmpdir, duplicates, expected):
+    file_path = os.path.join(str(tmpdir), 'test_duplicates.txt')
+    with open(file_path, 'w') as tin:
+        tin.write('\n'.join(duplicates))
+
+    unique_tracks = internals.get_unique_tracks(file_path)
+    assert tuple(unique_tracks) == expected