From 9c82ece6665a039080dc262822cf1c83c9de28fa Mon Sep 17 00:00:00 2001 From: KevinMidboe Date: Fri, 11 May 2018 16:57:46 +0200 Subject: [PATCH 01/98] Updated readme and knowledgebase. --- README.md | 12 +++++++++++- knowledgeBase.md | 5 ++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e7cc04f..79dd1da 100644 --- a/README.md +++ b/README.md @@ -16,4 +16,14 @@ There are some settings that need to be set for seasonedParser to be able to fin ### Download directory In your download client set a incomplete folder and a complete directory. This will allow seasonedParser to only parse items that have been completely downloaded. -*TODO:* Monitor multiple folders at the same time. \ No newline at end of file +*TODO:* Monitor multiple folders at the same time. + +## Run +There are many run commands for this, but here is a list of the current working run commands for this project. + +```bash + user@host:$ ~/seasonedParser/./seasonedMover.py move 'The.Big.Bang.Theory.S11E(7..14).720p.x264.mkv' '/mnt/mainframe/shows/The Big Bang Theory/The Big Bang Theory S11E' +``` + +Here the first parameter is our move command, which in turn calls motherMover. The second parameter is what we want the filenames to be called. Notice the (num1..num2), this is to create a range for all the episodes we want to move. The last parameter is the path we want to move our content. + > This will be done automatically by the parser based on the info in the media items name, but it is nice to have a manual command. \ No newline at end of file diff --git a/knowledgeBase.md b/knowledgeBase.md index 0f95342..0b8347d 100644 --- a/knowledgeBase.md +++ b/knowledgeBase.md @@ -564,4 +564,7 @@ Total: 5926, missed was: 29 real 2m0.766s user 1m41.482s sys 0m0.851s -``` \ No newline at end of file +``` + + +Keep nfo files? \ No newline at end of file From bfd8a2a1f5acd0faa34484f9cdc5aee1d4efc6de Mon Sep 17 00:00:00 2001 From: KevinMidboe Date: Fri, 14 Sep 2018 18:51:40 +0200 Subject: [PATCH 02/98] Snapshot before refactor. --- seasonedParser/core.py | 51 ++++++---- seasonedParser/pirateSearch.py | 48 ++------- seasonedParser/scandir.py | 175 +++++++++++++++++++++++++++++++++ seasonedParser/video.py | 4 +- seasonedParser/walk.py | 25 +++++ 5 files changed, 241 insertions(+), 62 deletions(-) create mode 100755 seasonedParser/scandir.py create mode 100755 seasonedParser/walk.py diff --git a/seasonedParser/core.py b/seasonedParser/core.py index dab6ba6..bd184da 100755 --- a/seasonedParser/core.py +++ b/seasonedParser/core.py @@ -3,10 +3,10 @@ # @Author: KevinMidboe # @Date: 2017-08-25 23:22:27 # @Last Modified by: KevinMidboe -# @Last Modified time: 2017-09-29 12:35:24 +# @Last Modified time: 2018-05-13 20:54:17 from guessit import guessit -import os, errno +import os, errno,sys import logging import tvdb_api from pprint import pprint @@ -19,6 +19,7 @@ from utils import sanitize logging.basicConfig(filename=env.logfile, level=logging.INFO) +from datetime import datetime #: Supported archive extensions ARCHIVE_EXTENSIONS = ('.rar',) @@ -45,7 +46,7 @@ def scan_video(path): # guess parent_path = path.strip(filename) video = Video.fromguess(filename, parent_path, guessit(path)) - # video = Video('test') + # video = Video(filename) # guessit(path) return video @@ -86,6 +87,8 @@ def scan_files(path, age=None, archives=True): if not os.path.isdir(path): raise ValueError('Path is not a directory') + name_dict = {} + # walk the path mediafiles = [] for dirpath, dirnames, filenames in os.walk(path): @@ -99,11 +102,9 @@ def scan_files(path, age=None, archives=True): # scan for videos for filename in filenames: - # filter on videos and archives if not (filename.endswith(VIDEO_EXTENSIONS) or filename.endswith(SUBTITLE_EXTENSIONS) or archives and filename.endswith(ARCHIVE_EXTENSIONS)): continue - # skip hidden files if filename.startswith('.'): logging.debug('Skipping hidden filename %r in %r', filename, dirpath) continue @@ -116,16 +117,19 @@ def scan_files(path, age=None, archives=True): logging.debug('Skipping link %r in %r', filename, dirpath) continue - # skip old files - if age and datetime.utcnow() - datetime.utcfromtimestamp(os.path.getmtime(filepath)) > age: - logging.debug('Skipping old file %r in %r', filename, dirpath) - continue - # scan if filename.endswith(VIDEO_EXTENSIONS): # video try: video = scan_video(filepath) + # try: + # name_dict[video.series] += 1 + # except KeyError: + # name_dict[video.series] = 0 + # except: + # print('video did not have attrib series') + # pass mediafiles.append(video) + except ValueError: # pragma: no cover logging.exception('Error scanning video') continue @@ -138,24 +142,26 @@ def scan_files(path, age=None, archives=True): # except (NotRarFile, RarCannotExec, ValueError): # pragma: no cover # logging.exception('Error scanning archive') # continue - elif filename.endswith(SUBTITLE_EXTENSIONS): # subtitle - try: - subtitle = scan_subtitle(filepath) - mediafiles.append(subtitle) - except ValueError: - logging.exception('Error scanning subtitle') - continue + # elif filename.endswith(SUBTITLE_EXTENSIONS): # subtitle + # try: + # subtitle = scan_subtitle(filepath) + # mediafiles.append(subtitle) + # except ValueError: + # logging.exception('Error scanning subtitle') + # continue else: # pragma: no cover - raise ValueError('Unsupported file %r' % filename) + print('Skipping unsupported file {}'.format(filename)) + # raise ValueError('Unsupported file %r' % filename) + pprint(name_dict) return mediafiles def organize_files(path): hashList = {} mediafiles = scan_files(path) - # print(mediafiles) + print(mediafiles) for file in mediafiles: hashList.setdefault(file.__hash__(),[]).append(file) @@ -251,10 +257,15 @@ def save_subtitles(files, single=False, directory=None, encoding=None): # return saved_subtitles +def stringTime(): + return str(datetime.now().strftime("%Y-%m-%d %H:%M:%S:%f")) + def main(): # episodePath = '/Volumes/media/tv/Black Mirror/Black Mirror Season 01/' - episodePath = '/media/hdd1/tv/' + episodePath = '/Volumes/mainframe/shows/Black Mirror/Black Mirror Season 01/' + episodePath = '/Volumes/mainframe/shows/The.Voice.S14E24.720p.WEB.x264-TBS[rarbg]' + episodePath = '/Volumes/mainframe/incomplete' t = tvdb_api.Tvdb() diff --git a/seasonedParser/pirateSearch.py b/seasonedParser/pirateSearch.py index 02b2b18..397aca4 100755 --- a/seasonedParser/pirateSearch.py +++ b/seasonedParser/pirateSearch.py @@ -3,7 +3,7 @@ # @Author: KevinMidboe # @Date: 2017-10-12 11:55:03 # @Last Modified by: KevinMidboe -# @Last Modified time: 2017-10-17 00:58:24 +# @Last Modified time: 2017-11-01 16:11:30 import sys, logging, re from urllib import parse, request @@ -28,39 +28,6 @@ RELEASE_TYPES = ('bdremux', 'brremux', 'remux', 'camrip', 'cam') -def sanitize(string, ignore_characters=None, replace_characters=None): - """Sanitize a string to strip special characters. - - :param str string: the string to sanitize. - :param set ignore_characters: characters to ignore. - :return: the sanitized string. - :rtype: str - - """ - # only deal with strings - if string is None: - return - - replace_characters = replace_characters or '' - - ignore_characters = ignore_characters or set() - - characters = ignore_characters - if characters: - string = re.sub(r'[%s]' % re.escape(''.join(characters)), replace_characters, string) - - return string - -def return_re_match(string, re_statement): - if string is None: - return - - m = re.search(re_statement, string) - if 'Y-day' in m.group(): - return datetime.datetime.now().strftime('%m-%d %Y') - return sanitize(m.group(), '\xa0', ' ') - - # Should maybe not be able to set values without checking if they are valid? class piratebay(object): def __init__(self, query=None, page=0, sort=None, category=None): @@ -157,7 +124,7 @@ class piratebay(object): print(self.page) # Fetch in parallel - n = self.total_pages + n = pagesToCount(multiple_pages, self.total_pages) while n > 1: torrents_found.extend(self.next_page()) n -= 1 @@ -276,7 +243,7 @@ def chooseCandidate(torrent_list): size, _, size_id = torrent.size.partition(' ') if intersecting_release_types and int(torrent.seed_count) > 0 and float(size) > 4 and size_id == 'GiB': - print('{} : {} : {}'.format(torrent.name, torrent.size, torrent.seed_count)) + print('{} : {} : {} {}'.format(torrent.name, torrent.size, torrent.seed_count, torrent.magnet)) interesting_torrents.append(torrent) # else: # print('Denied match! %s : %s : %s' % (torrent.name, torrent.size, torrent.seed_count)) @@ -286,10 +253,11 @@ def chooseCandidate(torrent_list): def searchTorrentSite(query, site='piratebay'): pirate = piratebay() - torrents_found = pirate.search(query, page=0, multiple_pages=0, sort='size') - # pprint(torrents_found) + torrents_found = pirate.search(query, page=0, multiple_pages=5, sort='size') + pprint(torrents_found) candidates = chooseCandidate(torrents_found) - + pprint(candidates) + exit(0) torrents_found = pirate.search(query, page=0, multiple_pages=0, sort='size', category='movies') movie_candidates = chooseCandidate(torrents_found) @@ -308,4 +276,4 @@ def main(): searchTorrentSite(query) if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/seasonedParser/scandir.py b/seasonedParser/scandir.py new file mode 100755 index 0000000..9c64eb7 --- /dev/null +++ b/seasonedParser/scandir.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python3.6 +# -*- coding: utf-8 -*- +# @Author: KevinMidboe +# @Date: 2017-10-02 16:29:25 +# @Last Modified by: KevinMidboe +# @Last Modified time: 2018-01-15 17:18:36 + +try: + from os import scandir +except ImportError: + from scandir import scandir # use scandir PyPI module on Python < 3.5 + +import env_variables as env +import multiprocessing as mp +import logging, re, datetime +from guessit import guessit + +from video import VIDEO_EXTENSIONS, Episode, Movie, Video +from subtitle import SUBTITLE_EXTENSIONS, Subtitle, get_subtitle_path + +logging.basicConfig(filename=env.logfile, level=logging.INFO) + +""" Move to utils file """ +def removeLeadingZero(number): + stringedNumber = str(number) + if (len(stringedNumber) > 1 and stringedNumber[0] == '0'): + return int(stringedNumber[1:]) + return int(number) + +class movie(object): + def __init__(self, path, title=None, year=None): + self.path = path + self.title = title + self.year = year + +class Episode(object): + def __init__(self, path, name, title=None, season=None, episode=None): + super(Episode, self).__init__() + self.path = path + self.name = name + self.title = title + self.season = season + self.episode = episode + + @classmethod + def fromname(cls, path, name): + title = cls.findTitle(cls, name) + season = cls.findSeasonNumber(cls, name) + episode = cls.findEpisodeNumber(cls, name) + + return cls(path, name, title, season, episode) + + def findTitle(self, name): + m = re.search("([a-zA-Z0-9\'\.\-\ ])+([sS][0-9]{1,3})", name) + if m: + return re.sub('[\ \.]*[sS][0-9]{1,2}', '', m.group(0)) + + def findSeasonNumber(self, name): + m = re.search('[sS][0-9]{1,2}', name) + if m: + seasonNumber = re.sub('[sS]', '', m.group(0)) + return removeLeadingZero(seasonNumber) + + def findEpisodeNumber(self, name): + m = re.search('[eE][0-9]{1,3}', name) + if m: + episodeNumber = re.sub('[eE]', '', m.group(0)) + return removeLeadingZero(episodeNumber) + +def get_tree_size(path): + """Return total size of files in given path and subdirs.""" + total = 0 + for entry in scandir(path): + if not ('.DS_Store' in entry.path or 'lost+found' in entry.path): + if entry.is_dir(follow_symlinks=False): + total += get_tree_size(entry.path) + else: + total += entry.stat(follow_symlinks=False).st_size + return int(total) + +def scantree(path): + """Recursively yield DirEntry objects for given directory.""" + for entry in scandir(path): + # Skip .DS_Store and lost+found + # TODO have a blacklist here + if not ('.DS_Store' in entry.path or 'lost+found' in entry.path): + if entry.is_dir(follow_symlinks=False): + yield from scantree(entry.path) + else: + yield entry + +# Find all the mediaobjects for a given path +# TODO handle list of path's +def get_objects_for_path(path, archives=None, match=False): + # Declare list to save the media objects found in the given path + hashList = {} + mediaFiles = [] + # All entries given from scantree functoin + for entry in scantree(path): + logging.debug('Looking at file %s', str(entry.name)) + name = entry.name # Pull out name for faster index + + # Skip if not corrent media extension + if not (name.endswith(VIDEO_EXTENSIONS) or name.endswith(SUBTITLE_EXTENSIONS) or archives and name.endswith(ARCHIVE_EXTENSIONS)): + continue + + # Skip if the file is a dotfile + if name.startswith('.'): + logging.debug('Skipping hidden file %s' % str(name)) + continue + + # If we have a video, create a class and append to mediaFiles + if name.endswith(VIDEO_EXTENSIONS): # video + episode = Episode.fromname(entry.path, entry.name) + if (episode.title is None): + logging.debug('None found for %s' % name) + continue + + title = re.sub('[\.]', ' ', episode.title) + mediaFiles.append(episode) + + return mediaFiles + +if __name__ == '__main__': + logging.info('Started: %s' % str(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S:%f"))) + import sys + from pprint import pprint + total = 0 + missed = 0 + + # print(get_tree_size(sys.argv[1] if len(sys.argv) > 1 else '.')) + # print(str(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S:%f"))) + path = sys.argv[1] if len(sys.argv) > 1 else '.' + mediaFiles = get_objects_for_path(path) + getTitle = lambda ep: ep.title + + for ep in mediaFiles: + print(getTitle(ep)) + + + mediaList = [] + for entry in scantree(sys.argv[1] if len(sys.argv) > 1 else '.'): + name = entry.name + manual = Episode.fromname(entry.path, entry.name) + size = int(entry.stat(follow_symlinks=False).st_size) / 1024 / 1024 / 1024 + # print(name + ' : ' + str(round(size, 2)) + 'GB') + + title = manual.title + if title is None: + logging.debug('None found for %s' % (name)) + continue + + title = re.sub('[\.]', ' ', manual.title) + + # try: + # print(name + ' : ' + "%s S%iE%i" % (str(title), manual.season, manual.episode)) + # except TypeError: + # logging.error('Unexpected error: ' + name) + + mediaList.append(manual) + if ('-m' in sys.argv): + guess = guessit(name) + + logging.info('Manual is: {} and guess is {}'.format(title, guess['title'])) + # # if not (guess['season'] == manual.season and guess['episode'] == manual.episode): + if (guess['title'].lower() != title.lower()): + logging.info('Missmatch: %s by manual guess: %s : %s' % (name, guess['title'], title)) + missed += 1 + + total += 1 + + + print('Total: %i, missed was: %i' % (total, missed)) + logging.info('Ended: %s' % str(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S:%f"))) + logging.info(' - - - - - - - - - ') diff --git a/seasonedParser/video.py b/seasonedParser/video.py index 68be105..f724598 100644 --- a/seasonedParser/video.py +++ b/seasonedParser/video.py @@ -3,7 +3,7 @@ # @Author: KevinMidboe # @Date: 2017-08-26 08:23:18 # @Last Modified by: KevinMidboe -# @Last Modified time: 2017-09-29 13:56:21 +# @Last Modified time: 2018-05-13 20:50:00 from guessit import guessit import os @@ -12,7 +12,7 @@ import hashlib, tvdb_api #: Video extensions VIDEO_EXTENSIONS = ('.3g2', '.3gp', '.3gp2', '.3gpp', '.60d', '.ajp', '.asf', '.asx', '.avchd', '.avi', '.bik', '.bix', '.box', '.cam', '.dat', '.divx', '.dmf', '.dv', '.dvr-ms', '.evo', '.flc', '.fli', - '.flic', '.flv', '.flx', '.gvi', '.gvp', '.h264', '.m1v', '.m2p', '.m2ts', '.m2v', '.m4e', + '.flic', '.flv', '.flx', '.gvi', '.gvp', '.h264', '.m1v', '.m2p', '.m2v', '.m4e', '.m4v', '.mjp', '.mjpeg', '.mjpg', '.mkv', '.moov', '.mov', '.movhd', '.movie', '.movx', '.mp4', '.mpe', '.mpeg', '.mpg', '.mpv', '.mpv2', '.mxf', '.nsv', '.nut', '.ogg', '.ogm' '.ogv', '.omf', '.ps', '.qt', '.ram', '.rm', '.rmvb', '.swf', '.ts', '.vfw', '.vid', '.video', '.viv', '.vivo', diff --git a/seasonedParser/walk.py b/seasonedParser/walk.py new file mode 100755 index 0000000..b922802 --- /dev/null +++ b/seasonedParser/walk.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3.6 +# -*- coding: utf-8 -*- +# @Author: KevinMidboe +# @Date: 2017-10-02 16:29:25 +# @Last Modified by: KevinMidboe +# @Last Modified time: 2017-10-02 18:07:26 + +import itertools, os +import multiprocessing + +def worker(filename): + print(filename) + +def main(): + with multiprocessing.Pool(48) as Pool: # pool of 48 processes + + walk = os.walk("/Volumes/mainframe/shows/") + fn_gen = itertools.chain.from_iterable((os.path.join(root, file) + for file in files) + for root, dirs, files in walk) + + results_of_work = Pool.map(worker, fn_gen) # this does the parallel processing + +if __name__ == '__main__': + main() \ No newline at end of file From 7ea82d4b33cffbceb265080be2a4fa4c85a34df2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Fri, 14 Sep 2018 19:18:59 +0200 Subject: [PATCH 03/98] Renamed seasonedParser subfolder to src --- {seasonedParser => src}/core.py | 0 {seasonedParser => src}/env_variables.py | 0 {seasonedParser => src}/pirateSearch.py | 0 {seasonedParser => src}/scandir.py | 0 {seasonedParser => src}/seasonGuesser.py | 0 {seasonedParser => src}/seasonMover.py | 0 {seasonedParser => src}/subtitle.py | 0 {seasonedParser => src}/tvdb.py | 0 {seasonedParser => src}/utils.py | 0 {seasonedParser => src}/video.py | 0 {seasonedParser => src}/walk.py | 0 {seasonedParser => src}/watcher.py | 0 12 files changed, 0 insertions(+), 0 deletions(-) rename {seasonedParser => src}/core.py (100%) rename {seasonedParser => src}/env_variables.py (100%) rename {seasonedParser => src}/pirateSearch.py (100%) rename {seasonedParser => src}/scandir.py (100%) rename {seasonedParser => src}/seasonGuesser.py (100%) rename {seasonedParser => src}/seasonMover.py (100%) rename {seasonedParser => src}/subtitle.py (100%) rename {seasonedParser => src}/tvdb.py (100%) rename {seasonedParser => src}/utils.py (100%) rename {seasonedParser => src}/video.py (100%) rename {seasonedParser => src}/walk.py (100%) rename {seasonedParser => src}/watcher.py (100%) diff --git a/seasonedParser/core.py b/src/core.py similarity index 100% rename from seasonedParser/core.py rename to src/core.py diff --git a/seasonedParser/env_variables.py b/src/env_variables.py similarity index 100% rename from seasonedParser/env_variables.py rename to src/env_variables.py diff --git a/seasonedParser/pirateSearch.py b/src/pirateSearch.py similarity index 100% rename from seasonedParser/pirateSearch.py rename to src/pirateSearch.py diff --git a/seasonedParser/scandir.py b/src/scandir.py similarity index 100% rename from seasonedParser/scandir.py rename to src/scandir.py diff --git a/seasonedParser/seasonGuesser.py b/src/seasonGuesser.py similarity index 100% rename from seasonedParser/seasonGuesser.py rename to src/seasonGuesser.py diff --git a/seasonedParser/seasonMover.py b/src/seasonMover.py similarity index 100% rename from seasonedParser/seasonMover.py rename to src/seasonMover.py diff --git a/seasonedParser/subtitle.py b/src/subtitle.py similarity index 100% rename from seasonedParser/subtitle.py rename to src/subtitle.py diff --git a/seasonedParser/tvdb.py b/src/tvdb.py similarity index 100% rename from seasonedParser/tvdb.py rename to src/tvdb.py diff --git a/seasonedParser/utils.py b/src/utils.py similarity index 100% rename from seasonedParser/utils.py rename to src/utils.py diff --git a/seasonedParser/video.py b/src/video.py similarity index 100% rename from seasonedParser/video.py rename to src/video.py diff --git a/seasonedParser/walk.py b/src/walk.py similarity index 100% rename from seasonedParser/walk.py rename to src/walk.py diff --git a/seasonedParser/watcher.py b/src/watcher.py similarity index 100% rename from seasonedParser/watcher.py rename to src/watcher.py From 1c81a0165818b0a5722607efbf270902c14c94ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Fri, 14 Sep 2018 19:25:17 +0200 Subject: [PATCH 04/98] Scans for files and subtitles in same folder as video files. --- src/core.py | 100 +++++++++++++++++++++++----------------------------- 1 file changed, 44 insertions(+), 56 deletions(-) diff --git a/src/core.py b/src/core.py index bd184da..d1a7071 100755 --- a/src/core.py +++ b/src/core.py @@ -3,10 +3,10 @@ # @Author: KevinMidboe # @Date: 2017-08-25 23:22:27 # @Last Modified by: KevinMidboe -# @Last Modified time: 2018-05-13 20:54:17 +# @Last Modified time: 2017-09-29 12:35:24 from guessit import guessit -import os, errno,sys +import os, errno import logging import tvdb_api from pprint import pprint @@ -19,11 +19,33 @@ from utils import sanitize logging.basicConfig(filename=env.logfile, level=logging.INFO) -from datetime import datetime #: Supported archive extensions ARCHIVE_EXTENSIONS = ('.rar',) +def search_external_subtitles(path, directory=None): + dirpath, filename = os.path.split(path) + dirpath = dirpath or '.' + fileroot, fileext = os.path.splitext(filename) + + subtitles = {} + for p in os.listdir(directory or dirpath): + if not p.endswith(SUBTITLE_EXTENSIONS): + continue + + language = Language('und') + language_code = p[len(fileroot):-len(os.path.splitext(p)[1])].replace(fileext, '').replace('_','-')[1:] + if language_code: + try: + language = Language.fromietf(language_code) + except (ValueError, LanguageReverseError): + logger.error('Cannot parse language code %r', language_code) + + subtitles[p] = language + logger.debug('Found subtitles %r', subtitles) + + return subtitles + def scan_video(path): """Scan a video from a `path`. @@ -36,18 +58,15 @@ def scan_video(path): if not os.path.exists(path): raise ValueError('Path does not exist') - # check video extension - # if not path.endswith(VIDEO_EXTENSIONS): - # raise ValueError('%r is not a valid video extension' % os.path.splitext(path)[1]) + check video extension + if not path.endswith(VIDEO_EXTENSIONS): + raise ValueError('%r is not a valid video extension' % os.path.splitext(path)[1]) dirpath, filename = os.path.split(path) logging.info('Scanning video %r in %r', filename, dirpath) # guess - parent_path = path.strip(filename) - video = Video.fromguess(filename, parent_path, guessit(path)) - # video = Video(filename) - # guessit(path) + video = Video.fromguess(path, guessit(path)) return video @@ -67,14 +86,12 @@ def scan_subtitle(path): return subtitle -def scan_files(path, age=None, archives=True): +def scan_videos(path): """Scan `path` for videos and their subtitles. See :func:`refine` to find additional information for the video. :param str path: existing directory path to scan. - :param datetime.timedelta age: maximum age of the video or archive. - :param bool archives: scan videos in archives. :return: the scanned videos. :rtype: list of :class:`~subliminal.video.Video` @@ -87,10 +104,8 @@ def scan_files(path, age=None, archives=True): if not os.path.isdir(path): raise ValueError('Path is not a directory') - name_dict = {} - # walk the path - mediafiles = [] + videos = [] for dirpath, dirnames, filenames in os.walk(path): logging.debug('Walking directory %r', dirpath) @@ -102,9 +117,11 @@ def scan_files(path, age=None, archives=True): # scan for videos for filename in filenames: - if not (filename.endswith(VIDEO_EXTENSIONS) or filename.endswith(SUBTITLE_EXTENSIONS) or archives and filename.endswith(ARCHIVE_EXTENSIONS)): + # filter on videos and archives + if not (filename.endswith(VIDEO_EXTENSIONS) or archives and filename.endswith(ARCHIVE_EXTENSIONS)): continue + # skip hidden files if filename.startswith('.'): logging.debug('Skipping hidden filename %r in %r', filename, dirpath) continue @@ -121,47 +138,21 @@ def scan_files(path, age=None, archives=True): if filename.endswith(VIDEO_EXTENSIONS): # video try: video = scan_video(filepath) - # try: - # name_dict[video.series] += 1 - # except KeyError: - # name_dict[video.series] = 0 - # except: - # print('video did not have attrib series') - # pass - mediafiles.append(video) - except ValueError: # pragma: no cover logging.exception('Error scanning video') continue - elif archives and filename.endswith(ARCHIVE_EXTENSIONS): # archive - print('archive') - pass - # try: - # video = scan_archive(filepath) - # mediafiles.append(video) - # except (NotRarFile, RarCannotExec, ValueError): # pragma: no cover - # logging.exception('Error scanning archive') - # continue - # elif filename.endswith(SUBTITLE_EXTENSIONS): # subtitle - # try: - # subtitle = scan_subtitle(filepath) - # mediafiles.append(subtitle) - # except ValueError: - # logging.exception('Error scanning subtitle') - # continue else: # pragma: no cover - print('Skipping unsupported file {}'.format(filename)) - # raise ValueError('Unsupported file %r' % filename) + raise ValueError('Unsupported file %r' % filename) + videos.append(video) - pprint(name_dict) - return mediafiles + return videos def organize_files(path): hashList = {} mediafiles = scan_files(path) - print(mediafiles) + # print(mediafiles) for file in mediafiles: hashList.setdefault(file.__hash__(),[]).append(file) @@ -257,21 +248,18 @@ def save_subtitles(files, single=False, directory=None, encoding=None): # return saved_subtitles -def stringTime(): - return str(datetime.now().strftime("%Y-%m-%d %H:%M:%S:%f")) - def main(): # episodePath = '/Volumes/media/tv/Black Mirror/Black Mirror Season 01/' - episodePath = '/Volumes/mainframe/shows/Black Mirror/Black Mirror Season 01/' - episodePath = '/Volumes/mainframe/shows/The.Voice.S14E24.720p.WEB.x264-TBS[rarbg]' - episodePath = '/Volumes/mainframe/incomplete' + path = '/mnt/rescue/' - t = tvdb_api.Tvdb() + # t = tvdb_api.Tvdb() - hashList = organize_files(episodePath) - pprint(hashList) + # hashList = organize_files(episodePath) + # pprint(hashList) + videos = scan_videos(path) + pprint(videos) if __name__ == '__main__': From 61a1307275a3cf4dbceb72d4bcb3a8e968404e3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Fri, 14 Sep 2018 19:31:16 +0200 Subject: [PATCH 05/98] Removed custom paramenters and class functions for subtitles and video. --- src/subtitle.py | 223 +++++++++++++++++++++++++++++++++++++++++++----- src/video.py | 4 +- 2 files changed, 206 insertions(+), 21 deletions(-) diff --git a/src/subtitle.py b/src/subtitle.py index ee36ff1..465c49b 100644 --- a/src/subtitle.py +++ b/src/subtitle.py @@ -15,7 +15,7 @@ from langdetect import detect logger = logging.getLogger(__name__) #: Subtitle extensions -SUBTITLE_EXTENSIONS = ('.srt', '.sub') +SUBTITLE_EXTENSIONS = ('.srt') class Subtitle(object): @@ -33,28 +33,214 @@ class Subtitle(object): #: Name of the provider that returns that class of subtitle provider_name = '' - def __init__(self, name, parent_path, series, season, episode, language=None, hash=None, container=None, format=None, sdh=False): - #: Language of the subtitle - - self.name = name + def __init__(self, language, hearing_impaired=False, page_link=None, encoding=None): + #: Language of the subtitle + self.language = language - self.parent_path = parent_path - - self.series = series - - self.season = season + #: Whether or not the subtitle is hearing impaired + self.hearing_impaired = hearing_impaired - self.episode = episode + #: URL of the web page from which the subtitle can be downloaded + self.page_link = page_link - self.language=language + #: Content as bytes + self.content = None - self.hash = hash + #: Encoding to decode with when accessing :attr:`text` + self.encoding = None - self.container = container - - self.format = format + # validate the encoding + if encoding: + try: + self.encoding = codecs.lookup(encoding).name + except (TypeError, LookupError): + logger.debug('Unsupported encoding %s', encoding) - self.sdh = sdh + @property + def id(self): + """Unique identifier of the subtitle""" + raise NotImplementedError + + @property + def text(self): + """Content as string + If :attr:`encoding` is None, the encoding is guessed with :meth:`guess_encoding` + """ + if not self.content: + return + + if self.encoding: + return self.content.decode(self.encoding, errors='replace') + + return self.content.decode(self.guess_encoding(), errors='replace') + + def is_valid(self): + """Check if a :attr:`text` is a valid SubRip format. + :return: whether or not the subtitle is valid. + :rtype: bool + """ + if not self.text: + return False + + try: + pysrt.from_string(self.text, error_handling=pysrt.ERROR_RAISE) + except pysrt.Error as e: + if e.args[0] < 80: + return False + + return True + + def guess_encoding(self): + """Guess encoding using the language, falling back on chardet. + :return: the guessed encoding. + :rtype: str + """ + logger.info('Guessing encoding for language %s', self.language) + + # always try utf-8 first + encodings = ['utf-8'] + + # add language-specific encodings + if self.language.alpha3 == 'zho': + encodings.extend(['gb18030', 'big5']) + elif self.language.alpha3 == 'jpn': + encodings.append('shift-jis') + elif self.language.alpha3 == 'ara': + encodings.append('windows-1256') + elif self.language.alpha3 == 'heb': + encodings.append('windows-1255') + elif self.language.alpha3 == 'tur': + encodings.extend(['iso-8859-9', 'windows-1254']) + elif self.language.alpha3 == 'pol': + # Eastern European Group 1 + encodings.extend(['windows-1250']) + elif self.language.alpha3 == 'bul': + # Eastern European Group 2 + encodings.extend(['windows-1251']) + else: + # Western European (windows-1252) + encodings.append('latin-1') + + # try to decode + logger.debug('Trying encodings %r', encodings) + for encoding in encodings: + try: + self.content.decode(encoding) + except UnicodeDecodeError: + pass + else: + logger.info('Guessed encoding %s', encoding) + return encoding + + logger.warning('Could not guess encoding from language') + + # fallback on chardet + encoding = chardet.detect(self.content)['encoding'] + logger.info('Chardet found encoding %s', encoding) + + return encoding + + def get_matches(self, video): + """Get the matches against the `video`. + :param video: the video to get the matches with. + :type video: :class:`~subliminal.video.Video` + :return: matches of the subtitle. + :rtype: set + """ + raise NotImplementedError + + def __hash__(self): + return hash(self.provider_name + '-' + self.id) + + def __repr__(self): + return '<%s %r [%s]>' % (self.__class__.__name__, self.id, self.language) + + +def get_subtitle_path(video_path, language=None, extension='.srt'): + """Get the subtitle path using the `video_path` and `language`. + :param str video_path: path to the video. + :param language: language of the subtitle to put in the path. + :type language: :class:`~babelfish.language.Language` + :param str extension: extension of the subtitle. + :return: path of the subtitle. + :rtype: str + """ + subtitle_root = os.path.splitext(video_path)[0] + + if language: + subtitle_root += '.' + str(language) + + return subtitle_root + extension + + +def guess_matches(video, guess, partial=False): + """Get matches between a `video` and a `guess`. + If a guess is `partial`, the absence information won't be counted as a match. + :param video: the video. + :type video: :class:`~subliminal.video.Video` + :param guess: the guess. + :type guess: dict + :param bool partial: whether or not the guess is partial. + :return: matches between the `video` and the `guess`. + :rtype: set + """ + matches = set() + if isinstance(video, Episode): + # series + if video.series and 'title' in guess and sanitize(guess['title']) == sanitize(video.series): + matches.add('series') + # title + if video.title and 'episode_title' in guess and sanitize(guess['episode_title']) == sanitize(video.title): + matches.add('title') + # season + if video.season and 'season' in guess and guess['season'] == video.season: + matches.add('season') + # episode + if video.episode and 'episode' in guess and guess['episode'] == video.episode: + matches.add('episode') + # year + if video.year and 'year' in guess and guess['year'] == video.year: + matches.add('year') + # count "no year" as an information + if not partial and video.original_series and 'year' not in guess: + matches.add('year') + elif isinstance(video, Movie): + # year + if video.year and 'year' in guess and guess['year'] == video.year: + matches.add('year') + # title + if video.title and 'title' in guess and sanitize(guess['title']) == sanitize(video.title): + matches.add('title') + # release_group + if (video.release_group and 'release_group' in guess and + sanitize_release_group(guess['release_group']) in + get_equivalent_release_groups(sanitize_release_group(video.release_group))): + matches.add('release_group') + # resolution + if video.resolution and 'screen_size' in guess and guess['screen_size'] == video.resolution: + matches.add('resolution') + # format + if video.format and 'format' in guess and guess['format'].lower() == video.format.lower(): + matches.add('format') + # video_codec + if video.video_codec and 'video_codec' in guess and guess['video_codec'] == video.video_codec: + matches.add('video_codec') + # audio_codec + if video.audio_codec and 'audio_codec' in guess and guess['audio_codec'] == video.audio_codec: + matches.add('audio_codec') + + return matches + + +def fix_line_ending(content): + """Fix line ending of `content` by changing it to \n. + :param bytes content: content of the subtitle. + :return: the content with fixed line endings. + :rtype: bytes + """ + return content.replace(b'\r\n', b'\n').replace(b'\r', b'\n') + +''' @classmethod def fromguess(cls, name, parent_path, guess): @@ -107,5 +293,4 @@ def get_subtitle_path(subtitles_path, language=None, extension='.srt'): return subtitle_root + extension - - +''' \ No newline at end of file diff --git a/src/video.py b/src/video.py index f724598..f2043a0 100644 --- a/src/video.py +++ b/src/video.py @@ -78,14 +78,14 @@ class Video(object): return timedelta() @classmethod - def fromguess(cls, name, parent_path, guess): + def fromguess(cls, name, guess): """Create an :class:`Episode` or a :class:`Movie` with the given `name` based on the `guess`. :param str name: name of the video. :param dict guess: guessed data. :raise: :class:`ValueError` if the `type` of the `guess` is invalid """ if guess['type'] == 'episode': - return Episode.fromguess(name, parent_path, guess) + return Episode.fromguess(name, guess) if guess['type'] == 'movie': return Movie.fromguess(name, guess) From 541a18ac9f79086201c03f93e06c78cfb385c969 Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Fri, 14 Sep 2018 19:32:24 +0200 Subject: [PATCH 06/98] Disabled import of langdetect library --- src/subtitle.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/subtitle.py b/src/subtitle.py index 465c49b..f377ebb 100644 --- a/src/subtitle.py +++ b/src/subtitle.py @@ -9,7 +9,7 @@ import hashlib from video import Episode, Movie from utils import sanitize -from langdetect import detect +# from langdetect import detect logger = logging.getLogger(__name__) @@ -293,4 +293,4 @@ def get_subtitle_path(subtitles_path, language=None, extension='.srt'): return subtitle_root + extension -''' \ No newline at end of file +''' From 1075942c5232ca36970c9fa6ce3bd9e13564048d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Fri, 14 Sep 2018 19:49:35 +0200 Subject: [PATCH 07/98] Crawls path and checks file if path is not directory. Subtitles for file is also collected. --- src/core.py | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/src/core.py b/src/core.py index d1a7071..49ab53a 100755 --- a/src/core.py +++ b/src/core.py @@ -258,8 +258,56 @@ def main(): # hashList = organize_files(episodePath) # pprint(hashList) - videos = scan_videos(path) - pprint(videos) + # videos = scan_videos(path) + # pprint(videos) + + force = False + + videos = [] + ignored_videos = [] + errored_paths = [] + logger.debug('Collecting path %s', path) + + # non-existing + if not os.path.exists(path): + try: + video = Video.fromname(path) + except: + logger.exception('Unexpected error while collecting non-existing path %s', path) + errored_paths.append(path) + continue + if not force: + video.subtitle_languages |= set(search_external_subtitles(video.name, directory=path).values()) + # refine(video, episode_refiners=refiner, movie_refiners=refiner, embedded_subtitles=not force) + videos.append(video) + continue + + # directories + if os.path.isdir(path): + try: + scanned_videos = scan_videos(path) + except: + logger.exception('Unexpected error while collecting directory path %s', path) + errored_paths.append(path) + continue + for video in scanned_videos: + if not force: + video.subtitle_languages |= set(search_external_subtitles(video.name, + directory=path).values()) + videos.append(video) + continue + + # other inputs + try: + video = scan_video(path) + except: + logger.exception('Unexpected error while collecting path %s', path) + errored_paths.append(path) + continue + if not force: + video.subtitle_languages |= set(search_external_subtitles(video.name, directory=path).values()) + + videos.append(video) if __name__ == '__main__': From 4321f967f8d5d5a31dd7c0423b6c20524deb634f Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Fri, 14 Sep 2018 19:52:02 +0200 Subject: [PATCH 08/98] Syntax error for comment resolved. More exact path and removed option for archive --- src/core.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core.py b/src/core.py index d1a7071..54e9e12 100755 --- a/src/core.py +++ b/src/core.py @@ -58,7 +58,7 @@ def scan_video(path): if not os.path.exists(path): raise ValueError('Path does not exist') - check video extension + # check video extension if not path.endswith(VIDEO_EXTENSIONS): raise ValueError('%r is not a valid video extension' % os.path.splitext(path)[1]) @@ -118,7 +118,7 @@ def scan_videos(path): # scan for videos for filename in filenames: # filter on videos and archives - if not (filename.endswith(VIDEO_EXTENSIONS) or archives and filename.endswith(ARCHIVE_EXTENSIONS)): + if not (filename.endswith(VIDEO_EXTENSIONS) or filename.endswith(ARCHIVE_EXTENSIONS)): continue # skip hidden files @@ -251,7 +251,7 @@ def save_subtitles(files, single=False, directory=None, encoding=None): def main(): # episodePath = '/Volumes/media/tv/Black Mirror/Black Mirror Season 01/' - path = '/mnt/rescue/' + path = '/mnt/rescue/#137101383' # t = tvdb_api.Tvdb() From 9c7a44b027aceb96e9dd9fa751748d0e8876ea69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Fri, 14 Sep 2018 19:54:42 +0200 Subject: [PATCH 09/98] Removed continues when not in loop. --- src/core.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/core.py b/src/core.py index 49ab53a..98c6078 100755 --- a/src/core.py +++ b/src/core.py @@ -275,12 +275,10 @@ def main(): except: logger.exception('Unexpected error while collecting non-existing path %s', path) errored_paths.append(path) - continue if not force: video.subtitle_languages |= set(search_external_subtitles(video.name, directory=path).values()) # refine(video, episode_refiners=refiner, movie_refiners=refiner, embedded_subtitles=not force) videos.append(video) - continue # directories if os.path.isdir(path): @@ -289,13 +287,11 @@ def main(): except: logger.exception('Unexpected error while collecting directory path %s', path) errored_paths.append(path) - continue for video in scanned_videos: if not force: video.subtitle_languages |= set(search_external_subtitles(video.name, directory=path).values()) videos.append(video) - continue # other inputs try: @@ -303,7 +299,6 @@ def main(): except: logger.exception('Unexpected error while collecting path %s', path) errored_paths.append(path) - continue if not force: video.subtitle_languages |= set(search_external_subtitles(video.name, directory=path).values()) From 417e6dda8fdc5c6a99d7a931e382910a858cde85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Fri, 14 Sep 2018 19:56:13 +0200 Subject: [PATCH 10/98] Renamed all loggers to logging --- src/core.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/core.py b/src/core.py index 98c6078..fe5fce0 100755 --- a/src/core.py +++ b/src/core.py @@ -39,10 +39,10 @@ def search_external_subtitles(path, directory=None): try: language = Language.fromietf(language_code) except (ValueError, LanguageReverseError): - logger.error('Cannot parse language code %r', language_code) + logging.error('Cannot parse language code %r', language_code) subtitles[p] = language - logger.debug('Found subtitles %r', subtitles) + logging.debug('Found subtitles %r', subtitles) return subtitles @@ -266,14 +266,14 @@ def main(): videos = [] ignored_videos = [] errored_paths = [] - logger.debug('Collecting path %s', path) + logging.debug('Collecting path %s', path) # non-existing if not os.path.exists(path): try: video = Video.fromname(path) except: - logger.exception('Unexpected error while collecting non-existing path %s', path) + logging.exception('Unexpected error while collecting non-existing path %s', path) errored_paths.append(path) if not force: video.subtitle_languages |= set(search_external_subtitles(video.name, directory=path).values()) @@ -285,7 +285,7 @@ def main(): try: scanned_videos = scan_videos(path) except: - logger.exception('Unexpected error while collecting directory path %s', path) + logging.exception('Unexpected error while collecting directory path %s', path) errored_paths.append(path) for video in scanned_videos: if not force: @@ -297,7 +297,7 @@ def main(): try: video = scan_video(path) except: - logger.exception('Unexpected error while collecting path %s', path) + logging.exception('Unexpected error while collecting path %s', path) errored_paths.append(path) if not force: video.subtitle_languages |= set(search_external_subtitles(video.name, directory=path).values()) From fb7f6238818e8e368b88d2fb826883a755a9aee3 Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Fri, 14 Sep 2018 21:58:51 +0200 Subject: [PATCH 11/98] Imported default conf for video. --- src/video.py | 106 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) diff --git a/src/video.py b/src/video.py index f2043a0..a6919c9 100644 --- a/src/video.py +++ b/src/video.py @@ -106,6 +106,111 @@ class Video(object): return hash(self.name) +class Episode(Video): + """Episode :class:`Video`. + :param str series: series of the episode. + :param int season: season number of the episode. + :param int episode: episode number of the episode. + :param str title: title of the episode. + :param int year: year of the series. + :param bool original_series: whether the series is the first with this name. + :param int tvdb_id: TVDB id of the episode. + :param \*\*kwargs: additional parameters for the :class:`Video` constructor. + """ + def __init__(self, name, series, season, episode, title=None, year=None, original_series=True, tvdb_id=None, + series_tvdb_id=None, series_imdb_id=None, **kwargs): + super(Episode, self).__init__(name, **kwargs) + + #: Series of the episode + self.series = series + + #: Season number of the episode + self.season = season + + #: Episode number of the episode + self.episode = episode + + #: Title of the episode + self.title = title + + #: Year of series + self.year = year + + #: The series is the first with this name + self.original_series = original_series + + #: TVDB id of the episode + self.tvdb_id = tvdb_id + + #: TVDB id of the series + self.series_tvdb_id = series_tvdb_id + + #: IMDb id of the series + self.series_imdb_id = series_imdb_id + + @classmethod + def fromguess(cls, name, guess): + if guess['type'] != 'episode': + raise ValueError('The guess must be an episode guess') + + if 'title' not in guess or 'episode' not in guess: + raise ValueError('Insufficient data to process the guess') + + return cls(name, guess['title'], guess.get('season', 1), guess['episode'], title=guess.get('episode_title'), + year=guess.get('year'), format=guess.get('format'), original_series='year' not in guess, + release_group=guess.get('release_group'), resolution=guess.get('screen_size'), + video_codec=guess.get('video_codec'), audio_codec=guess.get('audio_codec')) + + @classmethod + def fromname(cls, name): + return cls.fromguess(name, guessit(name, {'type': 'episode'})) + + def __repr__(self): + if self.year is None: + return '<%s [%r, %dx%s]>' % (self.__class__.__name__, self.series, self.season, self.episode) + if self.subtitle_languages is not None: + return '<%s [%r, %dx%s] %s>' % (self.__class__.__name__, self.series, self.season, self.episode, self.subtitle_languages) + + return '<%s [%r, %d, %dx%d]>' % (self.__class__.__name__, self.series, self.year, self.season, self.episode) + + +class Movie(Video): + """Movie :class:`Video`. + :param str title: title of the movie. + :param int year: year of the movie. + :param \*\*kwargs: additional parameters for the :class:`Video` constructor. + """ + def __init__(self, name, title, year=None, **kwargs): + super(Movie, self).__init__(name, **kwargs) + + #: Title of the movie + self.title = title + + #: Year of the movie + self.year = year + + @classmethod + def fromguess(cls, name, guess): + if guess['type'] != 'movie': + raise ValueError('The guess must be a movie guess') + + if 'title' not in guess: + raise ValueError('Insufficient data to process the guess') + + return cls(name, guess['title'], format=guess.get('format'), release_group=guess.get('release_group'), + resolution=guess.get('screen_size'), video_codec=guess.get('video_codec'), + audio_codec=guess.get('audio_codec'), year=guess.get('year')) + + @classmethod + def fromname(cls, name): + return cls.fromguess(name, guessit(name, {'type': 'movie'})) + + def __repr__(self): + if self.year is None: + return '<%s [%r]>' % (self.__class__.__name__, self.title) + + return '<%s [%r, %d]>' % (self.__class__.__name__, self.title, self.year) +''' class Episode(): """Episode :class:`Video`. :param str series: series of the episode. @@ -231,3 +336,4 @@ class Movie(): return '<%s [%r]>' % (self.__class__.__name__, self.title) return '<%s [%r, %d]>' % (self.__class__.__name__, self.title, self.year) +''' From eb055db2fd1643aaa9a0552b4d08d20c313713f6 Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Fri, 14 Sep 2018 22:00:33 +0200 Subject: [PATCH 12/98] Added input for hash folders --- src/core.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/core.py b/src/core.py index 49ffbb8..4e920cb 100755 --- a/src/core.py +++ b/src/core.py @@ -6,6 +6,7 @@ # @Last Modified time: 2017-09-29 12:35:24 from guessit import guessit +from babelfish import Language, LanguageReverseError import os, errno import logging import tvdb_api @@ -251,7 +252,9 @@ def save_subtitles(files, single=False, directory=None, encoding=None): def main(): # episodePath = '/Volumes/media/tv/Black Mirror/Black Mirror Season 01/' - path = '/mnt/rescue/#137101383' + path = '/mnt/rescue/' + # hash_path = input('Hash: ') + # path += hash_path # t = tvdb_api.Tvdb() @@ -304,6 +307,9 @@ def main(): videos.append(video) + for video in videos: + pprint(video) + if __name__ == '__main__': main() From 6b9de3a9750e0e576a5355db4a58d2a2ab2a9b3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sat, 15 Sep 2018 12:12:07 +0200 Subject: [PATCH 13/98] Included enzyme package version 0.4.1 for mkv parsing. --- requirements.txt | 2 + src/core.py | 153 +++++++++++++++++++++++++++-------------------- 2 files changed, 89 insertions(+), 66 deletions(-) diff --git a/requirements.txt b/requirements.txt index 0ed1f9c..dd58b86 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,4 @@ guessit==2.1.4 tvdb_api==2.0 +hashids==1.2.0 +enzyme>=0.4.1 diff --git a/src/core.py b/src/core.py index 4e920cb..179ee1d 100755 --- a/src/core.py +++ b/src/core.py @@ -7,6 +7,7 @@ from guessit import guessit from babelfish import Language, LanguageReverseError +from hashids import Hashids import os, errno import logging import tvdb_api @@ -67,7 +68,15 @@ def scan_video(path): logging.info('Scanning video %r in %r', filename, dirpath) # guess - video = Video.fromguess(path, guessit(path)) + video = Video.fromguess(path, filename, guessit(path)) + + # size + video.size = os.path.getsize(path) + + # hash of name + hashids = Hashids(min_length=16) + hashid = hashids.encode(path) + video.name_hash = name_hash return video @@ -210,6 +219,83 @@ def save_subtitles(files, single=False, directory=None, encoding=None): print() + def refine(video, episode_refiners=None, movie_refiners=None, **kwargs): + """Refine a video using :ref:`refiners`. + .. note:: + Exceptions raised in refiners are silently passed and logged. + :param video: the video to refine. + :type video: :class:`~subliminal.video.Video` + :param tuple episode_refiners: refiners to use for episodes. + :param tuple movie_refiners: refiners to use for movies. + :param \*\*kwargs: additional parameters for the :func:`~subliminal.refiners.refine` functions. + """ + refiners = () + if isinstance(video, Episode): + refiners = episode_refiners or ('metadata') + elif isinstance(video, Movie): + refiners = movie_refiners or ('metadata') + for refiner in refiners: + logger.info('Refining video with %s', refiner) + try: + print(refiner) + refiner_manager[refiner].plugin(video, **kwargs) + except: + logger.exception('Failed to refine video') + +def scan_folder(path): + videos = [] + ignored_videos = [] + errored_paths = [] + logging.debug('Collecting path %s', path) + + # non-existing + if not os.path.exists(path): + try: + video = Video.fromname(path) + except: + logging.exception('Unexpected error while collecting non-existing path %s', path) + errored_paths.append(path) + + video.subtitle_languages |= set(search_external_subtitles(video.name, directory=path).values()) + + refine(video, episode_refiners=refiner, movie_refiners=refiner, embedded_subtitles=not force) + videos.append(video) + + # directories + if os.path.isdir(path): + try: + scanned_videos = scan_videos(path) + except: + logging.exception('Unexpected error while collecting directory path %s', path) + errored_paths.append(path) + + for video in scanned_videos: + video.subtitle_languages |= set(search_external_subtitles(video.name, + directory=path).values()) + refine(video, episode_refiners=refiner, movie_refiners=refiner, embedded_subtitles=not force) + videos.append(video) + + return videos + +def main(): + path = '/mnt/rescue/' + # hash_path = input('Hash: ') + # path += hash_path + + # t = tvdb_api.Tvdb() + + # hashList = organize_files(episodePath) + # pprint(hashList) + + videos = scan_folder() + for video in videos: + pprint(video) + + +if __name__ == '__main__': + main() + + # for hash in files: # hashIndex = [files[hash]] # for hashItems in hashIndex: @@ -248,68 +334,3 @@ def save_subtitles(files, single=False, directory=None, encoding=None): # break # return saved_subtitles - - -def main(): - # episodePath = '/Volumes/media/tv/Black Mirror/Black Mirror Season 01/' - path = '/mnt/rescue/' - # hash_path = input('Hash: ') - # path += hash_path - - # t = tvdb_api.Tvdb() - - # hashList = organize_files(episodePath) - # pprint(hashList) - - # videos = scan_videos(path) - # pprint(videos) - - force = False - - videos = [] - ignored_videos = [] - errored_paths = [] - logging.debug('Collecting path %s', path) - - # non-existing - if not os.path.exists(path): - try: - video = Video.fromname(path) - except: - logging.exception('Unexpected error while collecting non-existing path %s', path) - errored_paths.append(path) - if not force: - video.subtitle_languages |= set(search_external_subtitles(video.name, directory=path).values()) - # refine(video, episode_refiners=refiner, movie_refiners=refiner, embedded_subtitles=not force) - videos.append(video) - - # directories - if os.path.isdir(path): - try: - scanned_videos = scan_videos(path) - except: - logging.exception('Unexpected error while collecting directory path %s', path) - errored_paths.append(path) - for video in scanned_videos: - if not force: - video.subtitle_languages |= set(search_external_subtitles(video.name, - directory=path).values()) - videos.append(video) - - # other inputs - try: - video = scan_video(path) - except: - logging.exception('Unexpected error while collecting path %s', path) - errored_paths.append(path) - if not force: - video.subtitle_languages |= set(search_external_subtitles(video.name, directory=path).values()) - - videos.append(video) - - for video in videos: - pprint(video) - - -if __name__ == '__main__': - main() From 077ac940b190a8a1e20fd58a83068359f0d6fcb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sat, 15 Sep 2018 12:15:00 +0200 Subject: [PATCH 14/98] Refine function added to utils to extract metadata from mkv files. --- src/utils.py | 92 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/src/utils.py b/src/utils.py index 2cd6ea7..9495dd3 100644 --- a/src/utils.py +++ b/src/utils.py @@ -2,9 +2,15 @@ from datetime import datetime import hashlib import os +import logging import re import struct +from babelfish import Error as BabelfishError, Language +from enzyme import MKV + +logger = logging.getLogger(__name__) + def sanitize(string, ignore_characters=None): """Sanitize a string to strip special characters. @@ -36,3 +42,89 @@ def sanitize(string, ignore_characters=None): # strip and lower case return string.strip().lower() + +def refine(video, embedded_subtitles=True, **kwargs): + """Refine a video by searching its metadata. + Several :class:`~subliminal.video.Video` attributes can be found: + * :attr:`~subliminal.video.Video.resolution` + * :attr:`~subliminal.video.Video.video_codec` + * :attr:`~subliminal.video.Video.audio_codec` + * :attr:`~subliminal.video.Video.subtitle_languages` + :param bool embedded_subtitles: search for embedded subtitles. + """ + # skip non existing videos + if not video.exists: + return + + # check extensions + extension = os.path.splitext(video.name)[1] + if extension == '.mkv': + with open(video.name, 'rb') as f: + mkv = MKV(f) + + # main video track + if mkv.video_tracks: + video_track = mkv.video_tracks[0] + + # resolution + if video_track.height in (480, 720, 1080): + if video_track.interlaced: + video.resolution = '%di' % video_track.height + else: + video.resolution = '%dp' % video_track.height + logger.debug('Found resolution %s', video.resolution) + + # video codec + if video_track.codec_id == 'V_MPEG4/ISO/AVC': + video.video_codec = 'h264' + logger.debug('Found video_codec %s', video.video_codec) + elif video_track.codec_id == 'V_MPEG4/ISO/SP': + video.video_codec = 'DivX' + logger.debug('Found video_codec %s', video.video_codec) + elif video_track.codec_id == 'V_MPEG4/ISO/ASP': + video.video_codec = 'XviD' + logger.debug('Found video_codec %s', video.video_codec) + else: + logger.warning('MKV has no video track') + + # main audio track + if mkv.audio_tracks: + audio_track = mkv.audio_tracks[0] + # audio codec + if audio_track.codec_id == 'A_AC3': + video.audio_codec = 'AC3' + logger.debug('Found audio_codec %s', video.audio_codec) + elif audio_track.codec_id == 'A_DTS': + video.audio_codec = 'DTS' + logger.debug('Found audio_codec %s', video.audio_codec) + elif audio_track.codec_id == 'A_AAC': + video.audio_codec = 'AAC' + logger.debug('Found audio_codec %s', video.audio_codec) + else: + logger.warning('MKV has no audio track') + + # subtitle tracks + if mkv.subtitle_tracks: + if embedded_subtitles: + embedded_subtitle_languages = set() + for st in mkv.subtitle_tracks: + if st.language: + try: + embedded_subtitle_languages.add(Language.fromalpha3b(st.language)) + except BabelfishError: + logger.error('Embedded subtitle track language %r is not a valid language', st.language) + embedded_subtitle_languages.add(Language('und')) + elif st.name: + try: + embedded_subtitle_languages.add(Language.fromname(st.name)) + except BabelfishError: + logger.debug('Embedded subtitle track name %r is not a valid language', st.name) + embedded_subtitle_languages.add(Language('und')) + else: + embedded_subtitle_languages.add(Language('und')) + logger.debug('Found embedded subtitle %r', embedded_subtitle_languages) + video.subtitle_languages |= embedded_subtitle_languages + else: + logger.debug('MKV has no subtitle track') + else: + logger.debug('Unsupported video extension %s', extension) \ No newline at end of file From 934aeeb37f0baec99966fd692c16dd101cab9ce1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sat, 15 Sep 2018 12:15:56 +0200 Subject: [PATCH 15/98] Refine imported from utils and all calls to Video class no longer include path. --- src/core.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/core.py b/src/core.py index 179ee1d..aecdd72 100755 --- a/src/core.py +++ b/src/core.py @@ -17,7 +17,7 @@ import env_variables as env from video import VIDEO_EXTENSIONS, Episode, Movie, Video from subtitle import SUBTITLE_EXTENSIONS, Subtitle, get_subtitle_path -from utils import sanitize +from utils import sanitize, refine logging.basicConfig(filename=env.logfile, level=logging.INFO) @@ -68,7 +68,7 @@ def scan_video(path): logging.info('Scanning video %r in %r', filename, dirpath) # guess - video = Video.fromguess(path, filename, guessit(path)) + video = Video.fromguess(path, guessit(path)) # size video.size = os.path.getsize(path) @@ -90,7 +90,7 @@ def scan_subtitle(path): # guess parent_path = path.strip(filename) - subtitle = Subtitle.fromguess(filename, parent_path, guessit(path)) + subtitle = Subtitle.fromguess(parent_path, guessit(path)) return subtitle @@ -238,6 +238,7 @@ def save_subtitles(files, single=False, directory=None, encoding=None): logger.info('Refining video with %s', refiner) try: print(refiner) + exit(0) refiner_manager[refiner].plugin(video, **kwargs) except: logger.exception('Failed to refine video') From 24047a2b1d08b59b3f2e0e5652e5953f5532034d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sat, 15 Sep 2018 12:16:48 +0200 Subject: [PATCH 16/98] Hash_name variable added to Video class and reordered varaibles in class declaration. --- src/video.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/video.py b/src/video.py index a6919c9..3859257 100644 --- a/src/video.py +++ b/src/video.py @@ -28,15 +28,21 @@ class Video(object): :param str video_codec: codec of the video stream. :param str audio_codec: codec of the main audio stream. :param str imdb_id: IMDb id of the video. - :param dict hashes: hashes of the video file by provider names. + :param dict name_hash: hashes of the video file by provider names. :param int size: size of the video file in bytes. :param set subtitle_languages: existing subtitle languages. """ - def __init__(self, name, format=None, release_group=None, resolution=None, video_codec=None, audio_codec=None, - imdb_id=None, hashes=None, size=None, subtitle_languages=None): + def __init__(self, name, name_hash=None, size=None, format=None, release_group=None, resolution=None, video_codec=None, audio_codec=None, + imdb_id=None, subtitle_languages=None): #: Name or path of the video self.name = name + #: Hashes of the video file by provider names + self.name_hash = name_hash + + #: Size of the video file in bytes + self.size = size + #: Format of the video (HDTV, WEB-DL, BluRay, ...) self.format = format @@ -55,12 +61,6 @@ class Video(object): #: IMDb id of the video self.imdb_id = imdb_id - #: Hashes of the video file by provider names - self.hashes = hashes or {} - - #: Size of the video file in bytes - self.size = size - #: Existing subtitle languages self.subtitle_languages = subtitle_languages or set() From 3eb3609a3894c308957ab246bbb44745c8211367 Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Sat, 15 Sep 2018 12:32:54 +0200 Subject: [PATCH 17/98] Fixed indentation of refine function and added path parameter the scan_folder function call --- src/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core.py b/src/core.py index aecdd72..5d6e0a0 100755 --- a/src/core.py +++ b/src/core.py @@ -219,7 +219,7 @@ def save_subtitles(files, single=False, directory=None, encoding=None): print() - def refine(video, episode_refiners=None, movie_refiners=None, **kwargs): +def refine(video, episode_refiners=None, movie_refiners=None, **kwargs): """Refine a video using :ref:`refiners`. .. note:: Exceptions raised in refiners are silently passed and logged. @@ -288,7 +288,7 @@ def main(): # hashList = organize_files(episodePath) # pprint(hashList) - videos = scan_folder() + videos = scan_folder(path) for video in videos: pprint(video) From 0badb4e9888f88922c06b211012d02d46c514e30 Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Sat, 15 Sep 2018 12:34:18 +0200 Subject: [PATCH 18/98] Sets name_hash to correct variable hashid not name_hash.wq --- src/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core.py b/src/core.py index 5d6e0a0..3839775 100755 --- a/src/core.py +++ b/src/core.py @@ -76,7 +76,7 @@ def scan_video(path): # hash of name hashids = Hashids(min_length=16) hashid = hashids.encode(path) - video.name_hash = name_hash + video.name_hash = hashid return video From 565921830d31be1854244fce2559c24ac4aac99f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sat, 15 Sep 2018 12:37:25 +0200 Subject: [PATCH 19/98] Refiner is not defined on first refine so pass None. --- src/core.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/core.py b/src/core.py index aecdd72..4a78876 100755 --- a/src/core.py +++ b/src/core.py @@ -259,7 +259,7 @@ def scan_folder(path): video.subtitle_languages |= set(search_external_subtitles(video.name, directory=path).values()) - refine(video, episode_refiners=refiner, movie_refiners=refiner, embedded_subtitles=not force) + refine(video, episode_refiners=None, movie_refiners=None, embedded_subtitles=not force) videos.append(video) # directories @@ -267,13 +267,14 @@ def scan_folder(path): try: scanned_videos = scan_videos(path) except: + print('Unexpected error while collecting directory path %s', path) logging.exception('Unexpected error while collecting directory path %s', path) errored_paths.append(path) for video in scanned_videos: video.subtitle_languages |= set(search_external_subtitles(video.name, directory=path).values()) - refine(video, episode_refiners=refiner, movie_refiners=refiner, embedded_subtitles=not force) + refine(video, episode_refiners=None, movie_refiners=None, embedded_subtitles=not force) videos.append(video) return videos From f11bb3339c51b489785841176398228c96c9b179 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sat, 15 Sep 2018 12:40:54 +0200 Subject: [PATCH 20/98] Removed all optional params for refine function. --- src/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core.py b/src/core.py index 180d137..9bf1c83 100755 --- a/src/core.py +++ b/src/core.py @@ -259,7 +259,7 @@ def scan_folder(path): video.subtitle_languages |= set(search_external_subtitles(video.name, directory=path).values()) - refine(video, episode_refiners=None, movie_refiners=None, embedded_subtitles=not force) + refine(video) videos.append(video) # directories @@ -274,7 +274,7 @@ def scan_folder(path): for video in scanned_videos: video.subtitle_languages |= set(search_external_subtitles(video.name, directory=path).values()) - refine(video, episode_refiners=None, movie_refiners=None, embedded_subtitles=not force) + refine(video) videos.append(video) return videos From d50b97daa3e4744277f963b992b5e08a30b37b2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sat, 15 Sep 2018 12:44:15 +0200 Subject: [PATCH 21/98] Changed all logger instances with the default logging. --- src/utils.py | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/src/utils.py b/src/utils.py index 9495dd3..f30b6fa 100644 --- a/src/utils.py +++ b/src/utils.py @@ -9,8 +9,6 @@ import struct from babelfish import Error as BabelfishError, Language from enzyme import MKV -logger = logging.getLogger(__name__) - def sanitize(string, ignore_characters=None): """Sanitize a string to strip special characters. @@ -72,20 +70,20 @@ def refine(video, embedded_subtitles=True, **kwargs): video.resolution = '%di' % video_track.height else: video.resolution = '%dp' % video_track.height - logger.debug('Found resolution %s', video.resolution) + logging.debug('Found resolution %s', video.resolution) # video codec if video_track.codec_id == 'V_MPEG4/ISO/AVC': video.video_codec = 'h264' - logger.debug('Found video_codec %s', video.video_codec) + logging.debug('Found video_codec %s', video.video_codec) elif video_track.codec_id == 'V_MPEG4/ISO/SP': video.video_codec = 'DivX' - logger.debug('Found video_codec %s', video.video_codec) + logging.debug('Found video_codec %s', video.video_codec) elif video_track.codec_id == 'V_MPEG4/ISO/ASP': video.video_codec = 'XviD' - logger.debug('Found video_codec %s', video.video_codec) + logging.debug('Found video_codec %s', video.video_codec) else: - logger.warning('MKV has no video track') + logging.warning('MKV has no video track') # main audio track if mkv.audio_tracks: @@ -93,15 +91,15 @@ def refine(video, embedded_subtitles=True, **kwargs): # audio codec if audio_track.codec_id == 'A_AC3': video.audio_codec = 'AC3' - logger.debug('Found audio_codec %s', video.audio_codec) + logging.debug('Found audio_codec %s', video.audio_codec) elif audio_track.codec_id == 'A_DTS': video.audio_codec = 'DTS' - logger.debug('Found audio_codec %s', video.audio_codec) + logging.debug('Found audio_codec %s', video.audio_codec) elif audio_track.codec_id == 'A_AAC': video.audio_codec = 'AAC' - logger.debug('Found audio_codec %s', video.audio_codec) + logging.debug('Found audio_codec %s', video.audio_codec) else: - logger.warning('MKV has no audio track') + logging.warning('MKV has no audio track') # subtitle tracks if mkv.subtitle_tracks: @@ -112,19 +110,19 @@ def refine(video, embedded_subtitles=True, **kwargs): try: embedded_subtitle_languages.add(Language.fromalpha3b(st.language)) except BabelfishError: - logger.error('Embedded subtitle track language %r is not a valid language', st.language) + logging.error('Embedded subtitle track language %r is not a valid language', st.language) embedded_subtitle_languages.add(Language('und')) elif st.name: try: embedded_subtitle_languages.add(Language.fromname(st.name)) except BabelfishError: - logger.debug('Embedded subtitle track name %r is not a valid language', st.name) + logging.debug('Embedded subtitle track name %r is not a valid language', st.name) embedded_subtitle_languages.add(Language('und')) else: embedded_subtitle_languages.add(Language('und')) - logger.debug('Found embedded subtitle %r', embedded_subtitle_languages) + logging.debug('Found embedded subtitle %r', embedded_subtitle_languages) video.subtitle_languages |= embedded_subtitle_languages else: - logger.debug('MKV has no subtitle track') + logging.debug('MKV has no subtitle track') else: - logger.debug('Unsupported video extension %s', extension) \ No newline at end of file + logging.debug('Unsupported video extension %s', extension) \ No newline at end of file From 388d1a927edfd083e396ccf9fec371e28af43d83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sat, 15 Sep 2018 12:47:09 +0200 Subject: [PATCH 22/98] Removed refined functions in core. Use the utils one instead. --- src/core.py | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/src/core.py b/src/core.py index 9bf1c83..a87ea46 100755 --- a/src/core.py +++ b/src/core.py @@ -219,29 +219,29 @@ def save_subtitles(files, single=False, directory=None, encoding=None): print() -def refine(video, episode_refiners=None, movie_refiners=None, **kwargs): - """Refine a video using :ref:`refiners`. - .. note:: - Exceptions raised in refiners are silently passed and logged. - :param video: the video to refine. - :type video: :class:`~subliminal.video.Video` - :param tuple episode_refiners: refiners to use for episodes. - :param tuple movie_refiners: refiners to use for movies. - :param \*\*kwargs: additional parameters for the :func:`~subliminal.refiners.refine` functions. - """ - refiners = () - if isinstance(video, Episode): - refiners = episode_refiners or ('metadata') - elif isinstance(video, Movie): - refiners = movie_refiners or ('metadata') - for refiner in refiners: - logger.info('Refining video with %s', refiner) - try: - print(refiner) - exit(0) - refiner_manager[refiner].plugin(video, **kwargs) - except: - logger.exception('Failed to refine video') +# def refine(video, episode_refiners=None, movie_refiners=None, **kwargs): +# """Refine a video using :ref:`refiners`. +# .. note:: +# Exceptions raised in refiners are silently passed and logged. +# :param video: the video to refine. +# :type video: :class:`~subliminal.video.Video` +# :param tuple episode_refiners: refiners to use for episodes. +# :param tuple movie_refiners: refiners to use for movies. +# :param \*\*kwargs: additional parameters for the :func:`~subliminal.refiners.refine` functions. +# """ +# refiners = () +# if isinstance(video, Episode): +# refiners = episode_refiners or ('metadata') +# elif isinstance(video, Movie): +# refiners = movie_refiners or ('metadata') +# for refiner in refiners: +# logger.info('Refining video with %s', refiner) +# try: +# print(refiner) +# exit(0) +# refiner_manager[refiner].plugin(video, **kwargs) +# except: +# logger.exception('Failed to refine video') def scan_folder(path): videos = [] From c0621f41d0ad2f045c2e717705ec2ac1ed248620 Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Sun, 16 Sep 2018 15:15:12 +0200 Subject: [PATCH 23/98] Removed archives from accepted filetypes --- src/core.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/core.py b/src/core.py index a87ea46..14cbbf2 100755 --- a/src/core.py +++ b/src/core.py @@ -22,9 +22,6 @@ from utils import sanitize, refine logging.basicConfig(filename=env.logfile, level=logging.INFO) -#: Supported archive extensions -ARCHIVE_EXTENSIONS = ('.rar',) - def search_external_subtitles(path, directory=None): dirpath, filename = os.path.split(path) dirpath = dirpath or '.' @@ -128,7 +125,8 @@ def scan_videos(path): # scan for videos for filename in filenames: # filter on videos and archives - if not (filename.endswith(VIDEO_EXTENSIONS) or filename.endswith(ARCHIVE_EXTENSIONS)): + if not (filename.endswith(VIDEO_EXTENSIONS)): + logging.debug('Skipping non-video file %s', filename) continue # skip hidden files From 3d63a3bf0816ae2e3a06987b79624e7a07b19dd7 Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Sun, 16 Sep 2018 15:21:42 +0200 Subject: [PATCH 24/98] Catches exceptions when reading mkv containers and return without crashing program. --- src/utils.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/utils.py b/src/utils.py index f30b6fa..af53255 100644 --- a/src/utils.py +++ b/src/utils.py @@ -7,7 +7,7 @@ import re import struct from babelfish import Error as BabelfishError, Language -from enzyme import MKV +from enzyme import MalformedMKVError, MKV def sanitize(string, ignore_characters=None): """Sanitize a string to strip special characters. @@ -58,7 +58,14 @@ def refine(video, embedded_subtitles=True, **kwargs): extension = os.path.splitext(video.name)[1] if extension == '.mkv': with open(video.name, 'rb') as f: - mkv = MKV(f) + try: + mkv = MKV(f) + except MalformedMKVError: + logging.error('Failed to parse mkv, malformed file') + return + except KeyError: + logging.error('Key error while opening file, uncompatible mkv container') + return # main video track if mkv.video_tracks: @@ -125,4 +132,4 @@ def refine(video, embedded_subtitles=True, **kwargs): else: logging.debug('MKV has no subtitle track') else: - logging.debug('Unsupported video extension %s', extension) \ No newline at end of file + logging.debug('Unsupported video extension %s', extension) From 5ce0e614650550c2b298b284589e6cefe237a9a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sun, 16 Sep 2018 15:25:12 +0200 Subject: [PATCH 25/98] Allow 4k metadata collection for mkv containers. --- src/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils.py b/src/utils.py index f30b6fa..fc5ffd4 100644 --- a/src/utils.py +++ b/src/utils.py @@ -65,7 +65,7 @@ def refine(video, embedded_subtitles=True, **kwargs): video_track = mkv.video_tracks[0] # resolution - if video_track.height in (480, 720, 1080): + if video_track.height in (480, 720, 1080, 2160): if video_track.interlaced: video.resolution = '%di' % video_track.height else: From 1b373087d27a61643ad9c2fa5d895ddc35c396fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sun, 16 Sep 2018 15:45:26 +0200 Subject: [PATCH 26/98] Added progressbar based on iterations of number of files in directory. --- src/core.py | 82 ++++++++++++++++++++--------------------------------- 1 file changed, 31 insertions(+), 51 deletions(-) diff --git a/src/core.py b/src/core.py index 14cbbf2..8c616ee 100755 --- a/src/core.py +++ b/src/core.py @@ -214,68 +214,48 @@ def save_subtitles(files, single=False, directory=None, encoding=None): print('Moved: %s ---> %s' % (old, newname)) os.rename(old, newname) - print() - - -# def refine(video, episode_refiners=None, movie_refiners=None, **kwargs): -# """Refine a video using :ref:`refiners`. -# .. note:: -# Exceptions raised in refiners are silently passed and logged. -# :param video: the video to refine. -# :type video: :class:`~subliminal.video.Video` -# :param tuple episode_refiners: refiners to use for episodes. -# :param tuple movie_refiners: refiners to use for movies. -# :param \*\*kwargs: additional parameters for the :func:`~subliminal.refiners.refine` functions. -# """ -# refiners = () -# if isinstance(video, Episode): -# refiners = episode_refiners or ('metadata') -# elif isinstance(video, Movie): -# refiners = movie_refiners or ('metadata') -# for refiner in refiners: -# logger.info('Refining video with %s', refiner) -# try: -# print(refiner) -# exit(0) -# refiner_manager[refiner].plugin(video, **kwargs) -# except: -# logger.exception('Failed to refine video') - def scan_folder(path): videos = [] ignored_videos = [] errored_paths = [] logging.debug('Collecting path %s', path) - # non-existing - if not os.path.exists(path): - try: - video = Video.fromname(path) - except: - logging.exception('Unexpected error while collecting non-existing path %s', path) - errored_paths.append(path) + content_count = 0 + for _ in os.listdir(path): + content_count += 1 - video.subtitle_languages |= set(search_external_subtitles(video.name, directory=path).values()) - - refine(video) - videos.append(video) + with click.progressbar(length=content_count, label='Collecting videos') as bar: + # non-existing + if not os.path.exists(path): + try: + video = Video.fromname(path) + except: + logging.exception('Unexpected error while collecting non-existing path %s', path) + errored_paths.append(path) - # directories - if os.path.isdir(path): - try: - scanned_videos = scan_videos(path) - except: - print('Unexpected error while collecting directory path %s', path) - logging.exception('Unexpected error while collecting directory path %s', path) - errored_paths.append(path) - - for video in scanned_videos: - video.subtitle_languages |= set(search_external_subtitles(video.name, - directory=path).values()) + video.subtitle_languages |= set(search_external_subtitles(video.name, directory=path).values()) + refine(video) videos.append(video) + # Increment bar to full ? - return videos + # directories + if os.path.isdir(path): + try: + scanned_videos = scan_videos(path) + except: + print('Unexpected error while collecting directory path %s', path) + logging.exception('Unexpected error while collecting directory path %s', path) + errored_paths.append(path) + + for video in scanned_videos: + video.subtitle_languages |= set(search_external_subtitles(video.name, + directory=path).values()) + refine(video) + videos.append(video) + bar.update(1) + + return videos def main(): path = '/mnt/rescue/' From 964a446a9bac8f1458575047a6f746615f74b33d Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Sun, 16 Sep 2018 15:47:42 +0200 Subject: [PATCH 27/98] Added click library --- requirements.txt | 1 + src/core.py | 1 + 2 files changed, 2 insertions(+) diff --git a/requirements.txt b/requirements.txt index dd58b86..79e1ed8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ guessit==2.1.4 tvdb_api==2.0 hashids==1.2.0 enzyme>=0.4.1 +click>=6.7 diff --git a/src/core.py b/src/core.py index 8c616ee..2f496fa 100755 --- a/src/core.py +++ b/src/core.py @@ -11,6 +11,7 @@ from hashids import Hashids import os, errno import logging import tvdb_api +import click from pprint import pprint import env_variables as env From 1ea94faa8eeb4c3d7a3afddc0f1d1543477c45ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sun, 16 Sep 2018 16:03:47 +0200 Subject: [PATCH 28/98] Moved progress bar from indexing folder content to parsing found videos. --- src/core.py | 53 ++++++++++++++++++++++++++--------------------------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/src/core.py b/src/core.py index 8c616ee..c72163c 100755 --- a/src/core.py +++ b/src/core.py @@ -224,38 +224,37 @@ def scan_folder(path): for _ in os.listdir(path): content_count += 1 - with click.progressbar(length=content_count, label='Collecting videos') as bar: - # non-existing - if not os.path.exists(path): - try: - video = Video.fromname(path) - except: - logging.exception('Unexpected error while collecting non-existing path %s', path) - errored_paths.append(path) + # non-existing + if not os.path.exists(path): + try: + video = Video.fromname(path) + except: + logging.exception('Unexpected error while collecting non-existing path %s', path) + errored_paths.append(path) - video.subtitle_languages |= set(search_external_subtitles(video.name, directory=path).values()) - - refine(video) - videos.append(video) - # Increment bar to full ? + video.subtitle_languages |= set(search_external_subtitles(video.name, directory=path).values()) + + refine(video) + videos.append(video) + # Increment bar to full ? - # directories - if os.path.isdir(path): - try: - scanned_videos = scan_videos(path) - except: - print('Unexpected error while collecting directory path %s', path) - logging.exception('Unexpected error while collecting directory path %s', path) - errored_paths.append(path) + # directories + if os.path.isdir(path): + try: + scanned_videos = scan_videos(path) + except: + print('Unexpected error while collecting directory path %s', path) + logging.exception('Unexpected error while collecting directory path %s', path) + errored_paths.append(path) - for video in scanned_videos: - video.subtitle_languages |= set(search_external_subtitles(video.name, + with click.progressbar(scanned_videos, label='Parsing found videos', item_show_function=lambda v: os.path.split(v.name)[1] if v is not None else '') as bar: + for v in bar: + v.subtitle_languages |= set(search_external_subtitles(v.name, directory=path).values()) - refine(video) - videos.append(video) - bar.update(1) + refine(v) + videos.append(v) - return videos + return videos def main(): path = '/mnt/rescue/' From 2eac7187491f4e3d0d5a46c079f47d5d2b45a1c5 Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Sun, 16 Sep 2018 16:04:22 +0200 Subject: [PATCH 29/98] Changed lading path --- src/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core.py b/src/core.py index 2f496fa..414d848 100755 --- a/src/core.py +++ b/src/core.py @@ -259,7 +259,7 @@ def scan_folder(path): return videos def main(): - path = '/mnt/rescue/' + path = '/mnt/mainframe/' # hash_path = input('Hash: ') # path += hash_path From 175b66b120d0fda93011b9786e0c5c9436a94834 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sun, 16 Sep 2018 16:28:16 +0200 Subject: [PATCH 30/98] Progressbar during parsing of scanned videos no longer shows the filename because it created newline per element. Scanning videos for a given path shows progressbar based on the number of outer folders walked throught --- src/core.py | 87 +++++++++++++++++++++++++++-------------------------- 1 file changed, 44 insertions(+), 43 deletions(-) diff --git a/src/core.py b/src/core.py index c72163c..ce36db7 100755 --- a/src/core.py +++ b/src/core.py @@ -111,50 +111,55 @@ def scan_videos(path): if not os.path.isdir(path): raise ValueError('Path is not a directory') - # walk the path - videos = [] - for dirpath, dirnames, filenames in os.walk(path): - logging.debug('Walking directory %r', dirpath) + # setup progress bar + with click.progressbar(length=len(os.listdir(path)), label='Searching for videos') as bar: - # remove badly encoded and hidden dirnames - for dirname in list(dirnames): - if dirname.startswith('.'): - logging.debug('Skipping hidden dirname %r in %r', dirname, dirpath) - dirnames.remove(dirname) + # walk the path + videos = [] + for dirpath, dirnames, filenames in os.walk(path): + logging.debug('Walking directory %r', dirpath) - # scan for videos - for filename in filenames: - # filter on videos and archives - if not (filename.endswith(VIDEO_EXTENSIONS)): - logging.debug('Skipping non-video file %s', filename) - continue + # remove badly encoded and hidden dirnames + for dirname in list(dirnames): + if dirname.startswith('.'): + logging.debug('Skipping hidden dirname %r in %r', dirname, dirpath) + dirnames.remove(dirname) - # skip hidden files - if filename.startswith('.'): - logging.debug('Skipping hidden filename %r in %r', filename, dirpath) - continue - - # reconstruct the file path - filepath = os.path.join(dirpath, filename) - - # skip links - if os.path.islink(filepath): - logging.debug('Skipping link %r in %r', filename, dirpath) - continue - - # scan - if filename.endswith(VIDEO_EXTENSIONS): # video - try: - video = scan_video(filepath) - except ValueError: # pragma: no cover - logging.exception('Error scanning video') + # scan for videos + for filename in filenames: + # filter on videos and archives + if not (filename.endswith(VIDEO_EXTENSIONS)): + logging.debug('Skipping non-video file %s', filename) continue - else: # pragma: no cover - raise ValueError('Unsupported file %r' % filename) - videos.append(video) + # skip hidden files + if filename.startswith('.'): + logging.debug('Skipping hidden filename %r in %r', filename, dirpath) + continue - return videos + # reconstruct the file path + filepath = os.path.join(dirpath, filename) + + # skip links + if os.path.islink(filepath): + logging.debug('Skipping link %r in %r', filename, dirpath) + continue + + # scan + if filename.endswith(VIDEO_EXTENSIONS): # video + try: + video = scan_video(filepath) + except ValueError: # pragma: no cover + logging.exception('Error scanning video') + continue + else: # pragma: no cover + raise ValueError('Unsupported file %r' % filename) + + videos.append(video) + + bar.update(1) + + return videos def organize_files(path): @@ -220,10 +225,6 @@ def scan_folder(path): errored_paths = [] logging.debug('Collecting path %s', path) - content_count = 0 - for _ in os.listdir(path): - content_count += 1 - # non-existing if not os.path.exists(path): try: @@ -247,7 +248,7 @@ def scan_folder(path): logging.exception('Unexpected error while collecting directory path %s', path) errored_paths.append(path) - with click.progressbar(scanned_videos, label='Parsing found videos', item_show_function=lambda v: os.path.split(v.name)[1] if v is not None else '') as bar: + with click.progressbar(scanned_videos, label='Parsing found videos') as bar: for v in bar: v.subtitle_languages |= set(search_external_subtitles(v.name, directory=path).values()) From f62b834cf9785d34cdbb31e524ce9ab2e0da6340 Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Sun, 16 Sep 2018 16:48:43 +0200 Subject: [PATCH 31/98] Progressbar when searching folders for videos now uses correct length parameter and changed the label text. --- src/core.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/core.py b/src/core.py index f2d4531..5a1ae2f 100755 --- a/src/core.py +++ b/src/core.py @@ -113,7 +113,9 @@ def scan_videos(path): raise ValueError('Path is not a directory') # setup progress bar - with click.progressbar(length=len(os.listdir(path)), label='Searching for videos') as bar: + path_children = 0 + for _ in os.walk(path): path_children += 1 + with click.progressbar(length=path_children, label='Searching folders for videos') as bar: # walk the path videos = [] From 0c1b153e03d8e104a4dd673704424e64e2b00324 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sun, 16 Sep 2018 16:55:05 +0200 Subject: [PATCH 32/98] Uses click.echo to display the number of found videos and erroros paths. --- src/core.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/core.py b/src/core.py index 5a1ae2f..7ab9fbb 100755 --- a/src/core.py +++ b/src/core.py @@ -251,6 +251,7 @@ def scan_folder(path): logging.exception('Unexpected error while collecting directory path %s', path) errored_paths.append(path) + # Iterates over our scanned videos with click.progressbar(scanned_videos, label='Parsing found videos') as bar: for v in bar: v.subtitle_languages |= set(search_external_subtitles(v.name, @@ -258,6 +259,13 @@ def scan_folder(path): refine(v) videos.append(v) + click.echo('%s video%s collected / %s error%s' % ( + click.style(str(len(videos)), bold=True, fg='green' if videos else None), + 's' if len(videos) > 1 else '', + click.style(str(len(errored_paths)), bold=True, fg='red' if errored_paths else None), + 's' if len(errored_paths) > 1 else '', + )) + return videos def main(): From 2541a2171505bd9adbef182ac2601795da1d116d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sun, 16 Sep 2018 17:26:58 +0200 Subject: [PATCH 33/98] Changed class var from name_hash to just be hash. --- src/video.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/video.py b/src/video.py index 3859257..d0c09e3 100644 --- a/src/video.py +++ b/src/video.py @@ -32,13 +32,13 @@ class Video(object): :param int size: size of the video file in bytes. :param set subtitle_languages: existing subtitle languages. """ - def __init__(self, name, name_hash=None, size=None, format=None, release_group=None, resolution=None, video_codec=None, audio_codec=None, + def __init__(self, name, hash=None, size=None, format=None, release_group=None, resolution=None, video_codec=None, audio_codec=None, imdb_id=None, subtitle_languages=None): #: Name or path of the video self.name = name #: Hashes of the video file by provider names - self.name_hash = name_hash + self.hash = hash #: Size of the video file in bytes self.size = size From d9880c77fb562dfbfdac3572af024fe4148acabd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sun, 16 Sep 2018 17:27:38 +0200 Subject: [PATCH 34/98] Sets hash value based on if movie or episode instance. --- src/core.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/core.py b/src/core.py index 7ab9fbb..3d6af28 100755 --- a/src/core.py +++ b/src/core.py @@ -73,8 +73,11 @@ def scan_video(path): # hash of name hashids = Hashids(min_length=16) - hashid = hashids.encode(path) - video.name_hash = hashid + if isinstance(v, Episode): + videoHash = hashids.encode(''.join(self.title, self.year or '')) + elif isinstance(v, Movie): + videoHash = hashids.encode(''.join(self.series, self.season, self.episode)) + video.hash = videoHash return video From ba5011d3a4f0a9450a735b9672af25eef5daa76d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sun, 16 Sep 2018 18:19:25 +0200 Subject: [PATCH 35/98] Updated progressbars to have more info and changed from procent to absolut value when searching for videos. --- src/core.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/core.py b/src/core.py index 3d6af28..fe2ab01 100755 --- a/src/core.py +++ b/src/core.py @@ -118,7 +118,7 @@ def scan_videos(path): # setup progress bar path_children = 0 for _ in os.walk(path): path_children += 1 - with click.progressbar(length=path_children, label='Searching folders for videos') as bar: + with click.progressbar(length=path_children, show_pos=True, label='Searching folders for videos') as bar: # walk the path videos = [] @@ -250,12 +250,11 @@ def scan_folder(path): try: scanned_videos = scan_videos(path) except: - print('Unexpected error while collecting directory path %s', path) logging.exception('Unexpected error while collecting directory path %s', path) errored_paths.append(path) # Iterates over our scanned videos - with click.progressbar(scanned_videos, label='Parsing found videos') as bar: + with click.progressbar(scanned_videos, label='Parsing videos') as bar: for v in bar: v.subtitle_languages |= set(search_external_subtitles(v.name, directory=path).values()) From 5916843e9538327c5f46094aa38f1e34885e86ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sun, 16 Sep 2018 21:45:23 +0200 Subject: [PATCH 36/98] Videos are picked for girl scouts and printed if become scout or remain civilian. --- src/core.py | 75 +++++++++++++++++++---------------------------------- 1 file changed, 27 insertions(+), 48 deletions(-) diff --git a/src/core.py b/src/core.py index fe2ab01..2c4518c 100755 --- a/src/core.py +++ b/src/core.py @@ -270,60 +270,39 @@ def scan_folder(path): return videos +def pickforgirlscouts(video): + if isinstance(video, Movie): + if video.title != None and video.year != None: + return True + + elif isinstance(video, Episode): + if video.series != None and video.season != None and video.episode != None and type(video.episode) != list: + return True + + return False + def main(): - path = '/mnt/mainframe/' - # hash_path = input('Hash: ') - # path += hash_path - - # t = tvdb_api.Tvdb() - - # hashList = organize_files(episodePath) - # pprint(hashList) + path = '/mnt/rescue/' videos = scan_folder(path) + + scout = [] + civilan = [] for video in videos: - pprint(video) + girl = pickforgirlscouts(video) + if girl: + scout.append(girl) + else: + civilan.append(girl) + + click.echo('%s scouts%s collected / %s civilans%s' % ( + click.style(str(len(scout)), bold=True, fg='green' if scout else None), + 's' if len(scout) > 1 else '', + click.style(str(len(civilan)), bold=True, fg='red' if civilan else None), + 's' if len(civilan) > 1 else '', + )) if __name__ == '__main__': main() - - # for hash in files: - # hashIndex = [files[hash]] - # for hashItems in hashIndex: - # for file in hashItems: - # print(file.series) - - # saved_subtitles = [] - # for subtitle in files: - # # check content - # if subtitle.name is None: - # logging.error('Skipping subtitle %r: no content', subtitle) - # continue - - # # check language - # if subtitle.language in set(s.language for s in saved_subtitles): - # logging.debug('Skipping subtitle %r: language already saved', subtitle) - # continue - - # # create subtitle path - # subtitle_path = get_subtitle_path(video.name, None if single else subtitle.language) - # if directory is not None: - # subtitle_path = os.path.join(directory, os.path.split(subtitle_path)[1]) - - # # save content as is or in the specified encoding - # logging.info('Saving %r to %r', subtitle, subtitle_path) - # if encoding is None: - # with io.open(subtitle_path, 'wb') as f: - # f.write(subtitle.content) - # else: - # with io.open(subtitle_path, 'w', encoding=encoding) as f: - # f.write(subtitle.text) - # saved_subtitles.append(subtitle) - - # # check single - # if single: - # break - - # return saved_subtitles From 5ed1019a4670d0af442e1195c184ab1a864920c3 Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Sun, 16 Sep 2018 21:47:15 +0200 Subject: [PATCH 37/98] Redid hash function. Hashids only supports ints and our hash text is string. Moved over to hashlib and md5 function. --- src/core.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/core.py b/src/core.py index fe2ab01..26d7239 100755 --- a/src/core.py +++ b/src/core.py @@ -7,7 +7,7 @@ from guessit import guessit from babelfish import Language, LanguageReverseError -from hashids import Hashids +import hashlib import os, errno import logging import tvdb_api @@ -72,11 +72,11 @@ def scan_video(path): video.size = os.path.getsize(path) # hash of name - hashids = Hashids(min_length=16) - if isinstance(v, Episode): - videoHash = hashids.encode(''.join(self.title, self.year or '')) - elif isinstance(v, Movie): - videoHash = hashids.encode(''.join(self.series, self.season, self.episode)) + if isinstance(video, Movie): + hash_str = ''.join([video.title, str(video.year) or '']) + elif isinstance(video, Episode): + hash_str = ''.join([video.series, str(video.season), str(video.episode)]) + videoHash = hashlib.md5(hash_str.encode()).hexdigest() video.hash = videoHash return video From 96ca355c50309507f3d43c4d264f9b44d3781840 Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Sun, 16 Sep 2018 22:26:07 +0200 Subject: [PATCH 38/98] Fixed naming error with civilian and girl. Also prints total found videos. --- src/core.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/core.py b/src/core.py index 3c4ae44..7cb286c 100755 --- a/src/core.py +++ b/src/core.py @@ -287,21 +287,25 @@ def main(): videos = scan_folder(path) scout = [] - civilan = [] + civilian = [] for video in videos: - girl = pickforgirlscouts(video) - if girl: - scout.append(girl) + sortingHat = pickforgirlscouts(video) + if sortingHat: + scout.append(video) else: - civilan.append(girl) + civilian.append(video) - click.echo('%s scouts%s collected / %s civilans%s' % ( + click.echo('%s scout%s collected / %s civilan%s / %s candidate%s' % ( click.style(str(len(scout)), bold=True, fg='green' if scout else None), 's' if len(scout) > 1 else '', - click.style(str(len(civilan)), bold=True, fg='red' if civilan else None), - 's' if len(civilan) > 1 else '', + click.style(str(len(civilian)), bold=True, fg='red' if civilian else None), + 's' if len(civilian) > 1 else '', + click.style(str(len(videos)), bold=True, fg='blue' if videos else None), + 's' if len(videos) > 1 else '' )) + for video in civilian: + print(video) if __name__ == '__main__': main() From 2108f2f25a20656680b520556e0025e0d25ec69e Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Sun, 16 Sep 2018 23:01:49 +0200 Subject: [PATCH 39/98] Removed directory path from function call. --- src/core.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/core.py b/src/core.py index 7cb286c..e37917b 100755 --- a/src/core.py +++ b/src/core.py @@ -256,8 +256,7 @@ def scan_folder(path): # Iterates over our scanned videos with click.progressbar(scanned_videos, label='Parsing videos') as bar: for v in bar: - v.subtitle_languages |= set(search_external_subtitles(v.name, - directory=path).values()) + v.subtitle_languages |= set(search_external_subtitles(v.name).values()) refine(v) videos.append(v) From 1ff34630ac8432bcd62a9172890060399bc9194a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sun, 16 Sep 2018 23:05:09 +0200 Subject: [PATCH 40/98] Renamed subtitle_languages to subtitles to save the subtitles and not only the subtitle languages available. --- src/core.py | 4 ++-- src/video.py | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/core.py b/src/core.py index e37917b..ca8856e 100755 --- a/src/core.py +++ b/src/core.py @@ -239,7 +239,7 @@ def scan_folder(path): logging.exception('Unexpected error while collecting non-existing path %s', path) errored_paths.append(path) - video.subtitle_languages |= set(search_external_subtitles(video.name, directory=path).values()) + video.subtitles |= set(search_external_subtitles(video.name, directory=path)) refine(video) videos.append(video) @@ -256,7 +256,7 @@ def scan_folder(path): # Iterates over our scanned videos with click.progressbar(scanned_videos, label='Parsing videos') as bar: for v in bar: - v.subtitle_languages |= set(search_external_subtitles(v.name).values()) + v.subtitles |= set(search_external_subtitles(v.name)) refine(v) videos.append(v) diff --git a/src/video.py b/src/video.py index d0c09e3..edfb777 100644 --- a/src/video.py +++ b/src/video.py @@ -30,10 +30,10 @@ class Video(object): :param str imdb_id: IMDb id of the video. :param dict name_hash: hashes of the video file by provider names. :param int size: size of the video file in bytes. - :param set subtitle_languages: existing subtitle languages. + :param set subtitles: existing subtitle languages. """ def __init__(self, name, hash=None, size=None, format=None, release_group=None, resolution=None, video_codec=None, audio_codec=None, - imdb_id=None, subtitle_languages=None): + imdb_id=None, subtitles=None): #: Name or path of the video self.name = name @@ -62,7 +62,7 @@ class Video(object): self.imdb_id = imdb_id #: Existing subtitle languages - self.subtitle_languages = subtitle_languages or set() + self.subtitles = subtitles or set() @property def exists(self): @@ -168,12 +168,11 @@ class Episode(Video): def __repr__(self): if self.year is None: return '<%s [%r, %dx%s]>' % (self.__class__.__name__, self.series, self.season, self.episode) - if self.subtitle_languages is not None: - return '<%s [%r, %dx%s] %s>' % (self.__class__.__name__, self.series, self.season, self.episode, self.subtitle_languages) + if self.subtitles is not None: + return '<%s [%r, %dx%s] %s>' % (self.__class__.__name__, self.series, self.season, self.episode, self.subtitles) return '<%s [%r, %d, %dx%d]>' % (self.__class__.__name__, self.series, self.year, self.season, self.episode) - class Movie(Video): """Movie :class:`Video`. :param str title: title of the movie. @@ -210,6 +209,7 @@ class Movie(Video): return '<%s [%r]>' % (self.__class__.__name__, self.title) return '<%s [%r, %d]>' % (self.__class__.__name__, self.title, self.year) + ''' class Episode(): """Episode :class:`Video`. From 3e2faadc2ffc42748441ebdcc0f3aa0c1d3c5f15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sun, 16 Sep 2018 23:26:06 +0200 Subject: [PATCH 41/98] Video gets a new parameter embeded_subtitles to represent the subtitle files found in the mkv container. --- src/utils.py | 18 +++++++++--------- src/video.py | 5 ++++- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/utils.py b/src/utils.py index 91caf4e..b9534fe 100644 --- a/src/utils.py +++ b/src/utils.py @@ -47,7 +47,7 @@ def refine(video, embedded_subtitles=True, **kwargs): * :attr:`~subliminal.video.Video.resolution` * :attr:`~subliminal.video.Video.video_codec` * :attr:`~subliminal.video.Video.audio_codec` - * :attr:`~subliminal.video.Video.subtitle_languages` + * :attr:`~subliminal.video.Video.embeded_subtitles` :param bool embedded_subtitles: search for embedded subtitles. """ # skip non existing videos @@ -111,24 +111,24 @@ def refine(video, embedded_subtitles=True, **kwargs): # subtitle tracks if mkv.subtitle_tracks: if embedded_subtitles: - embedded_subtitle_languages = set() + embeded_subtitles = set() for st in mkv.subtitle_tracks: if st.language: try: - embedded_subtitle_languages.add(Language.fromalpha3b(st.language)) + embeded_subtitles.add(Language.fromalpha3b(st.language)) except BabelfishError: logging.error('Embedded subtitle track language %r is not a valid language', st.language) - embedded_subtitle_languages.add(Language('und')) + embeded_subtitles.add(Language('und')) elif st.name: try: - embedded_subtitle_languages.add(Language.fromname(st.name)) + embeded_subtitles.add(Language.fromname(st.name)) except BabelfishError: logging.debug('Embedded subtitle track name %r is not a valid language', st.name) - embedded_subtitle_languages.add(Language('und')) + embeded_subtitles.add(Language('und')) else: - embedded_subtitle_languages.add(Language('und')) - logging.debug('Found embedded subtitle %r', embedded_subtitle_languages) - video.subtitle_languages |= embedded_subtitle_languages + embeded_subtitles.add(Language('und')) + logging.debug('Found embedded subtitle %r', embeded_subtitles) + video.embeded_subtitles |= embeded_subtitles else: logging.debug('MKV has no subtitle track') else: diff --git a/src/video.py b/src/video.py index edfb777..d78e614 100644 --- a/src/video.py +++ b/src/video.py @@ -33,7 +33,7 @@ class Video(object): :param set subtitles: existing subtitle languages. """ def __init__(self, name, hash=None, size=None, format=None, release_group=None, resolution=None, video_codec=None, audio_codec=None, - imdb_id=None, subtitles=None): + imdb_id=None, subtitles=None, embeded_subtitles=None): #: Name or path of the video self.name = name @@ -64,6 +64,9 @@ class Video(object): #: Existing subtitle languages self.subtitles = subtitles or set() + #: Embeded subtitle languages + self.embeded_subtitles = embeded_subtitles or set() + @property def exists(self): """Test whether the video exists""" From 16afdb4cd8fdc8ca4473118600f2acc1ceef2d33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sun, 16 Sep 2018 23:38:19 +0200 Subject: [PATCH 42/98] Video has a new attribute home that is the optimal parent folder for this element. --- src/core.py | 7 +++++++ src/video.py | 8 ++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/core.py b/src/core.py index ca8856e..c46637d 100755 --- a/src/core.py +++ b/src/core.py @@ -269,13 +269,20 @@ def scan_folder(path): return videos +def movingToCollege(video, home_path): + video.home = home_path + def pickforgirlscouts(video): if isinstance(video, Movie): if video.title != None and video.year != None: + home_path = '{} ({})'.format(video.title, video.year) + movingToCollege(video, home_path) return True elif isinstance(video, Episode): if video.series != None and video.season != None and video.episode != None and type(video.episode) != list: + home_path = '{} S{:02d}E{:02d}'.format(video.series, video.season, video.episode) + movingToCollege(video, home_path) return True return False diff --git a/src/video.py b/src/video.py index d78e614..fb9a918 100644 --- a/src/video.py +++ b/src/video.py @@ -23,8 +23,8 @@ class Video(object): Represent a video, existing or not. :param str name: name or path of the video. :param str format: format of the video (HDTV, WEB-DL, BluRay, ...). - :param str release_group: release group of the video. - :param str resolution: resolution of the video stream (480p, 720p, 1080p or 1080i). + :param str home: home is the optimal parent folder. + :param str resolution: resolution of the video stream (480p, 720p, 1080p or 1080i, 4K). :param str video_codec: codec of the video stream. :param str audio_codec: codec of the main audio stream. :param str imdb_id: IMDb id of the video. @@ -32,7 +32,7 @@ class Video(object): :param int size: size of the video file in bytes. :param set subtitles: existing subtitle languages. """ - def __init__(self, name, hash=None, size=None, format=None, release_group=None, resolution=None, video_codec=None, audio_codec=None, + def __init__(self, name, hash=None, size=None, format=None, home=None, resolution=None, video_codec=None, audio_codec=None, imdb_id=None, subtitles=None, embeded_subtitles=None): #: Name or path of the video self.name = name @@ -47,7 +47,7 @@ class Video(object): self.format = format #: Release group of the video - self.release_group = release_group + self.home = home #: Resolution of the video stream (480p, 720p, 1080p or 1080i) self.resolution = resolution From 1fc62479be4143ee92cbd6454b553bbb8c57ebda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sun, 16 Sep 2018 23:46:39 +0200 Subject: [PATCH 43/98] Conflict resolved by re-adding release_group --- src/video.py | 145 +++------------------------------------------------ 1 file changed, 7 insertions(+), 138 deletions(-) diff --git a/src/video.py b/src/video.py index fb9a918..3c46eda 100644 --- a/src/video.py +++ b/src/video.py @@ -23,17 +23,17 @@ class Video(object): Represent a video, existing or not. :param str name: name or path of the video. :param str format: format of the video (HDTV, WEB-DL, BluRay, ...). - :param str home: home is the optimal parent folder. + :param str release_group: release group of the video. :param str resolution: resolution of the video stream (480p, 720p, 1080p or 1080i, 4K). :param str video_codec: codec of the video stream. :param str audio_codec: codec of the main audio stream. - :param str imdb_id: IMDb id of the video. + :param str home: optimal parent folder. :param dict name_hash: hashes of the video file by provider names. :param int size: size of the video file in bytes. :param set subtitles: existing subtitle languages. """ - def __init__(self, name, hash=None, size=None, format=None, home=None, resolution=None, video_codec=None, audio_codec=None, - imdb_id=None, subtitles=None, embeded_subtitles=None): + def __init__(self, name, hash=None, size=None, format=None, release_group=None, resolution=None, video_codec=None, audio_codec=None, + home=None, subtitles=None, embeded_subtitles=None): #: Name or path of the video self.name = name @@ -58,8 +58,8 @@ class Video(object): #: Codec of the main audio stream self.audio_codec = audio_codec - #: IMDb id of the video - self.imdb_id = imdb_id + #: optimal home path; parent folder. + self.home = home #: Existing subtitle languages self.subtitles = subtitles or set() @@ -121,7 +121,7 @@ class Episode(Video): :param \*\*kwargs: additional parameters for the :class:`Video` constructor. """ def __init__(self, name, series, season, episode, title=None, year=None, original_series=True, tvdb_id=None, - series_tvdb_id=None, series_imdb_id=None, **kwargs): + series_tvdb_id=None, **kwargs): super(Episode, self).__init__(name, **kwargs) #: Series of the episode @@ -148,9 +148,6 @@ class Episode(Video): #: TVDB id of the series self.series_tvdb_id = series_tvdb_id - #: IMDb id of the series - self.series_imdb_id = series_imdb_id - @classmethod def fromguess(cls, name, guess): if guess['type'] != 'episode': @@ -212,131 +209,3 @@ class Movie(Video): return '<%s [%r]>' % (self.__class__.__name__, self.title) return '<%s [%r, %d]>' % (self.__class__.__name__, self.title, self.year) - -''' -class Episode(): - """Episode :class:`Video`. - :param str series: series of the episode. - :param int season: season number of the episode. - :param int episode: episode number of the episode. - :param str title: title of the episode. - :param int year: year of the series. - :param bool original_series: whether the series is the first with this name. - :param int tvdb_id: TVDB id of the episode. - :param \*\*kwargs: additional parameters for the :class:`Video` constructor. - """ - def __init__(self, name, parent_path, series, season, episode, year=None, original_series=True, tvdb_id=None, - series_tvdb_id=None, series_imdb_id=None, release_group=None, video_codec=None, container=None, - format=None, screen_size=None, **kwargs): - super(Episode, self).__init__() - - self.name = name - - self.parent_path = parent_path - - #: Series of the episode - self.series = series - - #: Season number of the episode - self.season = season - - #: Episode number of the episode - self.episode = episode - - #: Year of series - self.year = year - - #: The series is the first with this name - self.original_series = original_series - - #: TVDB id of the episode - self.tvdb_id = tvdb_id - - #: TVDB id of the series - self.series_tvdb_id = series_tvdb_id - - #: IMDb id of the series - self.series_imdb_id = series_imdb_id - - # The release group of the episode - self.release_group = release_group - - # The video vodec of the series - self.video_codec = video_codec - - # The Video container of the episode - self.container = container - - # The Video format of the episode - self.format = format - - # The Video screen_size of the episode - self.screen_size = screen_size - - @classmethod - def fromguess(cls, name, parent_path, guess): - if guess['type'] != 'episode': - raise ValueError('The guess must be an episode guess') - - if 'title' not in guess or 'episode' not in guess: - raise ValueError('Insufficient data to process the guess') - - return cls(name, parent_path, guess['title'], guess.get('season', 1), guess['episode'], - year=guess.get('year'), original_series='year' not in guess, release_group=guess.get('release_group'), - video_codec=guess.get('video_codec'), audio_codec=guess.get('audio_codec'), container=guess.get('container'), - format=guess.get('format'), screen_size=guess.get('screen_size')) - - @classmethod - def fromname(cls, name): - return cls.fromguess(name, guessit(name, {'type': 'episode'})) - - def __hash__(self): - return hashlib.md5("b'{}'".format(str(self.series) + str(self.season) + str(self.episode)).encode()).hexdigest() - - # THE EP NUMBER IS CONVERTED TO STRING AS A QUICK FIX FOR MULTIPLE NUMBERS IN ONE - def __repr__(self): - if self.year is None: - return '<%s [%r, %sx%s]>' % (self.__class__.__name__, self.series, self.season, str(self.episode)) - - return '<%s [%r, %d, %sx%s]>' % (self.__class__.__name__, self.series, self.year, self.season, str(self.episode)) - - - -class Movie(): - """Movie :class:`Video`. - :param str title: title of the movie. - :param int year: year of the movie. - :param \*\*kwargs: additional parameters for the :class:`Video` constructor. - """ - def __init__(self, name, title, year=None, format=None, **kwargs): - super(Movie, self).__init__() - - #: Title of the movie - self.title = title - - #: Year of the movie - self.year = year - self.format = format - - @classmethod - def fromguess(cls, name, guess): - if guess['type'] != 'movie': - raise ValueError('The guess must be a movie guess') - - if 'title' not in guess: - raise ValueError('Insufficient data to process the guess') - - return cls(name, guess['title'], format=guess.get('format'), release_group=guess.get('release_group'), - resolution=guess.get('screen_size'), video_codec=guess.get('video_codec'), - audio_codec=guess.get('audio_codec'), year=guess.get('year')) - - @classmethod - def fromname(cls, name): - return cls.fromguess(name, guessit(name, {'type': 'movie'})) - - def __repr__(self): - if self.year is None: - return '<%s [%r]>' % (self.__class__.__name__, self.title) - - return '<%s [%r, %d]>' % (self.__class__.__name__, self.title, self.year) -''' From 916ce45fecb6d2b9edbed0a408ea1a16b7d85d0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Thu, 20 Sep 2018 23:24:17 +0200 Subject: [PATCH 44/98] If any error return false when picking for girlscouts --- src/core.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/core.py b/src/core.py index c46637d..7d4c140 100755 --- a/src/core.py +++ b/src/core.py @@ -133,7 +133,6 @@ def scan_videos(path): # scan for videos for filename in filenames: - # filter on videos and archives if not (filename.endswith(VIDEO_EXTENSIONS)): logging.debug('Skipping non-video file %s', filename) continue @@ -146,7 +145,6 @@ def scan_videos(path): # reconstruct the file path filepath = os.path.join(dirpath, filename) - # skip links if os.path.islink(filepath): logging.debug('Skipping link %r in %r', filename, dirpath) continue @@ -270,20 +268,27 @@ def scan_folder(path): return videos def movingToCollege(video, home_path): - video.home = home_path + video.home = path.join(home_path) def pickforgirlscouts(video): if isinstance(video, Movie): if video.title != None and video.year != None: home_path = '{} ({})'.format(video.title, video.year) - movingToCollege(video, home_path) - return True + try: + movingToCollege(video, home_path) + return True + except: + return False elif isinstance(video, Episode): if video.series != None and video.season != None and video.episode != None and type(video.episode) != list: - home_path = '{} S{:02d}E{:02d}'.format(video.series, video.season, video.episode) - movingToCollege(video, home_path) - return True + # Handle the list problems + home_path = '{} S{:02d}E{:02d}'.format(str(video.series), str(video.season), str(video.episode)) + try: + movingToCollege(video, home_path) + return True + except: + return False return False @@ -295,8 +300,7 @@ def main(): scout = [] civilian = [] for video in videos: - sortingHat = pickforgirlscouts(video) - if sortingHat: + if pickforgirlscouts(video): scout.append(video) else: civilian.append(video) From 4b09d6cd2ce6ba02fb5f15d9f204acb8c2afc4cc Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Thu, 20 Sep 2018 23:25:47 +0200 Subject: [PATCH 45/98] Titlecase the homepath and try except creating hash for filename. --- src/core.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/src/core.py b/src/core.py index c46637d..42bd89f 100755 --- a/src/core.py +++ b/src/core.py @@ -13,6 +13,7 @@ import logging import tvdb_api import click from pprint import pprint +from titlecase import titlecase import env_variables as env @@ -72,12 +73,15 @@ def scan_video(path): video.size = os.path.getsize(path) # hash of name - if isinstance(video, Movie): - hash_str = ''.join([video.title, str(video.year) or '']) - elif isinstance(video, Episode): - hash_str = ''.join([video.series, str(video.season), str(video.episode)]) - videoHash = hashlib.md5(hash_str.encode()).hexdigest() - video.hash = videoHash + try: + if isinstance(video, Movie): + hash_str = ''.join([video.title, str(video.year) or '']) + elif isinstance(video, Episode): + hash_str = ''.join([video.series, str(video.season), str(video.episode)]) + videoHash = hashlib.md5(hash_str.encode()).hexdigest() + video.hash = videoHash + except: + print(video) return video @@ -96,7 +100,6 @@ def scan_subtitle(path): return subtitle - def scan_videos(path): """Scan `path` for videos and their subtitles. @@ -118,7 +121,7 @@ def scan_videos(path): # setup progress bar path_children = 0 for _ in os.walk(path): path_children += 1 - with click.progressbar(length=path_children, show_pos=True, label='Searching folders for videos') as bar: + with click.progressbar(length=path_children, show_pos=True, label='Collecting videos') as bar: # walk the path videos = [] @@ -270,7 +273,7 @@ def scan_folder(path): return videos def movingToCollege(video, home_path): - video.home = home_path + video.home = titlecase(home_path) def pickforgirlscouts(video): if isinstance(video, Movie): @@ -281,14 +284,14 @@ def pickforgirlscouts(video): elif isinstance(video, Episode): if video.series != None and video.season != None and video.episode != None and type(video.episode) != list: - home_path = '{} S{:02d}E{:02d}'.format(video.series, video.season, video.episode) + home_path = '{} S{}E{}'.format(str(video.series), str(video.season), str(video.episode)) movingToCollege(video, home_path) return True return False def main(): - path = '/mnt/rescue/' + path = '/mnt/mainframe/' videos = scan_folder(path) @@ -310,8 +313,8 @@ def main(): 's' if len(videos) > 1 else '' )) - for video in civilian: - print(video) + for video in scout: + print('{} lives: {}'.format(video, video.home)) if __name__ == '__main__': main() From 8fab27a95f0a197335669ee835075d9d5c5fca38 Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Thu, 20 Sep 2018 23:28:47 +0200 Subject: [PATCH 46/98] And the other conflict --- src/core.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/core.py b/src/core.py index 011a56f..9cf7c11 100755 --- a/src/core.py +++ b/src/core.py @@ -285,11 +285,6 @@ def pickforgirlscouts(video): elif isinstance(video, Episode): if video.series != None and video.season != None and video.episode != None and type(video.episode) != list: -<<<<<<< HEAD - home_path = '{} S{}E{}'.format(str(video.series), str(video.season), str(video.episode)) - movingToCollege(video, home_path) - return True -======= # Handle the list problems home_path = '{} S{:02d}E{:02d}'.format(str(video.series), str(video.season), str(video.episode)) try: @@ -297,7 +292,6 @@ def pickforgirlscouts(video): return True except: return False ->>>>>>> 916ce45fecb6d2b9edbed0a408ea1a16b7d85d0b return False From 379d789adc3b18e5fb712dee5eb02dfce4cb76a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sun, 23 Sep 2018 16:28:44 +0200 Subject: [PATCH 47/98] Disbled hashing of filenames for now --- src/core.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/core.py b/src/core.py index 9cf7c11..8d06822 100755 --- a/src/core.py +++ b/src/core.py @@ -73,15 +73,17 @@ def scan_video(path): video.size = os.path.getsize(path) # hash of name - try: - if isinstance(video, Movie): - hash_str = ''.join([video.title, str(video.year) or '']) - elif isinstance(video, Episode): - hash_str = ''.join([video.series, str(video.season), str(video.episode)]) - videoHash = hashlib.md5(hash_str.encode()).hexdigest() - video.hash = videoHash - except: - print(video) + # if isinstance(video, Movie): + # if type(video.title) is str and type(video.year) is int: + # home_path = '{} ({})'.format(video.title, video.year) + # hash_str = ''.join([video.title, str(video.year) or '']) + # elif isinstance(video, Episode): + # if type(video.series) is str and type(video.season) is int and type(video.episode) is int: + # home_path = '{} ({})'.format(video.title, video.year) + # hash_str = ''.join([video.series, str(video.season), str(video.episode)]) + # video.hash = hashlib.md5(hash_str.encode()).hexdigest() + # except: + # print(video) return video From a106a35d42219a04340ddb362ce93cad87540a77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sun, 23 Sep 2018 16:29:41 +0200 Subject: [PATCH 48/98] Moved checks for sufficient info in order to move series or movie to the class. Homelocation in now a class function for each movie and series class. --- src/core.py | 25 ++++--------------------- src/video.py | 23 +++++++++++++++++++++++ 2 files changed, 27 insertions(+), 21 deletions(-) diff --git a/src/core.py b/src/core.py index 8d06822..387a8b5 100755 --- a/src/core.py +++ b/src/core.py @@ -272,28 +272,11 @@ def scan_folder(path): return videos -def movingToCollege(video, home_path): - video.home = titlecase(home_path) - def pickforgirlscouts(video): - if isinstance(video, Movie): - if video.title != None and video.year != None: - home_path = '{} ({})'.format(video.title, video.year) - try: - movingToCollege(video, home_path) - return True - except: - return False - - elif isinstance(video, Episode): - if video.series != None and video.season != None and video.episode != None and type(video.episode) != list: - # Handle the list problems - home_path = '{} S{:02d}E{:02d}'.format(str(video.series), str(video.season), str(video.episode)) - try: - movingToCollege(video, home_path) - return True - except: - return False + if video.sufficientInfo(): + video.moveLocation() + print(video.home) + return True return False diff --git a/src/video.py b/src/video.py index 3c46eda..c75b297 100644 --- a/src/video.py +++ b/src/video.py @@ -7,6 +7,7 @@ from guessit import guessit import os +from titlecase import titlecase import hashlib, tvdb_api #: Video extensions @@ -165,6 +166,17 @@ class Episode(Video): def fromname(cls, name): return cls.fromguess(name, guessit(name, {'type': 'episode'})) + @classmethod + def sufficientInfo(self): + return None not in [self.series, self.season, self.episode] and list not in [type(self.series), type(self.episode)] + + @classmethod + def moveLocation(self): + series = titlecase(self.series) + grandParent = '{}/{} {:02d}'.format(series, series, self.season) + parent = '{} S{:02d}E{:02d}'.format(series, self.season, self.episode) + self.home = os.path.join(grandParent, parent, self.name) + def __repr__(self): if self.year is None: return '<%s [%r, %dx%s]>' % (self.__class__.__name__, self.series, self.season, self.episode) @@ -204,6 +216,17 @@ class Movie(Video): def fromname(cls, name): return cls.fromguess(name, guessit(name, {'type': 'movie'})) + @classmethod + def sufficientInfo(self): + return None not in [self.title, self.year] + + @classmethod + def moveLocation(self): + title = titlecase(self.title) + parent = '{} ({})'.format(title, self.year) + self.home = os.path.join(parent, self.name) + + def __repr__(self): if self.year is None: return '<%s [%r]>' % (self.__class__.__name__, self.title) From bfab59d49c64e6a184f13db7da483cb4a776ac21 Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Sun, 23 Sep 2018 17:26:43 +0200 Subject: [PATCH 49/98] Methods should not have been tagged with classmethods and now the basename of class name is attributed to the final home path. --- src/video.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/video.py b/src/video.py index c75b297..a761d06 100644 --- a/src/video.py +++ b/src/video.py @@ -48,7 +48,7 @@ class Video(object): self.format = format #: Release group of the video - self.home = home + self.release_group = release_group #: Resolution of the video stream (480p, 720p, 1080p or 1080i) self.resolution = resolution @@ -166,11 +166,9 @@ class Episode(Video): def fromname(cls, name): return cls.fromguess(name, guessit(name, {'type': 'episode'})) - @classmethod def sufficientInfo(self): return None not in [self.series, self.season, self.episode] and list not in [type(self.series), type(self.episode)] - @classmethod def moveLocation(self): series = titlecase(self.series) grandParent = '{}/{} {:02d}'.format(series, series, self.season) @@ -216,16 +214,15 @@ class Movie(Video): def fromname(cls, name): return cls.fromguess(name, guessit(name, {'type': 'movie'})) - @classmethod def sufficientInfo(self): - return None not in [self.title, self.year] - - @classmethod + t = hasattr(self, "title") + y = hasattr(self, "year") + return None not in [t, y] + def moveLocation(self): title = titlecase(self.title) parent = '{} ({})'.format(title, self.year) - self.home = os.path.join(parent, self.name) - + self.home = os.path.join(parent, os.path.basename(self.name)) def __repr__(self): if self.year is None: From 81a53465e7cbde846601a41589b84fa939241d97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sun, 23 Sep 2018 17:42:42 +0200 Subject: [PATCH 50/98] If a movie or series does not have sufficient info it is logged before returning value. --- src/video.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/src/video.py b/src/video.py index a761d06..141cd1e 100644 --- a/src/video.py +++ b/src/video.py @@ -167,7 +167,19 @@ class Episode(Video): return cls.fromguess(name, guessit(name, {'type': 'episode'})) def sufficientInfo(self): - return None not in [self.series, self.season, self.episode] and list not in [type(self.series), type(self.episode)] + ser = hasattr(self, 'series') + sea = hasattr(self, 'season') + ep = hasattr(self, 'episode') + + if False in [ser, sea, ep]: + logger.error('{}, {} or {} found to have none value, manual correction required'.format(self.series, self.season, self.episode)) + return False + + if list in [type(self.series), type(self.season), type(self.episode)]: + logger.error('{}, {} or {} found to have list values, manual correction required'.format(self.series, self.season, self.episode)) + return False + + return True def moveLocation(self): series = titlecase(self.series) @@ -217,7 +229,15 @@ class Movie(Video): def sufficientInfo(self): t = hasattr(self, "title") y = hasattr(self, "year") - return None not in [t, y] + + if None in [t, y]: + logger.error('{} or {} found to have none value, manual correction required'.format(self.title, self.year)) + return False + if list in [type(self.title), type(self.year)]: + logger.error('{} or {} found to have list value, manual correction required'.format(self.title, self.year)) + return False + + return True def moveLocation(self): title = titlecase(self.title) From f92a3fc2a2ecb38d2880884f1df782c683dd420f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sun, 23 Sep 2018 18:00:24 +0200 Subject: [PATCH 51/98] Added custom logger instance to core and import in video. --- src/core.py | 37 ++++++++++++++++++++++++------------- src/video.py | 5 ++++- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/src/core.py b/src/core.py index 387a8b5..b1b1376 100755 --- a/src/core.py +++ b/src/core.py @@ -22,6 +22,17 @@ from subtitle import SUBTITLE_EXTENSIONS, Subtitle, get_subtitle_path from utils import sanitize, refine logging.basicConfig(filename=env.logfile, level=logging.INFO) +logger = logging.getLogger('seasonedParser_core') +fh = logging.FileHandler(env.logfile) +fh.setLevel(logging.INFO) +ch = logging.StreamHandler() +ch.setLevel(logging.ERROR) +formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +fh.setFormatter(formatter) +ch.setFormatter(formatter) + +logger.addHandler(fh) +logger.addHandler(ch) def search_external_subtitles(path, directory=None): @@ -40,10 +51,10 @@ def search_external_subtitles(path, directory=None): try: language = Language.fromietf(language_code) except (ValueError, LanguageReverseError): - logging.error('Cannot parse language code %r', language_code) + logger.error('Cannot parse language code %r', language_code) subtitles[p] = language - logging.debug('Found subtitles %r', subtitles) + logger.debug('Found subtitles %r', subtitles) return subtitles @@ -64,7 +75,7 @@ def scan_video(path): raise ValueError('%r is not a valid video extension' % os.path.splitext(path)[1]) dirpath, filename = os.path.split(path) - logging.info('Scanning video %r in %r', filename, dirpath) + logger.info('Scanning video %r in %r', filename, dirpath) # guess video = Video.fromguess(path, guessit(path)) @@ -93,7 +104,7 @@ def scan_subtitle(path): raise ValueError('Path does not exist') dirpath, filename = os.path.split(path) - logging.info('Scanning subtitle %r in %r', filename, dirpath) + logger.info('Scanning subtitle %r in %r', filename, dirpath) # guess parent_path = path.strip(filename) @@ -128,30 +139,30 @@ def scan_videos(path): # walk the path videos = [] for dirpath, dirnames, filenames in os.walk(path): - logging.debug('Walking directory %r', dirpath) + logger.debug('Walking directory %r', dirpath) # remove badly encoded and hidden dirnames for dirname in list(dirnames): if dirname.startswith('.'): - logging.debug('Skipping hidden dirname %r in %r', dirname, dirpath) + logger.debug('Skipping hidden dirname %r in %r', dirname, dirpath) dirnames.remove(dirname) # scan for videos for filename in filenames: if not (filename.endswith(VIDEO_EXTENSIONS)): - logging.debug('Skipping non-video file %s', filename) + logger.debug('Skipping non-video file %s', filename) continue # skip hidden files if filename.startswith('.'): - logging.debug('Skipping hidden filename %r in %r', filename, dirpath) + logger.debug('Skipping hidden filename %r in %r', filename, dirpath) continue # reconstruct the file path filepath = os.path.join(dirpath, filename) if os.path.islink(filepath): - logging.debug('Skipping link %r in %r', filename, dirpath) + logger.debug('Skipping link %r in %r', filename, dirpath) continue # scan @@ -159,7 +170,7 @@ def scan_videos(path): try: video = scan_video(filepath) except ValueError: # pragma: no cover - logging.exception('Error scanning video') + logger.exception('Error scanning video') continue else: # pragma: no cover raise ValueError('Unsupported file %r' % filename) @@ -232,14 +243,14 @@ def scan_folder(path): videos = [] ignored_videos = [] errored_paths = [] - logging.debug('Collecting path %s', path) + logger.debug('Collecting path %s', path) # non-existing if not os.path.exists(path): try: video = Video.fromname(path) except: - logging.exception('Unexpected error while collecting non-existing path %s', path) + logger.exception('Unexpected error while collecting non-existing path %s', path) errored_paths.append(path) video.subtitles |= set(search_external_subtitles(video.name, directory=path)) @@ -253,7 +264,7 @@ def scan_folder(path): try: scanned_videos = scan_videos(path) except: - logging.exception('Unexpected error while collecting directory path %s', path) + logger.exception('Unexpected error while collecting directory path %s', path) errored_paths.append(path) # Iterates over our scanned videos diff --git a/src/video.py b/src/video.py index 141cd1e..9019b91 100644 --- a/src/video.py +++ b/src/video.py @@ -7,9 +7,12 @@ from guessit import guessit import os +import logging from titlecase import titlecase import hashlib, tvdb_api +logger = logging.getLogger('seasonedParser_core') + #: Video extensions VIDEO_EXTENSIONS = ('.3g2', '.3gp', '.3gp2', '.3gpp', '.60d', '.ajp', '.asf', '.asx', '.avchd', '.avi', '.bik', '.bix', '.box', '.cam', '.dat', '.divx', '.dmf', '.dv', '.dvr-ms', '.evo', '.flc', '.fli', @@ -190,7 +193,7 @@ class Episode(Video): def __repr__(self): if self.year is None: return '<%s [%r, %dx%s]>' % (self.__class__.__name__, self.series, self.season, self.episode) - if self.subtitles is not None: + if self.subtitles is not (None or set): return '<%s [%r, %dx%s] %s>' % (self.__class__.__name__, self.series, self.season, self.episode, self.subtitles) return '<%s [%r, %d, %dx%d]>' % (self.__class__.__name__, self.series, self.year, self.season, self.episode) From abb9ddbf09ce65e1eda74171b1f8beb8300f9c31 Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Sun, 23 Sep 2018 18:01:55 +0200 Subject: [PATCH 52/98] Removed unused print and fixed show path and sanity check. --- src/core.py | 1 - src/video.py | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/core.py b/src/core.py index 387a8b5..3fd927c 100755 --- a/src/core.py +++ b/src/core.py @@ -275,7 +275,6 @@ def scan_folder(path): def pickforgirlscouts(video): if video.sufficientInfo(): video.moveLocation() - print(video.home) return True return False diff --git a/src/video.py b/src/video.py index 141cd1e..4eae332 100644 --- a/src/video.py +++ b/src/video.py @@ -183,14 +183,14 @@ class Episode(Video): def moveLocation(self): series = titlecase(self.series) - grandParent = '{}/{} {:02d}'.format(series, series, self.season) + grandParent = '{}/{} Season {:02d}'.format(series, series, self.season) parent = '{} S{:02d}E{:02d}'.format(series, self.season, self.episode) - self.home = os.path.join(grandParent, parent, self.name) + self.home = os.path.join(grandParent, parent, os.path.basename(self.name)) def __repr__(self): if self.year is None: return '<%s [%r, %dx%s]>' % (self.__class__.__name__, self.series, self.season, self.episode) - if self.subtitles is not None: + if self.subtitles is not None and len(self.subtitles) > 0: return '<%s [%r, %dx%s] %s>' % (self.__class__.__name__, self.series, self.season, self.episode, self.subtitles) return '<%s [%r, %d, %dx%d]>' % (self.__class__.__name__, self.series, self.year, self.season, self.episode) From 11f5c16336e4e5732dc085d52d2852c9d52a6e3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sun, 23 Sep 2018 20:13:30 +0200 Subject: [PATCH 53/98] Added moviebase and showbase that represents the absolute base for the movie and show folders on disk. --- src/env_variables.py | 2 ++ src/video.py | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/env_variables.py b/src/env_variables.py index 4fd250a..0bb275a 100644 --- a/src/env_variables.py +++ b/src/env_variables.py @@ -1 +1,3 @@ logfile = 'conf/output.log' +MOVIEBASE = '/mnt/mainframe/movies' +SHOWBASE = '/mnt/mainframe/shows' \ No newline at end of file diff --git a/src/video.py b/src/video.py index baeae98..baf2348 100644 --- a/src/video.py +++ b/src/video.py @@ -11,6 +11,8 @@ import logging from titlecase import titlecase import hashlib, tvdb_api +import env_variables as env + logger = logging.getLogger('seasonedParser_core') #: Video extensions @@ -188,7 +190,7 @@ class Episode(Video): series = titlecase(self.series) grandParent = '{}/{} Season {:02d}'.format(series, series, self.season) parent = '{} S{:02d}E{:02d}'.format(series, self.season, self.episode) - self.home = os.path.join(grandParent, parent, os.path.basename(self.name)) + self.home = os.path.join(env.SHOWBASE, grandParent, parent, os.path.basename(self.name)) def __repr__(self): if self.year is None: @@ -245,7 +247,7 @@ class Movie(Video): def moveLocation(self): title = titlecase(self.title) parent = '{} ({})'.format(title, self.year) - self.home = os.path.join(parent, os.path.basename(self.name)) + self.home = os.path.join(env.MOVIEBASE, parent, os.path.basename(self.name)) def __repr__(self): if self.year is None: From 55f435109f0d298e945d13847cae73c8b8ce3979 Mon Sep 17 00:00:00 2001 From: KevinMidboe Date: Sun, 23 Sep 2018 20:37:26 +0200 Subject: [PATCH 54/98] Added langdetect for last resort when trying to find the subtitle language. It reads the first 1000 lines of the file and guesses the locale of the text. --- requirements.txt | 1 + src/core.py | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/requirements.txt b/requirements.txt index 79e1ed8..01e0530 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ tvdb_api==2.0 hashids==1.2.0 enzyme>=0.4.1 click>=6.7 +langdetect>=1.0.7 diff --git a/src/core.py b/src/core.py index 3b0521e..8457b54 100755 --- a/src/core.py +++ b/src/core.py @@ -14,6 +14,7 @@ import tvdb_api import click from pprint import pprint from titlecase import titlecase +import langdetect import env_variables as env @@ -49,6 +50,16 @@ def search_external_subtitles(path, directory=None): except (ValueError, LanguageReverseError): logger.error('Cannot parse language code %r', language_code) + f = open(p, 'r', encoding='ISO-8859-15') + + pattern = re.compile('[0-9:\,-<>]+') + # head = list(islice(f.read(), 10)) + filecontent = pattern.sub('', f.read()) + filecontent = filecontent[0:1000] + language = langdetect.detect(filecontent) + print(language) + f.close() + subtitles[p] = language logger.debug('Found subtitles %r', subtitles) From 4e055403d1a104278ad1630736d0b342fcfdbeb4 Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Sun, 23 Sep 2018 21:00:14 +0200 Subject: [PATCH 55/98] Imported re (regex lib) and fixed pathing issue when reading subtitle --- src/core.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core.py b/src/core.py index 8457b54..b7da6ca 100755 --- a/src/core.py +++ b/src/core.py @@ -10,6 +10,7 @@ from babelfish import Language, LanguageReverseError import hashlib import os, errno import logging +import re import tvdb_api import click from pprint import pprint @@ -50,7 +51,7 @@ def search_external_subtitles(path, directory=None): except (ValueError, LanguageReverseError): logger.error('Cannot parse language code %r', language_code) - f = open(p, 'r', encoding='ISO-8859-15') + f = open(os.path.join(dirpath, p), 'r', encoding='ISO-8859-15') pattern = re.compile('[0-9:\,-<>]+') # head = list(islice(f.read(), 10)) From b40f0f91ef672183d19bf7f0bbc6b5306ce68a04 Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Sun, 23 Sep 2018 21:26:57 +0200 Subject: [PATCH 56/98] Added subtitle_path function which finds the dirpath based on video sibling and subtitle name. --- src/core.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/core.py b/src/core.py index b7da6ca..014cdb5 100755 --- a/src/core.py +++ b/src/core.py @@ -58,7 +58,6 @@ def search_external_subtitles(path, directory=None): filecontent = pattern.sub('', f.read()) filecontent = filecontent[0:1000] language = langdetect.detect(filecontent) - print(language) f.close() subtitles[p] = language @@ -121,6 +120,10 @@ def scan_subtitle(path): return subtitle +def subtitle_path(sibling, subtitle): + parent_path = os.path.dirname(sibling) + return os.path.join(parent_path, subtitle) + def scan_videos(path): """Scan `path` for videos and their subtitles. From 03902eeecbab20d319e0cc47c11382a58e276a1c Mon Sep 17 00:00:00 2001 From: KevinMidboe Date: Sun, 23 Sep 2018 21:36:03 +0200 Subject: [PATCH 57/98] Created the big move home function which will create the parent folders and move all videos and subtitles. --- src/core.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/core.py b/src/core.py index 014cdb5..7a1aa5f 100755 --- a/src/core.py +++ b/src/core.py @@ -301,6 +301,20 @@ def pickforgirlscouts(video): return False +def moveHome(video): + dir = os.path.dirname(video.home) + + if not os.path.exists(dir): + logger.info('Creating directory {}'.format(dir)) + os.makedirs(dir) + + logger.info("Moving video file from: '{}' to: '{}'".format(video.name, video.home)) + shutil.move(video.name, video.home) + for sub in video.subtitles: + sub_home = subtitle_path(sub) + logger.info("Moving subtitle file from: '{}' to: '{}'".format(sub, sub_home)) + shutil.move(sub, sub_home) + def main(): path = '/mnt/mainframe/' @@ -324,7 +338,7 @@ def main(): )) for video in scout: - print('{} lives: {}'.format(video, video.home)) + moveHome(video) if __name__ == '__main__': main() From 66f7ea3e24b61e2719677e86252ade9e347737e5 Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Sun, 23 Sep 2018 22:38:26 +0200 Subject: [PATCH 58/98] Because we read subtitles from mkv containers a check for if subtitle file exists was added so not to try move a sub file within a mkv container. --- src/core.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/core.py b/src/core.py index 7a1aa5f..6eb7fb4 100755 --- a/src/core.py +++ b/src/core.py @@ -9,6 +9,7 @@ from guessit import guessit from babelfish import Language, LanguageReverseError import hashlib import os, errno +import shutil import logging import re import tvdb_api @@ -60,7 +61,7 @@ def search_external_subtitles(path, directory=None): language = langdetect.detect(filecontent) f.close() - subtitles[p] = language + subtitles[os.path.join(dirpath, p)] = language logger.debug('Found subtitles %r', subtitles) return subtitles @@ -122,7 +123,7 @@ def scan_subtitle(path): def subtitle_path(sibling, subtitle): parent_path = os.path.dirname(sibling) - return os.path.join(parent_path, subtitle) + return os.path.join(parent_path, os.path.basename(subtitle)) def scan_videos(path): """Scan `path` for videos and their subtitles. @@ -311,9 +312,12 @@ def moveHome(video): logger.info("Moving video file from: '{}' to: '{}'".format(video.name, video.home)) shutil.move(video.name, video.home) for sub in video.subtitles: - sub_home = subtitle_path(sub) - logger.info("Moving subtitle file from: '{}' to: '{}'".format(sub, sub_home)) - shutil.move(sub, sub_home) + if not os.path.isfile(sub): + continue + oldpath = sub + newpath = subtitle_path(video.home, sub) + logger.info("Moving subtitle file from: '{}' to: '{}'".format(oldpath, newpath)) + shutil.move(oldpath, newpath) def main(): path = '/mnt/mainframe/' From aeacd8a5b67986c560eb96b869d3a518c18b4105 Mon Sep 17 00:00:00 2001 From: KevinMidboe Date: Tue, 16 Oct 2018 23:06:45 +0200 Subject: [PATCH 59/98] Started cli for seasonedparser --- src/cli.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100755 src/cli.py diff --git a/src/cli.py b/src/cli.py new file mode 100755 index 0000000..074eb68 --- /dev/null +++ b/src/cli.py @@ -0,0 +1,39 @@ +#/usr/local/bin/python3 +import click +import os +import logging + +import env_variables as env + +logging.basicConfig(filename=env.logfile, level=logging.INFO) +logger = logging.getLogger('seasonedParser') +fh = logging.FileHandler(env.logfile) +fh.setLevel(logging.INFO) + +formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +fh.setFormatter(formatter) + +logger.addHandler(fh) + +def listPath(path): + if (os.path.isdir(path)): + print('Contents of path:') + print(os.listdir(path)) + + elif os.path.isfile(path): + print('File to parse:') + print(path) + + else: + print('Path does not exists') + +@click.command() +@click.argument('path') +@click.option('--greeting', '-g') +def main(path, greeting): + logger.info('Received cli variables: \n\t path: {}'.format(path)) + listPath(path) + + +if __name__ == '__main__': + main() From 6d67ce121423f48d36cbb51d3aa1e2aedacaa59c Mon Sep 17 00:00:00 2001 From: KevinMidboe Date: Tue, 16 Oct 2018 23:07:05 +0200 Subject: [PATCH 60/98] Cli roadmap --- README.md | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 79dd1da..1244e8c 100644 --- a/README.md +++ b/README.md @@ -26,4 +26,19 @@ There are many run commands for this, but here is a list of the current working ``` Here the first parameter is our move command, which in turn calls motherMover. The second parameter is what we want the filenames to be called. Notice the (num1..num2), this is to create a range for all the episodes we want to move. The last parameter is the path we want to move our content. - > This will be done automatically by the parser based on the info in the media items name, but it is nice to have a manual command. \ No newline at end of file + > This will be done automatically by the parser based on the info in the media items name, but it is nice to have a manual command. + + +## Cli + +Arguments +* Dry run with --dry +* Path variable +* daemon with -d option + * Still need the path variable + * Daemon sends confirmation and on missing asks tweetf or correction + +Functions +* Should ask for input when missing info, always when cli + + From 9639b5625187bea53a78aee7c2be7f9d46542f6e Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Wed, 17 Oct 2018 19:46:46 +0200 Subject: [PATCH 61/98] Readded streamhandler to logger. --- src/core.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/core.py b/src/core.py index 6eb7fb4..1eeb665 100755 --- a/src/core.py +++ b/src/core.py @@ -24,15 +24,19 @@ from video import VIDEO_EXTENSIONS, Episode, Movie, Video from subtitle import SUBTITLE_EXTENSIONS, Subtitle, get_subtitle_path from utils import sanitize, refine -logging.basicConfig(filename=env.logfile, level=logging.INFO) +logging.basicConfig(filename=env.logfile, level=logging.DEBUG) logger = logging.getLogger('seasonedParser_core') fh = logging.FileHandler(env.logfile) -fh.setLevel(logging.INFO) +fh.setLevel(logging.DEBUG) +sh = logging.StreamHandler() +sh.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') fh.setFormatter(formatter) +sh.setFormatter(formatter) logger.addHandler(fh) +logger.addHandler(sh) def search_external_subtitles(path, directory=None): dirpath, filename = os.path.split(path) From dd1f49d53b4a7d58641efe619dba581de2a2d528 Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Wed, 17 Oct 2018 19:55:38 +0200 Subject: [PATCH 62/98] When path does not exist the user is notified and the path is added to errored_paths. Added info logs for if path is a file or directory. --- src/core.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/core.py b/src/core.py index 1eeb665..d721319 100755 --- a/src/core.py +++ b/src/core.py @@ -261,22 +261,29 @@ def scan_folder(path): errored_paths = [] logger.debug('Collecting path %s', path) + # non-existing if not os.path.exists(path): - try: - video = Video.fromname(path) - except: - logger.exception('Unexpected error while collecting non-existing path %s', path) - errored_paths.append(path) + errored_paths.append(path) + logger.exception("The path '{}' does not exist".format(path)) - video.subtitles |= set(search_external_subtitles(video.name, directory=path)) + # file + # if path is a file + if os.path.isfile(path): + logger.info('Path is a file') + try: + video = scan_video(path) + except: + logger.exception('Unexpected error while collection file with path {}'.format(path)) + video.subtitles |= set(search_external_subtitles(video.name)) + refine(video) videos.append(video) - # Increment bar to full ? # directories if os.path.isdir(path): + logger.info('Path is a directory') try: scanned_videos = scan_videos(path) except: From 5a6486189e6c1e02ceb3d03f5f8dbc6ce77e605b Mon Sep 17 00:00:00 2001 From: KevinMidboe Date: Wed, 17 Oct 2018 19:59:43 +0200 Subject: [PATCH 63/98] Merged to updated logger name. --- src/cli.py | 4 ++++ src/core.py | 10 +++++++--- src/video.py | 24 ++++++++++++------------ 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/src/cli.py b/src/cli.py index 074eb68..05c817e 100755 --- a/src/cli.py +++ b/src/cli.py @@ -27,6 +27,10 @@ def listPath(path): else: print('Path does not exists') +def guessFromInput(video): + print('Insufficient info for {}'.format(video.name)) + video_name = input('Input + @click.command() @click.argument('path') @click.option('--greeting', '-g') diff --git a/src/core.py b/src/core.py index d721319..480e49d 100755 --- a/src/core.py +++ b/src/core.py @@ -25,7 +25,7 @@ from subtitle import SUBTITLE_EXTENSIONS, Subtitle, get_subtitle_path from utils import sanitize, refine logging.basicConfig(filename=env.logfile, level=logging.DEBUG) -logger = logging.getLogger('seasonedParser_core') +logger = logging.getLogger('seasonedParser') fh = logging.FileHandler(env.logfile) fh.setLevel(logging.DEBUG) sh = logging.StreamHandler() @@ -70,6 +70,9 @@ def search_external_subtitles(path, directory=None): return subtitles +def find_file_size(video): + return os.path.getsize(video.name) + def scan_video(path): """Scan a video from a `path`. @@ -92,8 +95,8 @@ def scan_video(path): # guess video = Video.fromguess(path, guessit(path)) - # size - video.size = os.path.getsize(path) + if video.sufficientInfo(): + video.setMoveLocation() # hash of name # if isinstance(video, Movie): @@ -296,6 +299,7 @@ def scan_folder(path): v.subtitles |= set(search_external_subtitles(v.name)) refine(v) videos.append(v) + video.size = find_file_size() click.echo('%s video%s collected / %s error%s' % ( click.style(str(len(videos)), bold=True, fg='green' if videos else None), diff --git a/src/video.py b/src/video.py index baf2348..9790788 100644 --- a/src/video.py +++ b/src/video.py @@ -13,7 +13,7 @@ import hashlib, tvdb_api import env_variables as env -logger = logging.getLogger('seasonedParser_core') +logger = logging.getLogger('seasonedParser') #: Video extensions VIDEO_EXTENSIONS = ('.3g2', '.3gp', '.3gp2', '.3gpp', '.60d', '.ajp', '.asf', '.asx', '.avchd', '.avi', '.bik', @@ -33,13 +33,13 @@ class Video(object): :param str resolution: resolution of the video stream (480p, 720p, 1080p or 1080i, 4K). :param str video_codec: codec of the video stream. :param str audio_codec: codec of the main audio stream. - :param str home: optimal parent folder. + :param str move_location: location to move file to. :param dict name_hash: hashes of the video file by provider names. :param int size: size of the video file in bytes. :param set subtitles: existing subtitle languages. """ def __init__(self, name, hash=None, size=None, format=None, release_group=None, resolution=None, video_codec=None, audio_codec=None, - home=None, subtitles=None, embeded_subtitles=None): + move_location=None, subtitles=None, embeded_subtitles=None): #: Name or path of the video self.name = name @@ -64,8 +64,8 @@ class Video(object): #: Codec of the main audio stream self.audio_codec = audio_codec - #: optimal home path; parent folder. - self.home = home + #: optimal move_location path; parent folder. + self.move_location = move_location #: Existing subtitle languages self.subtitles = subtitles or set() @@ -159,7 +159,7 @@ class Episode(Video): if guess['type'] != 'episode': raise ValueError('The guess must be an episode guess') - if 'title' not in guess or 'episode' not in guess: + if 'title' not in guess or 'season' not in guess or 'episode' not in guess: raise ValueError('Insufficient data to process the guess') return cls(name, guess['title'], guess.get('season', 1), guess['episode'], title=guess.get('episode_title'), @@ -186,11 +186,11 @@ class Episode(Video): return True - def moveLocation(self): + def setMoveLocation(self): series = titlecase(self.series) grandParent = '{}/{} Season {:02d}'.format(series, series, self.season) parent = '{} S{:02d}E{:02d}'.format(series, self.season, self.episode) - self.home = os.path.join(env.SHOWBASE, grandParent, parent, os.path.basename(self.name)) + self.move_location = os.path.join(env.SHOWBASE, grandParent, parent, os.path.basename(self.name)) def __repr__(self): if self.year is None: @@ -220,7 +220,7 @@ class Movie(Video): if guess['type'] != 'movie': raise ValueError('The guess must be a movie guess') - if 'title' not in guess: + if 'title' not in guess or 'year' not in guess: raise ValueError('Insufficient data to process the guess') return cls(name, guess['title'], format=guess.get('format'), release_group=guess.get('release_group'), @@ -228,7 +228,7 @@ class Movie(Video): audio_codec=guess.get('audio_codec'), year=guess.get('year')) @classmethod - def fromname(cls, name): + def fromname(cls, name, year): return cls.fromguess(name, guessit(name, {'type': 'movie'})) def sufficientInfo(self): @@ -244,10 +244,10 @@ class Movie(Video): return True - def moveLocation(self): + def setMoveLocation(self): title = titlecase(self.title) parent = '{} ({})'.format(title, self.year) - self.home = os.path.join(env.MOVIEBASE, parent, os.path.basename(self.name)) + self.move_location = os.path.join(env.MOVIEBASE, parent, os.path.basename(self.name)) def __repr__(self): if self.year is None: From 86d17c5e22a41a44a24df0107d9677070e343054 Mon Sep 17 00:00:00 2001 From: KevinMidboe Date: Wed, 17 Oct 2018 22:51:56 +0200 Subject: [PATCH 64/98] Recatored the cli program. --- requirements.txt | 1 + src/cli.py | 52 ++++++++++++------------------------------------ 2 files changed, 14 insertions(+), 39 deletions(-) diff --git a/requirements.txt b/requirements.txt index 01e0530..0a9c345 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ hashids==1.2.0 enzyme>=0.4.1 click>=6.7 langdetect>=1.0.7 +titlecase>=0.12.0 diff --git a/src/cli.py b/src/cli.py index 05c817e..9060dfe 100755 --- a/src/cli.py +++ b/src/cli.py @@ -1,43 +1,17 @@ -#/usr/local/bin/python3 -import click -import os -import logging +#!usr/bin/env python3.6 -import env_variables as env +from core import scan_folder, moveHome +from video import Video +from guessit import guessit -logging.basicConfig(filename=env.logfile, level=logging.INFO) -logger = logging.getLogger('seasonedParser') -fh = logging.FileHandler(env.logfile) -fh.setLevel(logging.INFO) +videos, insufficient_info = scan_folder('Spider.Man') +print('Sweet lemonade: {} {}'.format(videos, insufficient_info)) -formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') -fh.setFormatter(formatter) +for video in videos: + moveHome(video) -logger.addHandler(fh) - -def listPath(path): - if (os.path.isdir(path)): - print('Contents of path:') - print(os.listdir(path)) - - elif os.path.isfile(path): - print('File to parse:') - print(path) - - else: - print('Path does not exists') - -def guessFromInput(video): - print('Insufficient info for {}'.format(video.name)) - video_name = input('Input - -@click.command() -@click.argument('path') -@click.option('--greeting', '-g') -def main(path, greeting): - logger.info('Received cli variables: \n\t path: {}'.format(path)) - listPath(path) - - -if __name__ == '__main__': - main() +for file in insufficient_info: + supplementary_info = input("Insufficient info for match file: '{}'\nSupplementary info: ".format(file)) + print(supplementary_info) + video = Video.fromguess(file, guessit(supplementary_info)) + moveHome(video) From 50e6e4a2596659a1311ee6901ce6901c93a309a3 Mon Sep 17 00:00:00 2001 From: KevinMidboe Date: Wed, 17 Oct 2018 22:58:22 +0200 Subject: [PATCH 65/98] Insufficient exception is thrown when not enough info is needed to move file to correct location. All insufficient items are returned along with the found videos. The wanted path we want to move the file is no longer class vairable, but gets the string by function. --- src/core.py | 46 +++++++++++++++++++++++----------------------- src/video.py | 28 +++++++--------------------- 2 files changed, 30 insertions(+), 44 deletions(-) diff --git a/src/core.py b/src/core.py index 480e49d..29ca813 100755 --- a/src/core.py +++ b/src/core.py @@ -19,6 +19,7 @@ from titlecase import titlecase import langdetect import env_variables as env +from exceptions import InsufficientInfoError from video import VIDEO_EXTENSIONS, Episode, Movie, Video from subtitle import SUBTITLE_EXTENSIONS, Subtitle, get_subtitle_path @@ -95,8 +96,8 @@ def scan_video(path): # guess video = Video.fromguess(path, guessit(path)) - if video.sufficientInfo(): - video.setMoveLocation() + video.subtitles |= set(search_external_subtitles(video.name)) + refine(video) # hash of name # if isinstance(video, Movie): @@ -260,7 +261,7 @@ def save_subtitles(files, single=False, directory=None, encoding=None): def scan_folder(path): videos = [] - ignored_videos = [] + insufficient_info = [] errored_paths = [] logger.debug('Collecting path %s', path) @@ -274,41 +275,39 @@ def scan_folder(path): # if path is a file if os.path.isfile(path): logger.info('Path is a file') + try: video = scan_video(path) - except: - logger.exception('Unexpected error while collection file with path {}'.format(path)) - - video.subtitles |= set(search_external_subtitles(video.name)) + videos.append(video) - refine(video) - videos.append(video) + except InsufficientInfoError as e: + logger.error(e) + insufficient_info.append(path) # directories if os.path.isdir(path): logger.info('Path is a directory') + + scanned_videos = [] try: scanned_videos = scan_videos(path) + except InsufficientInfoError as e: + logger.error(e) + insufficient_info.append(path) except: logger.exception('Unexpected error while collecting directory path %s', path) errored_paths.append(path) - # Iterates over our scanned videos - with click.progressbar(scanned_videos, label='Parsing videos') as bar: - for v in bar: - v.subtitles |= set(search_external_subtitles(v.name)) - refine(v) - videos.append(v) - video.size = find_file_size() - - click.echo('%s video%s collected / %s error%s' % ( + click.echo('%s video%s collected / %s file%s with insufficient info / %s error%s' % ( click.style(str(len(videos)), bold=True, fg='green' if videos else None), 's' if len(videos) > 1 else '', + click.style(str(len(insufficient_info)), bold=True, fg='yellow' if insufficient_info else None), + 's' if len(insufficient_info) > 1 else '', click.style(str(len(errored_paths)), bold=True, fg='red' if errored_paths else None), 's' if len(errored_paths) > 1 else '', )) - return videos + return videos, insufficient_info def pickforgirlscouts(video): if video.sufficientInfo(): @@ -318,19 +317,20 @@ def pickforgirlscouts(video): return False def moveHome(video): - dir = os.path.dirname(video.home) + wantedFilePath = video.wantedFilePath() + dir = os.path.dirname(wantedFilePath) if not os.path.exists(dir): logger.info('Creating directory {}'.format(dir)) os.makedirs(dir) - logger.info("Moving video file from: '{}' to: '{}'".format(video.name, video.home)) - shutil.move(video.name, video.home) + logger.info("Moving video file from: '{}' to: '{}'".format(video.name, wantedFilePath)) + shutil.move(video.name, wantedFilePath) for sub in video.subtitles: if not os.path.isfile(sub): continue oldpath = sub - newpath = subtitle_path(video.home, sub) + newpath = subtitle_path(wantedFilePath, sub) logger.info("Moving subtitle file from: '{}' to: '{}'".format(oldpath, newpath)) shutil.move(oldpath, newpath) diff --git a/src/video.py b/src/video.py index 9790788..78af278 100644 --- a/src/video.py +++ b/src/video.py @@ -12,6 +12,7 @@ from titlecase import titlecase import hashlib, tvdb_api import env_variables as env +from exceptions import InsufficientInfoError logger = logging.getLogger('seasonedParser') @@ -160,7 +161,7 @@ class Episode(Video): raise ValueError('The guess must be an episode guess') if 'title' not in guess or 'season' not in guess or 'episode' not in guess: - raise ValueError('Insufficient data to process the guess') + raise InsufficientInfoError('Insufficient data to process the guess') return cls(name, guess['title'], guess.get('season', 1), guess['episode'], title=guess.get('episode_title'), year=guess.get('year'), format=guess.get('format'), original_series='year' not in guess, @@ -171,26 +172,11 @@ class Episode(Video): def fromname(cls, name): return cls.fromguess(name, guessit(name, {'type': 'episode'})) - def sufficientInfo(self): - ser = hasattr(self, 'series') - sea = hasattr(self, 'season') - ep = hasattr(self, 'episode') - - if False in [ser, sea, ep]: - logger.error('{}, {} or {} found to have none value, manual correction required'.format(self.series, self.season, self.episode)) - return False - - if list in [type(self.series), type(self.season), type(self.episode)]: - logger.error('{}, {} or {} found to have list values, manual correction required'.format(self.series, self.season, self.episode)) - return False - - return True - - def setMoveLocation(self): + def wantedFilePath(self): series = titlecase(self.series) grandParent = '{}/{} Season {:02d}'.format(series, series, self.season) parent = '{} S{:02d}E{:02d}'.format(series, self.season, self.episode) - self.move_location = os.path.join(env.SHOWBASE, grandParent, parent, os.path.basename(self.name)) + return os.path.join(env.SHOWBASE, grandParent, parent, os.path.basename(self.name)) def __repr__(self): if self.year is None: @@ -221,7 +207,7 @@ class Movie(Video): raise ValueError('The guess must be a movie guess') if 'title' not in guess or 'year' not in guess: - raise ValueError('Insufficient data to process the guess') + raise InsufficientInfoError('Insufficient data to process the guess') return cls(name, guess['title'], format=guess.get('format'), release_group=guess.get('release_group'), resolution=guess.get('screen_size'), video_codec=guess.get('video_codec'), @@ -244,10 +230,10 @@ class Movie(Video): return True - def setMoveLocation(self): + def wantedFilePath(self): title = titlecase(self.title) parent = '{} ({})'.format(title, self.year) - self.move_location = os.path.join(env.MOVIEBASE, parent, os.path.basename(self.name)) + return os.path.join(env.MOVIEBASE, parent, os.path.basename(self.name)) def __repr__(self): if self.year is None: From 7e7eebd462949032e1024484557791ab86f9bbac Mon Sep 17 00:00:00 2001 From: KevinMidboe Date: Wed, 17 Oct 2018 23:00:06 +0200 Subject: [PATCH 66/98] Removed main function from core. --- src/core.py | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/src/core.py b/src/core.py index 29ca813..79c83a4 100755 --- a/src/core.py +++ b/src/core.py @@ -334,31 +334,3 @@ def moveHome(video): logger.info("Moving subtitle file from: '{}' to: '{}'".format(oldpath, newpath)) shutil.move(oldpath, newpath) -def main(): - path = '/mnt/mainframe/' - - videos = scan_folder(path) - - scout = [] - civilian = [] - for video in videos: - if pickforgirlscouts(video): - scout.append(video) - else: - civilian.append(video) - - click.echo('%s scout%s collected / %s civilan%s / %s candidate%s' % ( - click.style(str(len(scout)), bold=True, fg='green' if scout else None), - 's' if len(scout) > 1 else '', - click.style(str(len(civilian)), bold=True, fg='red' if civilian else None), - 's' if len(civilian) > 1 else '', - click.style(str(len(videos)), bold=True, fg='blue' if videos else None), - 's' if len(videos) > 1 else '' - )) - - for video in scout: - moveHome(video) - -if __name__ == '__main__': - main() - From 37a0c6f62bc4b5d04d602c370473c69ce13794e6 Mon Sep 17 00:00:00 2001 From: KevinMidboe Date: Wed, 17 Oct 2018 23:29:09 +0200 Subject: [PATCH 67/98] Will iterate over all files until all have a files have sufficient info. --- src/cli.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/cli.py b/src/cli.py index 9060dfe..0abf09f 100755 --- a/src/cli.py +++ b/src/cli.py @@ -4,14 +4,22 @@ from core import scan_folder, moveHome from video import Video from guessit import guessit +from exceptions import InsufficientInfoError + videos, insufficient_info = scan_folder('Spider.Man') print('Sweet lemonade: {} {}'.format(videos, insufficient_info)) for video in videos: moveHome(video) -for file in insufficient_info: - supplementary_info = input("Insufficient info for match file: '{}'\nSupplementary info: ".format(file)) - print(supplementary_info) - video = Video.fromguess(file, guessit(supplementary_info)) - moveHome(video) +while len(insufficient_info) > 1: + for file in insufficient_info: + supplementary_info = input("Insufficient info for match file: '{}'\nSupplementary info: ".format(file)) + print(supplementary_info) + try: + video = Video.fromguess(file, guessit(supplementary_info)) + insufficient_info.pop() + except InsufficientInfoError: + pass + + moveHome(video) From 2d4f2b003bbcec27fdac63e22f0b2e2da4144412 Mon Sep 17 00:00:00 2001 From: KevinMidboe Date: Wed, 17 Oct 2018 23:34:49 +0200 Subject: [PATCH 68/98] Added exceptions file. --- src/exceptions.py | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 src/exceptions.py diff --git a/src/exceptions.py b/src/exceptions.py new file mode 100644 index 0000000..116b7fd --- /dev/null +++ b/src/exceptions.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python3.6 + +class InsufficientInfoError(Exception): + pass + From 8b3d083938d438b32d4290b6d9851fec4bcb62e6 Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Thu, 18 Oct 2018 20:27:03 +0200 Subject: [PATCH 69/98] Handles input from argv as path. --- src/cli.py | 46 +++++++++++++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/src/cli.py b/src/cli.py index 0abf09f..e1bc7bb 100755 --- a/src/cli.py +++ b/src/cli.py @@ -1,25 +1,37 @@ #!usr/bin/env python3.6 +from guessit import guessit +import click from core import scan_folder, moveHome from video import Video -from guessit import guessit - from exceptions import InsufficientInfoError -videos, insufficient_info = scan_folder('Spider.Man') -print('Sweet lemonade: {} {}'.format(videos, insufficient_info)) +@click.command() +@click.argument('path') +def main(path): + videos, insufficient_info = scan_folder(path) + # print('Sweet lemonade: {} {}'.format(videos, insufficient_info)) -for video in videos: - moveHome(video) - -while len(insufficient_info) > 1: - for file in insufficient_info: - supplementary_info = input("Insufficient info for match file: '{}'\nSupplementary info: ".format(file)) - print(supplementary_info) - try: - video = Video.fromguess(file, guessit(supplementary_info)) - insufficient_info.pop() - except InsufficientInfoError: - pass - + for video in videos: moveHome(video) + + while len(insufficient_info) >= 1: + for file in insufficient_info: + supplementary_info = input("Insufficient info for match file: '{}'\nSupplementary info: ".format(file)) + + if supplementary_info is 'q': + exit(0) + if supplementary_info is 's': + insufficient_info.pop() + continue + + try: + video = Video.fromguess(file, guessit(supplementary_info)) + print(video) + moveHome(video) + insufficient_info.pop() + except InsufficientInfoError: + pass + +if __name__ == '__main__': + main() From c3bb6c646975423c8a46ee01d4cc33036116e456 Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Thu, 18 Oct 2018 20:48:59 +0200 Subject: [PATCH 70/98] If the name is not sufficient for a valid guess a insuffienet name error is thrown which is not handled correctly for both files and folders, then returned to the user to try move it by changing input name. --- src/core.py | 39 +++++++++++++++++++++++---------------- src/exceptions.py | 2 +- src/video.py | 22 +++++++++------------- 3 files changed, 33 insertions(+), 30 deletions(-) diff --git a/src/core.py b/src/core.py index 79c83a4..71eb0f8 100755 --- a/src/core.py +++ b/src/core.py @@ -19,7 +19,7 @@ from titlecase import titlecase import langdetect import env_variables as env -from exceptions import InsufficientInfoError +from exceptions import InsufficientNameError from video import VIDEO_EXTENSIONS, Episode, Movie, Video from subtitle import SUBTITLE_EXTENSIONS, Subtitle, get_subtitle_path @@ -28,9 +28,9 @@ from utils import sanitize, refine logging.basicConfig(filename=env.logfile, level=logging.DEBUG) logger = logging.getLogger('seasonedParser') fh = logging.FileHandler(env.logfile) -fh.setLevel(logging.DEBUG) +fh.setLevel(logging.INFO) sh = logging.StreamHandler() -sh.setLevel(logging.DEBUG) +sh.setLevel(logging.ERROR) formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') fh.setFormatter(formatter) @@ -158,6 +158,8 @@ def scan_videos(path): # walk the path videos = [] + insufficient_name = [] + errors_path = [] for dirpath, dirnames, filenames in os.walk(path): logger.debug('Walking directory %r', dirpath) @@ -189,8 +191,13 @@ def scan_videos(path): if filename.endswith(VIDEO_EXTENSIONS): # video try: video = scan_video(filepath) + except InsufficientInfoError as e: + logger.info(e) + insufficient_name.append(filepath) + continue except ValueError: # pragma: no cover logger.exception('Error scanning video') + errors_path.append(filepath) continue else: # pragma: no cover raise ValueError('Unsupported file %r' % filename) @@ -199,7 +206,7 @@ def scan_videos(path): bar.update(1) - return videos + return videos, insufficient_name, errors_path def organize_files(path): @@ -261,7 +268,7 @@ def save_subtitles(files, single=False, directory=None, encoding=None): def scan_folder(path): videos = [] - insufficient_info = [] + insufficient_name = [] errored_paths = [] logger.debug('Collecting path %s', path) @@ -280,9 +287,9 @@ def scan_folder(path): video = scan_video(path) videos.append(video) - except InsufficientInfoError as e: - logger.error(e) - insufficient_info.append(path) + except InsufficientNameError as e: + logger.info(e) + insufficient_name.append(path) # directories if os.path.isdir(path): @@ -290,24 +297,21 @@ def scan_folder(path): scanned_videos = [] try: - scanned_videos = scan_videos(path) - except InsufficientInfoError as e: - logger.error(e) - insufficient_info.append(path) + videos, insufficient_name, errored_paths = scan_videos(path) except: logger.exception('Unexpected error while collecting directory path %s', path) errored_paths.append(path) - click.echo('%s video%s collected / %s file%s with insufficient info / %s error%s' % ( + click.echo('%s video%s collected / %s file%s with insufficient name / %s error%s' % ( click.style(str(len(videos)), bold=True, fg='green' if videos else None), 's' if len(videos) > 1 else '', - click.style(str(len(insufficient_info)), bold=True, fg='yellow' if insufficient_info else None), - 's' if len(insufficient_info) > 1 else '', + click.style(str(len(insufficient_name)), bold=True, fg='yellow' if insufficient_name else None), + 's' if len(insufficient_name) > 1 else '', click.style(str(len(errored_paths)), bold=True, fg='red' if errored_paths else None), 's' if len(errored_paths) > 1 else '', )) - return videos, insufficient_info + return videos, insufficient_name def pickforgirlscouts(video): if video.sufficientInfo(): @@ -334,3 +338,6 @@ def moveHome(video): logger.info("Moving subtitle file from: '{}' to: '{}'".format(oldpath, newpath)) shutil.move(oldpath, newpath) +# Give feedback before delete ? +def empthDirectory(paths): + pass diff --git a/src/exceptions.py b/src/exceptions.py index 116b7fd..692e40a 100644 --- a/src/exceptions.py +++ b/src/exceptions.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3.6 -class InsufficientInfoError(Exception): +class InsufficientNameError(Exception): pass diff --git a/src/video.py b/src/video.py index 78af278..6c52f47 100644 --- a/src/video.py +++ b/src/video.py @@ -12,7 +12,7 @@ from titlecase import titlecase import hashlib, tvdb_api import env_variables as env -from exceptions import InsufficientInfoError +from exceptions import InsufficientNameError logger = logging.getLogger('seasonedParser') @@ -34,13 +34,11 @@ class Video(object): :param str resolution: resolution of the video stream (480p, 720p, 1080p or 1080i, 4K). :param str video_codec: codec of the video stream. :param str audio_codec: codec of the main audio stream. - :param str move_location: location to move file to. - :param dict name_hash: hashes of the video file by provider names. :param int size: size of the video file in bytes. :param set subtitles: existing subtitle languages. """ def __init__(self, name, hash=None, size=None, format=None, release_group=None, resolution=None, video_codec=None, audio_codec=None, - move_location=None, subtitles=None, embeded_subtitles=None): + subtitles=None, embeded_subtitles=None): #: Name or path of the video self.name = name @@ -65,9 +63,6 @@ class Video(object): #: Codec of the main audio stream self.audio_codec = audio_codec - #: optimal move_location path; parent folder. - self.move_location = move_location - #: Existing subtitle languages self.subtitles = subtitles or set() @@ -157,11 +152,15 @@ class Episode(Video): @classmethod def fromguess(cls, name, guess): + logger.info('Guess: {}'.format(guess)) if guess['type'] != 'episode': raise ValueError('The guess must be an episode guess') if 'title' not in guess or 'season' not in guess or 'episode' not in guess: - raise InsufficientInfoError('Insufficient data to process the guess') + raise InsufficientNameError('Guess failed to have sufficient data from query: {}'.format(name)) + + if any([isinstance(x, list) for x in [guess['title'], guess['season'], guess['episode']]]): + raise InsufficientNameError('Guess could not be parsed, list values found.') return cls(name, guess['title'], guess.get('season', 1), guess['episode'], title=guess.get('episode_title'), year=guess.get('year'), format=guess.get('format'), original_series='year' not in guess, @@ -179,12 +178,9 @@ class Episode(Video): return os.path.join(env.SHOWBASE, grandParent, parent, os.path.basename(self.name)) def __repr__(self): - if self.year is None: - return '<%s [%r, %dx%s]>' % (self.__class__.__name__, self.series, self.season, self.episode) if self.subtitles is not None and len(self.subtitles) > 0: return '<%s [%r, %dx%s] %s>' % (self.__class__.__name__, self.series, self.season, self.episode, self.subtitles) - - return '<%s [%r, %d, %dx%d]>' % (self.__class__.__name__, self.series, self.year, self.season, self.episode) + return '<%s [%r, %dx%d]>' % (self.__class__.__name__, self.series, self.season, self.episode) class Movie(Video): """Movie :class:`Video`. @@ -207,7 +203,7 @@ class Movie(Video): raise ValueError('The guess must be a movie guess') if 'title' not in guess or 'year' not in guess: - raise InsufficientInfoError('Insufficient data to process the guess') + raise InsufficientNameError('Guess failed to have sufficient data from query: {}'.format(name)) return cls(name, guess['title'], format=guess.get('format'), release_group=guess.get('release_group'), resolution=guess.get('screen_size'), video_codec=guess.get('video_codec'), From bac145c6a1ee91c18856bf072b5810871463fa52 Mon Sep 17 00:00:00 2001 From: KevinMidboe Date: Thu, 18 Oct 2018 22:09:47 +0200 Subject: [PATCH 71/98] Prompt for user input has been moved out to a funtion. Reflecting changes of exception name to InsufficientNameError. --- src/cli.py | 66 ++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 44 insertions(+), 22 deletions(-) diff --git a/src/cli.py b/src/cli.py index e1bc7bb..204d679 100755 --- a/src/cli.py +++ b/src/cli.py @@ -1,37 +1,59 @@ #!usr/bin/env python3.6 -from guessit import guessit import click +from guessit import guessit from core import scan_folder, moveHome from video import Video -from exceptions import InsufficientInfoError +from exceptions import InsufficientNameError + +def moveHome(video): + print('Would have moved: {}'.format(video)) + +def tweet(video): + pass + +def prompt(name): + manual_name = input("Insufficient name: '{}'\nInput name manually: ".format(name)) + + if manual_name is 'q': + assert KeyboardInterrupt + if manual_name is 's': + return None + + + return manual_name @click.command() @click.argument('path') -def main(path): - videos, insufficient_info = scan_folder(path) - # print('Sweet lemonade: {} {}'.format(videos, insufficient_info)) +@click.option('--daemon', '-d', daemon) +def main(path, daemon): + videos, insufficient_name = scan_folder(path) for video in videos: moveHome(video) - while len(insufficient_info) >= 1: - for file in insufficient_info: - supplementary_info = input("Insufficient info for match file: '{}'\nSupplementary info: ".format(file)) - - if supplementary_info is 'q': - exit(0) - if supplementary_info is 's': - insufficient_info.pop() - continue - + while len(insufficient_name) >= 1: + for file in insufficient_name: try: - video = Video.fromguess(file, guessit(supplementary_info)) - print(video) - moveHome(video) - insufficient_info.pop() - except InsufficientInfoError: - pass - + manual_name = prompt(file) + + if manual_name is None: + insufficient_name.pop() + continue + + try: + video = Video.fromguess(file, guessit(manual_name)) + moveHome(video) + insufficient_name.pop() + + except InsufficientNameError: + continue + + except KeyboardInterrupt: + # Logger: Received interrupt, exiting parser. + # should the class objects be deleted ? + print('Interrupt detected. Exiting') + exit(0) + if __name__ == '__main__': main() From 028301c4c444d31d78be982b88e7905d07812d6a Mon Sep 17 00:00:00 2001 From: KevinMidboe Date: Thu, 18 Oct 2018 22:14:08 +0200 Subject: [PATCH 72/98] =?UTF-8?q?Added=20more=20knowledge=20=F0=9F=A7=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- knowledgeBase.md | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/knowledgeBase.md b/knowledgeBase.md index 0b8347d..e1736e8 100644 --- a/knowledgeBase.md +++ b/knowledgeBase.md @@ -567,4 +567,38 @@ sys 0m0.851s ``` -Keep nfo files? \ No newline at end of file +Keep nfo files? + + +# Program flow + +videos -> scan_folder +scan_folder + videos + ├ scan_video + ├ search_external_subtitles + └ refine + ignored_videos -> None + error_paths -> not exists or directory but error while scan_videos + +scan_folder-videos-scan_video = + raise error -> not exists or not VIDEO_EXT + video -> Video.fromguess + +Video + -> fromguess + * Raise ValueError not episode or movie + +Can I raise an error for everything that is not sufficient to move. Then return the errors with the videos. This catches the same number as video.sufficient, pickforgirlscouts. + + +Tweet argument for correction + +How are the insufficient supposed to be handled? +* Return with videos to main and + + + +Should not create dependencies in the code by have an exception doing something very specific to install and external data by checking if a guess can be resolved from checking earlier matches. I would be more nimble and modular approach to have our errors send back and returned and have a separate excution path which has the database element for checking eariler matches. Now the dependencies are segmented more in two different files, increasing upgradability. +Another note would be to not have rearly and error-prone calls happend deep in a execution path but have it separated so handling the errors for it can be high level and not have others functions error handling take over or missrepersent the original error. + From 0ff205615f98490cf56e483f6414a3302097aec0 Mon Sep 17 00:00:00 2001 From: KevinMidboe Date: Sat, 2 Feb 2019 00:28:56 +0100 Subject: [PATCH 73/98] This is not js is it? --- src/cli.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cli.py b/src/cli.py index 204d679..802d465 100755 --- a/src/cli.py +++ b/src/cli.py @@ -15,9 +15,9 @@ def tweet(video): def prompt(name): manual_name = input("Insufficient name: '{}'\nInput name manually: ".format(name)) - if manual_name is 'q': - assert KeyboardInterrupt - if manual_name is 's': + if manual_name == 'q': + raise KeyboardInterrupt + if manual_name == 's': return None From 2977ab53b2bb50b0186f2a555e09896f36fc34e7 Mon Sep 17 00:00:00 2001 From: KevinMidboe Date: Sat, 2 Feb 2019 00:33:32 +0100 Subject: [PATCH 74/98] Allow a instuffient name to be thrown. Also, pop was not the correct way to remove elements from the list, del on a index is now used. --- src/cli.py | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/src/cli.py b/src/cli.py index 802d465..9ab67be 100755 --- a/src/cli.py +++ b/src/cli.py @@ -6,9 +6,6 @@ from core import scan_folder, moveHome from video import Video from exceptions import InsufficientNameError -def moveHome(video): - print('Would have moved: {}'.format(video)) - def tweet(video): pass @@ -25,7 +22,7 @@ def prompt(name): @click.command() @click.argument('path') -@click.option('--daemon', '-d', daemon) +@click.option('--daemon', '-d') def main(path, daemon): videos, insufficient_name = scan_folder(path) @@ -33,22 +30,18 @@ def main(path, daemon): moveHome(video) while len(insufficient_name) >= 1: - for file in insufficient_name: + for i, file in enumerate(insufficient_name): try: manual_name = prompt(file) if manual_name is None: - insufficient_name.pop() + del insufficient_name[i] continue - try: - video = Video.fromguess(file, guessit(manual_name)) - moveHome(video) - insufficient_name.pop() + video = Video.fromguess(file, guessit(manual_name)) + moveHome(video) + del insufficient_name[i] - except InsufficientNameError: - continue - except KeyboardInterrupt: # Logger: Received interrupt, exiting parser. # should the class objects be deleted ? From e3c1f18e3d28aa2577333adbf35edb9a5f39f80c Mon Sep 17 00:00:00 2001 From: KevinMidboe Date: Sat, 2 Feb 2019 00:34:40 +0100 Subject: [PATCH 75/98] Changed default logging level. --- src/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core.py b/src/core.py index 71eb0f8..16e00a7 100755 --- a/src/core.py +++ b/src/core.py @@ -25,7 +25,7 @@ from video import VIDEO_EXTENSIONS, Episode, Movie, Video from subtitle import SUBTITLE_EXTENSIONS, Subtitle, get_subtitle_path from utils import sanitize, refine -logging.basicConfig(filename=env.logfile, level=logging.DEBUG) +logging.basicConfig(filename=env.logfile, level=logging.INFO) logger = logging.getLogger('seasonedParser') fh = logging.FileHandler(env.logfile) fh.setLevel(logging.INFO) @@ -191,7 +191,7 @@ def scan_videos(path): if filename.endswith(VIDEO_EXTENSIONS): # video try: video = scan_video(filepath) - except InsufficientInfoError as e: + except InsufficientNameError as e: logger.info(e) insufficient_name.append(filepath) continue From 297f35aa6c6adfd291b4685c4c81924a1987e036 Mon Sep 17 00:00:00 2001 From: KevinMidboe Date: Sat, 2 Feb 2019 00:44:53 +0100 Subject: [PATCH 76/98] More knowledge --- knowledgeBase.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/knowledgeBase.md b/knowledgeBase.md index e1736e8..0d20f9a 100644 --- a/knowledgeBase.md +++ b/knowledgeBase.md @@ -602,3 +602,13 @@ How are the insufficient supposed to be handled? Should not create dependencies in the code by have an exception doing something very specific to install and external data by checking if a guess can be resolved from checking earlier matches. I would be more nimble and modular approach to have our errors send back and returned and have a separate excution path which has the database element for checking eariler matches. Now the dependencies are segmented more in two different files, increasing upgradability. Another note would be to not have rearly and error-prone calls happend deep in a execution path but have it separated so handling the errors for it can be high level and not have others functions error handling take over or missrepersent the original error. +| +|\ Could look at the SHOW_PATH and search for nodes with same +| | structure as the guess. A pretty high number or match the +| o partent dirs and then compare the file's guess with itself. + + +* Daemon argument - Throw exception if notificiation agent is not defined. +* should the path be saved in the video object? + +NB! New path is not defined. From 4a93dbfd2ee8766f2d7d50b03830eb3c051539d3 Mon Sep 17 00:00:00 2001 From: KevinMidboe Date: Tue, 19 Mar 2019 23:23:43 +0100 Subject: [PATCH 77/98] Dry and daemon cli parameters added for only displaying, not moving; and daemon to let program know not to ask for user input by cmdline, instead exit. --- src/cli.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/cli.py b/src/cli.py index 9ab67be..b664d57 100755 --- a/src/cli.py +++ b/src/cli.py @@ -2,7 +2,7 @@ import click from guessit import guessit -from core import scan_folder, moveHome +from core import scan_folder from video import Video from exceptions import InsufficientNameError @@ -20,15 +20,29 @@ def prompt(name): return manual_name +def _moveHome(file): + print('- - -\nMatch: \t\t {}. \nDestination:\t {}'.format(file, file.wantedFilePath())) + @click.command() @click.argument('path') -@click.option('--daemon', '-d') -def main(path, daemon): +@click.option('--daemon', '-d', is_flag=True) +@click.option('--dry', is_flag=True) +def main(path, daemon, dry): + if dry: + def moveHome(file): _moveHome(file) + else: + from core import moveHome + + videos, insufficient_name = scan_folder(path) for video in videos: moveHome(video) + if daemon: + print('Exiting! Daemon flag set. Insufficient name for: ', insufficient_name) + exit(0) + while len(insufficient_name) >= 1: for i, file in enumerate(insufficient_name): try: From 92b0dbafca40066e58faa792b19a9892d825df7e Mon Sep 17 00:00:00 2001 From: KevinMidboe Date: Wed, 20 Mar 2019 00:28:07 +0100 Subject: [PATCH 78/98] Imported logger and stricter logic when exiting because of insuffient_name and daemon flag set. --- src/cli.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/cli.py b/src/cli.py index b664d57..fd921cc 100755 --- a/src/cli.py +++ b/src/cli.py @@ -1,11 +1,14 @@ #!usr/bin/env python3.6 import click from guessit import guessit +import logging from core import scan_folder from video import Video from exceptions import InsufficientNameError +logger = logging.getLogger('seasonedParser') + def tweet(video): pass @@ -22,6 +25,7 @@ def prompt(name): def _moveHome(file): print('- - -\nMatch: \t\t {}. \nDestination:\t {}'.format(file, file.wantedFilePath())) + logger.info('- - -\nMatch: \t\t {}. \nDestination:\t {}'.format(file, file.wantedFilePath())) @click.command() @click.argument('path') @@ -39,8 +43,8 @@ def main(path, daemon, dry): for video in videos: moveHome(video) - if daemon: - print('Exiting! Daemon flag set. Insufficient name for: ', insufficient_name) + if len(insufficient_name) and daemon: + logger.warning('Daemon flag set. Insufficient name for: %r', insufficient_name) exit(0) while len(insufficient_name) >= 1: From 8d3a15ad8d3707f56bb7ea05386fd65c3b16879e Mon Sep 17 00:00:00 2001 From: KevinMidboe Date: Wed, 20 Mar 2019 00:28:53 +0100 Subject: [PATCH 79/98] Logging formatter has separate formatting for cmd output and its log level set to warning. --- src/core.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/core.py b/src/core.py index 16e00a7..b4397ba 100755 --- a/src/core.py +++ b/src/core.py @@ -3,7 +3,7 @@ # @Author: KevinMidboe # @Date: 2017-08-25 23:22:27 # @Last Modified by: KevinMidboe -# @Last Modified time: 2017-09-29 12:35:24 +# @Last Modified time: 2019-02-02 01:04:25 from guessit import guessit from babelfish import Language, LanguageReverseError @@ -30,11 +30,12 @@ logger = logging.getLogger('seasonedParser') fh = logging.FileHandler(env.logfile) fh.setLevel(logging.INFO) sh = logging.StreamHandler() -sh.setLevel(logging.ERROR) +sh.setLevel(logging.WARNING) -formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') -fh.setFormatter(formatter) -sh.setFormatter(formatter) +fh_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +sh_formatter = logging.Formatter('%(levelname)s: %(message)s') +fh.setFormatter(fh_formatter) +sh.setFormatter(sh_formatter) logger.addHandler(fh) logger.addHandler(sh) From 04da9152de4f99d57918c61cfa415183a6ec8159 Mon Sep 17 00:00:00 2001 From: KevinMidboe Date: Wed, 20 Mar 2019 00:28:53 +0100 Subject: [PATCH 80/98] Logging formatter has separate formatting for cmd output and its log level set to warning. --- src/core.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/core.py b/src/core.py index 16e00a7..8d68eff 100755 --- a/src/core.py +++ b/src/core.py @@ -3,7 +3,7 @@ # @Author: KevinMidboe # @Date: 2017-08-25 23:22:27 # @Last Modified by: KevinMidboe -# @Last Modified time: 2017-09-29 12:35:24 +# @Last Modified time: 2019-02-02 01:04:25 from guessit import guessit from babelfish import Language, LanguageReverseError @@ -30,11 +30,12 @@ logger = logging.getLogger('seasonedParser') fh = logging.FileHandler(env.logfile) fh.setLevel(logging.INFO) sh = logging.StreamHandler() -sh.setLevel(logging.ERROR) +sh.setLevel(logging.WARNING) -formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') -fh.setFormatter(formatter) -sh.setFormatter(formatter) +fh_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +sh_formatter = logging.Formatter('%(levelname)s: %(message)s') +fh.setFormatter(fh_formatter) +sh.setFormatter(sh_formatter) logger.addHandler(fh) logger.addHandler(sh) @@ -94,7 +95,7 @@ def scan_video(path): logger.info('Scanning video %r in %r', filename, dirpath) # guess - video = Video.fromguess(path, guessit(path)) + video = Video.fromguess(path, guessit(filename)) video.subtitles |= set(search_external_subtitles(video.name)) refine(video) From 3bce8e84b04815b3006d7f75ca3773973bccd148 Mon Sep 17 00:00:00 2001 From: Kevin Date: Mon, 1 Jul 2019 19:59:15 +0200 Subject: [PATCH 81/98] Guessit upgradet to v3 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0a9c345..636925a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -guessit==2.1.4 +guessit==3.0.0 tvdb_api==2.0 hashids==1.2.0 enzyme>=0.4.1 From 0be3beb87bb3568a567d7bd65485c75c392f1452 Mon Sep 17 00:00:00 2001 From: Kevin Date: Mon, 1 Jul 2019 19:59:46 +0200 Subject: [PATCH 82/98] Guessing only returns a single match --- src/video.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/video.py b/src/video.py index 6c52f47..77730e3 100644 --- a/src/video.py +++ b/src/video.py @@ -169,7 +169,7 @@ class Episode(Video): @classmethod def fromname(cls, name): - return cls.fromguess(name, guessit(name, {'type': 'episode'})) + return cls.fromguess(name, guessit(name, {'type': 'episode', 'single_value': True })) def wantedFilePath(self): series = titlecase(self.series) From fd95b0f3ae62d0b04b4d45a017e0387c2a058d90 Mon Sep 17 00:00:00 2001 From: KevinMidboe Date: Tue, 18 Feb 2020 20:29:28 +0100 Subject: [PATCH 83/98] Inital drone config. Checks that it can install requirements with python v 3.6 & 3.8. --- .drone.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .drone.yml diff --git a/.drone.yml b/.drone.yml new file mode 100644 index 0000000..198fd33 --- /dev/null +++ b/.drone.yml @@ -0,0 +1,28 @@ +--- +kind: pipeline +type: docker +name: seasonedParser + +platform: + os: linux + arch: amd64 + +steps: +- name: install-python3.6 + image: python:3.6-alpine + commands: + - pip install -r requirements.txt + +- name: install-python3.8 + image: python:3.8-alpine + commands: + - pip install -r requirements.txt + +trigger: + branch: + - master + event: + include: + - pull_request + - push + From 138a6b5fec8e41c6e6acb5e34fc61ed2b4ee179a Mon Sep 17 00:00:00 2001 From: KevinMidboe Date: Tue, 18 Feb 2020 20:31:27 +0100 Subject: [PATCH 84/98] Installs scripts first prints python version. --- .drone.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.drone.yml b/.drone.yml index 198fd33..fa064ac 100644 --- a/.drone.yml +++ b/.drone.yml @@ -11,11 +11,13 @@ steps: - name: install-python3.6 image: python:3.6-alpine commands: + - python --version - pip install -r requirements.txt - name: install-python3.8 image: python:3.8-alpine commands: + - python --version - pip install -r requirements.txt trigger: From c95568e3b8ba6e59deb0f04d32c4e9c5203d5ae0 Mon Sep 17 00:00:00 2001 From: KevinMidboe Date: Tue, 18 Feb 2020 21:02:50 +0100 Subject: [PATCH 85/98] Import pytest and created some test- test scripts. --- requirements.txt | 1 + test/__init__.py | 0 test/test_import.py | 6 ++++++ test/test_square.py | 9 +++++++++ 4 files changed, 16 insertions(+) create mode 100644 test/__init__.py create mode 100644 test/test_import.py create mode 100644 test/test_square.py diff --git a/requirements.txt b/requirements.txt index 636925a..8e800f0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,4 @@ enzyme>=0.4.1 click>=6.7 langdetect>=1.0.7 titlecase>=0.12.0 +pytest>=5.3.5 diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/test_import.py b/test/test_import.py new file mode 100644 index 0000000..dedea7c --- /dev/null +++ b/test/test_import.py @@ -0,0 +1,6 @@ +import sys, os +sys.path.append(os.path.realpath(os.path.dirname(__file__)+"/../src")) + +def test_import_env_variables(): + import env_variables as env + assert env.logfile == 'conf/output.log' diff --git a/test/test_square.py b/test/test_square.py new file mode 100644 index 0000000..419c80b --- /dev/null +++ b/test/test_square.py @@ -0,0 +1,9 @@ + +def square(x): + return x * x + +def test_square(): + assert square(2) == 4 + +def test_square_negative(): + assert square(-2) == 4 From b8f9ddbbf4bf0a8c02059caa70c7924016c89276 Mon Sep 17 00:00:00 2001 From: KevinMidboe Date: Tue, 18 Feb 2020 21:04:33 +0100 Subject: [PATCH 86/98] Updated drone to run unit tests using py.test. Renamed steps from install-* to test-*. --- .drone.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.drone.yml b/.drone.yml index fa064ac..95efa8f 100644 --- a/.drone.yml +++ b/.drone.yml @@ -8,17 +8,19 @@ platform: arch: amd64 steps: -- name: install-python3.6 +- name: test-python3.6 image: python:3.6-alpine commands: - python --version - pip install -r requirements.txt + - py.test test -- name: install-python3.8 +- name: test-python3.8 image: python:3.8-alpine commands: - python --version - pip install -r requirements.txt + - py.test test trigger: branch: From abe4b5745c0e8d3b1896f7c6390d9f4f16f67ff5 Mon Sep 17 00:00:00 2001 From: KevinMidboe Date: Tue, 18 Feb 2020 21:21:23 +0100 Subject: [PATCH 87/98] Py package: pytest-cov. Upload test coverage to codecov. --- .drone.yml | 28 +++++++++++++++++++--------- requirements.txt | 1 + 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/.drone.yml b/.drone.yml index 95efa8f..01c670c 100644 --- a/.drone.yml +++ b/.drone.yml @@ -11,22 +11,32 @@ steps: - name: test-python3.6 image: python:3.6-alpine commands: - - python --version - - pip install -r requirements.txt - - py.test test + - python --version + - pip install -r requirements.txt + - py.test test - name: test-python3.8 image: python:3.8-alpine commands: - - python --version - - pip install -r requirements.txt - - py.test test + - python --version + - pip install -r requirements.txt + - py.test test +- name: codecov + image: python3:6-alpine + environment: + CODECOV_TOKEN: + from_secret: CODECOV_TOKEN + commands: + - py.test --cov-report=xml --cov=src test + - pip install codecov + - codecov -t + trigger: branch: - - master + - master event: include: - - pull_request - - push + - pull_request + - push diff --git a/requirements.txt b/requirements.txt index 8e800f0..1450541 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,4 @@ click>=6.7 langdetect>=1.0.7 titlecase>=0.12.0 pytest>=5.3.5 +pytest-cov>=2.8.1 From e4b3243a569c0fbf335c641bc2be900dadac7f8e Mon Sep 17 00:00:00 2001 From: KevinMidboe Date: Tue, 18 Feb 2020 21:23:15 +0100 Subject: [PATCH 88/98] Miss spelled docker image for codecov. --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 01c670c..338d6b3 100644 --- a/.drone.yml +++ b/.drone.yml @@ -23,7 +23,7 @@ steps: - py.test test - name: codecov - image: python3:6-alpine + image: python:3.6-alpine environment: CODECOV_TOKEN: from_secret: CODECOV_TOKEN From 9ef175b7154bd9534f285b9ccf52138671b9e5cf Mon Sep 17 00:00:00 2001 From: KevinMidboe Date: Tue, 18 Feb 2020 21:24:48 +0100 Subject: [PATCH 89/98] Codecov task must first install all requirements. --- .drone.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.drone.yml b/.drone.yml index 338d6b3..32aac0b 100644 --- a/.drone.yml +++ b/.drone.yml @@ -28,6 +28,7 @@ steps: CODECOV_TOKEN: from_secret: CODECOV_TOKEN commands: + - pip install -r requirements.txt - py.test --cov-report=xml --cov=src test - pip install codecov - codecov -t From 756dcb51ce54f873e3ab8cbb65fadd76c36d89d0 Mon Sep 17 00:00:00 2001 From: KevinMidboe Date: Tue, 18 Feb 2020 21:26:43 +0100 Subject: [PATCH 90/98] Missing environment variable for codecov. --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 32aac0b..78acd1e 100644 --- a/.drone.yml +++ b/.drone.yml @@ -31,7 +31,7 @@ steps: - pip install -r requirements.txt - py.test --cov-report=xml --cov=src test - pip install codecov - - codecov -t + - codecov -t CODECOV_TOKEN trigger: branch: From 824bf707fb89929166342c0f74443f7d02a884fd Mon Sep 17 00:00:00 2001 From: KevinMidboe Date: Tue, 18 Feb 2020 21:32:47 +0100 Subject: [PATCH 91/98] replaced codecov python uploader with bash uploader. --- .drone.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.drone.yml b/.drone.yml index 78acd1e..27909b7 100644 --- a/.drone.yml +++ b/.drone.yml @@ -30,9 +30,7 @@ steps: commands: - pip install -r requirements.txt - py.test --cov-report=xml --cov=src test - - pip install codecov - - codecov -t CODECOV_TOKEN - + - bash <(curl -s https://codecov.io/bash) trigger: branch: - master From 5aaf1197b7b7c7c82ef706136103d2cd6bd65ac0 Mon Sep 17 00:00:00 2001 From: KevinMidboe Date: Tue, 18 Feb 2020 21:48:59 +0100 Subject: [PATCH 92/98] Make sure bash and curl are installed before using. --- .drone.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 27909b7..9d7b895 100644 --- a/.drone.yml +++ b/.drone.yml @@ -30,7 +30,9 @@ steps: commands: - pip install -r requirements.txt - py.test --cov-report=xml --cov=src test - - bash <(curl -s https://codecov.io/bash) + - apk add curl + - apk add bash + - bash -c "$(curl -s https://codecov.io/bash)" trigger: branch: - master From 307a412c83d17be1c0eaf6db2897511f36a03cc7 Mon Sep 17 00:00:00 2001 From: KevinMidboe Date: Tue, 18 Feb 2020 22:01:06 +0100 Subject: [PATCH 93/98] Require git to be installed. --- .drone.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.drone.yml b/.drone.yml index 9d7b895..e61bedf 100644 --- a/.drone.yml +++ b/.drone.yml @@ -32,6 +32,7 @@ steps: - py.test --cov-report=xml --cov=src test - apk add curl - apk add bash + - apk add git - bash -c "$(curl -s https://codecov.io/bash)" trigger: branch: From 90eff227a050e421fad6d13be9a564e180a252b3 Mon Sep 17 00:00:00 2001 From: KevinMidboe Date: Tue, 18 Feb 2020 22:13:17 +0100 Subject: [PATCH 94/98] Make it a package --- src/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 src/__init__.py diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 From eb13e34e6677e2d098462041bd94722ba28a6c14 Mon Sep 17 00:00:00 2001 From: KevinMidboe Date: Tue, 18 Feb 2020 23:44:16 +0100 Subject: [PATCH 95/98] Restructured project to better run as a package. This should have been setup correctly, but now try to better follow structure for python modules, which this is supposed to be. - Renamed folder from src -> seasonedParser - Moved test/ into seasonedParser/ - test has __init__.py script which sets location to the project folder (seasonedParser/). - Removed cli.py and moved contents to __main__.py - Updated drone to run pytest without test folder parameter --- .drone.yml | 8 +++---- seasonedParser/__init__.py | 7 +++++++ src/cli.py => seasonedParser/__main__.py | 22 +++++++++++++++++++- seasonedParser/__version__.py | 6 ++++++ {src => seasonedParser}/core.py | 18 ++-------------- {src => seasonedParser}/env_variables.py | 0 {src => seasonedParser}/exceptions.py | 0 seasonedParser/logger.py | 17 +++++++++++++++ {src => seasonedParser}/pirateSearch.py | 0 {src => seasonedParser}/scandir.py | 0 {src => seasonedParser}/seasonGuesser.py | 0 {src => seasonedParser}/seasonMover.py | 0 {src => seasonedParser}/subtitle.py | 0 seasonedParser/test/__init__.py | 2 ++ {test => seasonedParser/test}/test_import.py | 1 - {test => seasonedParser/test}/test_square.py | 0 {src => seasonedParser}/tvdb.py | 0 {src => seasonedParser}/utils.py | 0 {src => seasonedParser}/video.py | 0 {src => seasonedParser}/walk.py | 0 {src => seasonedParser}/watcher.py | 0 src/__init__.py | 0 test/__init__.py | 0 23 files changed, 58 insertions(+), 23 deletions(-) create mode 100644 seasonedParser/__init__.py rename src/cli.py => seasonedParser/__main__.py (77%) mode change 100755 => 100644 create mode 100644 seasonedParser/__version__.py rename {src => seasonedParser}/core.py (96%) rename {src => seasonedParser}/env_variables.py (100%) rename {src => seasonedParser}/exceptions.py (100%) create mode 100644 seasonedParser/logger.py rename {src => seasonedParser}/pirateSearch.py (100%) rename {src => seasonedParser}/scandir.py (100%) rename {src => seasonedParser}/seasonGuesser.py (100%) rename {src => seasonedParser}/seasonMover.py (100%) rename {src => seasonedParser}/subtitle.py (100%) create mode 100644 seasonedParser/test/__init__.py rename {test => seasonedParser/test}/test_import.py (63%) rename {test => seasonedParser/test}/test_square.py (100%) rename {src => seasonedParser}/tvdb.py (100%) rename {src => seasonedParser}/utils.py (100%) rename {src => seasonedParser}/video.py (100%) rename {src => seasonedParser}/walk.py (100%) rename {src => seasonedParser}/watcher.py (100%) delete mode 100644 src/__init__.py delete mode 100644 test/__init__.py diff --git a/.drone.yml b/.drone.yml index e61bedf..124efc7 100644 --- a/.drone.yml +++ b/.drone.yml @@ -13,14 +13,14 @@ steps: commands: - python --version - pip install -r requirements.txt - - py.test test + - py.test - name: test-python3.8 image: python:3.8-alpine commands: - python --version - pip install -r requirements.txt - - py.test test + - py.test - name: codecov image: python:3.6-alpine @@ -29,9 +29,7 @@ steps: from_secret: CODECOV_TOKEN commands: - pip install -r requirements.txt - - py.test --cov-report=xml --cov=src test - - apk add curl - - apk add bash + - py.test --cov-report=xml --cov=seasonedParserseasonedParserseasonedParserseasonedParserseasonedParserseasonedParserseasonedParserseasonedParser - apk add git - bash -c "$(curl -s https://codecov.io/bash)" trigger: diff --git a/seasonedParser/__init__.py b/seasonedParser/__init__.py new file mode 100644 index 0000000..a08b226 --- /dev/null +++ b/seasonedParser/__init__.py @@ -0,0 +1,7 @@ +print('hello from init') +import sys,os +# sys.path.append(os.path.join(os.path.dirname(__file__),os.pardir)) + +# from + +from .__version__ import __version__ diff --git a/src/cli.py b/seasonedParser/__main__.py old mode 100755 new mode 100644 similarity index 77% rename from src/cli.py rename to seasonedParser/__main__.py index fd921cc..c29c2c4 --- a/src/cli.py +++ b/seasonedParser/__main__.py @@ -1,4 +1,9 @@ -#!usr/bin/env python3.6 +#!/usr/bin/env python3.6 +# -*- coding: utf-8 -*- +""" +Entry point module +""" + import click from guessit import guessit import logging @@ -7,7 +12,22 @@ from core import scan_folder from video import Video from exceptions import InsufficientNameError +import env_variables as env + +logging.basicConfig(filename=env.logfile, level=logging.INFO) logger = logging.getLogger('seasonedParser') +fh = logging.FileHandler(env.logfile) +fh.setLevel(logging.INFO) +sh = logging.StreamHandler() +sh.setLevel(logging.WARNING) + +fh_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +sh_formatter = logging.Formatter('%(levelname)s: %(message)s') +fh.setFormatter(fh_formatter) +sh.setFormatter(sh_formatter) + +logger.addHandler(fh) +logger.addHandler(sh) def tweet(video): pass diff --git a/seasonedParser/__version__.py b/seasonedParser/__version__.py new file mode 100644 index 0000000..d19b4e5 --- /dev/null +++ b/seasonedParser/__version__.py @@ -0,0 +1,6 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +Version module +""" +__version__ = '0.2.0' diff --git a/src/core.py b/seasonedParser/core.py similarity index 96% rename from src/core.py rename to seasonedParser/core.py index 8d68eff..a18a4b2 100755 --- a/src/core.py +++ b/seasonedParser/core.py @@ -10,7 +10,6 @@ from babelfish import Language, LanguageReverseError import hashlib import os, errno import shutil -import logging import re import tvdb_api import click @@ -20,26 +19,13 @@ import langdetect import env_variables as env from exceptions import InsufficientNameError +import logging +logger = logging.getLogger('seasonedParser') from video import VIDEO_EXTENSIONS, Episode, Movie, Video from subtitle import SUBTITLE_EXTENSIONS, Subtitle, get_subtitle_path from utils import sanitize, refine -logging.basicConfig(filename=env.logfile, level=logging.INFO) -logger = logging.getLogger('seasonedParser') -fh = logging.FileHandler(env.logfile) -fh.setLevel(logging.INFO) -sh = logging.StreamHandler() -sh.setLevel(logging.WARNING) - -fh_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') -sh_formatter = logging.Formatter('%(levelname)s: %(message)s') -fh.setFormatter(fh_formatter) -sh.setFormatter(sh_formatter) - -logger.addHandler(fh) -logger.addHandler(sh) - def search_external_subtitles(path, directory=None): dirpath, filename = os.path.split(path) dirpath = dirpath or '.' diff --git a/src/env_variables.py b/seasonedParser/env_variables.py similarity index 100% rename from src/env_variables.py rename to seasonedParser/env_variables.py diff --git a/src/exceptions.py b/seasonedParser/exceptions.py similarity index 100% rename from src/exceptions.py rename to seasonedParser/exceptions.py diff --git a/seasonedParser/logger.py b/seasonedParser/logger.py new file mode 100644 index 0000000..aaa42d3 --- /dev/null +++ b/seasonedParser/logger.py @@ -0,0 +1,17 @@ +import logging +import env_variables as env + +logging.basicConfig(filename=env.logfile, level=logging.INFO) +logger = logging.getLogger('seasonedParser') +fh = logging.FileHandler(env.logfile) +fh.setLevel(logging.INFO) +sh = logging.StreamHandler() +sh.setLevel(logging.WARNING) + +fh_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +sh_formatter = logging.Formatter('%(levelname)s: %(message)s') +fh.setFormatter(fh_formatter) +sh.setFormatter(sh_formatter) + +logger.addHandler(fh) +logger.addHandler(sh) \ No newline at end of file diff --git a/src/pirateSearch.py b/seasonedParser/pirateSearch.py similarity index 100% rename from src/pirateSearch.py rename to seasonedParser/pirateSearch.py diff --git a/src/scandir.py b/seasonedParser/scandir.py similarity index 100% rename from src/scandir.py rename to seasonedParser/scandir.py diff --git a/src/seasonGuesser.py b/seasonedParser/seasonGuesser.py similarity index 100% rename from src/seasonGuesser.py rename to seasonedParser/seasonGuesser.py diff --git a/src/seasonMover.py b/seasonedParser/seasonMover.py similarity index 100% rename from src/seasonMover.py rename to seasonedParser/seasonMover.py diff --git a/src/subtitle.py b/seasonedParser/subtitle.py similarity index 100% rename from src/subtitle.py rename to seasonedParser/subtitle.py diff --git a/seasonedParser/test/__init__.py b/seasonedParser/test/__init__.py new file mode 100644 index 0000000..736a215 --- /dev/null +++ b/seasonedParser/test/__init__.py @@ -0,0 +1,2 @@ +import sys, os +sys.path.append(os.path.realpath(os.path.dirname(__file__)+"/..")) diff --git a/test/test_import.py b/seasonedParser/test/test_import.py similarity index 63% rename from test/test_import.py rename to seasonedParser/test/test_import.py index dedea7c..476b95a 100644 --- a/test/test_import.py +++ b/seasonedParser/test/test_import.py @@ -1,5 +1,4 @@ import sys, os -sys.path.append(os.path.realpath(os.path.dirname(__file__)+"/../src")) def test_import_env_variables(): import env_variables as env diff --git a/test/test_square.py b/seasonedParser/test/test_square.py similarity index 100% rename from test/test_square.py rename to seasonedParser/test/test_square.py diff --git a/src/tvdb.py b/seasonedParser/tvdb.py similarity index 100% rename from src/tvdb.py rename to seasonedParser/tvdb.py diff --git a/src/utils.py b/seasonedParser/utils.py similarity index 100% rename from src/utils.py rename to seasonedParser/utils.py diff --git a/src/video.py b/seasonedParser/video.py similarity index 100% rename from src/video.py rename to seasonedParser/video.py diff --git a/src/walk.py b/seasonedParser/walk.py similarity index 100% rename from src/walk.py rename to seasonedParser/walk.py diff --git a/src/watcher.py b/seasonedParser/watcher.py similarity index 100% rename from src/watcher.py rename to seasonedParser/watcher.py diff --git a/src/__init__.py b/src/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/test/__init__.py b/test/__init__.py deleted file mode 100644 index e69de29..0000000 From 0d376df8f89b293d625294004d19740c933190a9 Mon Sep 17 00:00:00 2001 From: KevinMidboe Date: Wed, 19 Feb 2020 00:26:08 +0100 Subject: [PATCH 96/98] bash and curl was missing for some reason. --- .drone.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.drone.yml b/.drone.yml index 124efc7..1e24220 100644 --- a/.drone.yml +++ b/.drone.yml @@ -31,6 +31,8 @@ steps: - pip install -r requirements.txt - py.test --cov-report=xml --cov=seasonedParserseasonedParserseasonedParserseasonedParserseasonedParserseasonedParserseasonedParserseasonedParser - apk add git + - apk add bash + - apk add curl - bash -c "$(curl -s https://codecov.io/bash)" trigger: branch: From 80931be151efa0bcfbeb6d5f1955a66e6713512d Mon Sep 17 00:00:00 2001 From: KevinMidboe Date: Wed, 19 Feb 2020 00:29:08 +0100 Subject: [PATCH 97/98] Weird cov path fixed. --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 1e24220..e61c330 100644 --- a/.drone.yml +++ b/.drone.yml @@ -29,7 +29,7 @@ steps: from_secret: CODECOV_TOKEN commands: - pip install -r requirements.txt - - py.test --cov-report=xml --cov=seasonedParserseasonedParserseasonedParserseasonedParserseasonedParserseasonedParserseasonedParserseasonedParser + - py.test --cov-report=xml --cov=seasonedParser - apk add git - apk add bash - apk add curl From 25b2dd380479af96ba3e294a43f504f6f6020c14 Mon Sep 17 00:00:00 2001 From: KevinMidboe Date: Wed, 19 Feb 2020 00:32:56 +0100 Subject: [PATCH 98/98] Build and codecov badges to readme. --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 1244e8c..134417b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,9 @@ # seaonedParser seasoned Parser is python based parser that indexes a given directory for media files and identifies if it is a movie or show file and renames + moves it to a the correct place in library. +[![codecov](https://codecov.io/gh/KevinMidboe/seasonedParser/branch/master/graph/badge.svg)](https://codecov.io/gh/KevinMidboe/seasonedParser) +[![Build Status](https://drone.kevinmidboe.com/api/badges/KevinMidboe/seasonedParser/status.svg)](https://drone.kevinmidboe.com/KevinMidboe/seasonedParser) + ## Table of Conents - [Config](#config) - [Setup for automation](#setup-for-automation)