From 9c82ece6665a039080dc262822cf1c83c9de28fa Mon Sep 17 00:00:00 2001 From: KevinMidboe Date: Fri, 11 May 2018 16:57:46 +0200 Subject: [PATCH 01/60] Updated readme and knowledgebase. --- README.md | 12 +++++++++++- knowledgeBase.md | 5 ++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e7cc04f..79dd1da 100644 --- a/README.md +++ b/README.md @@ -16,4 +16,14 @@ There are some settings that need to be set for seasonedParser to be able to fin ### Download directory In your download client set a incomplete folder and a complete directory. This will allow seasonedParser to only parse items that have been completely downloaded. -*TODO:* Monitor multiple folders at the same time. \ No newline at end of file +*TODO:* Monitor multiple folders at the same time. + +## Run +There are many run commands for this, but here is a list of the current working run commands for this project. + +```bash + user@host:$ ~/seasonedParser/./seasonedMover.py move 'The.Big.Bang.Theory.S11E(7..14).720p.x264.mkv' '/mnt/mainframe/shows/The Big Bang Theory/The Big Bang Theory S11E' +``` + +Here the first parameter is our move command, which in turn calls motherMover. The second parameter is what we want the filenames to be called. Notice the (num1..num2), this is to create a range for all the episodes we want to move. The last parameter is the path we want to move our content. + > This will be done automatically by the parser based on the info in the media items name, but it is nice to have a manual command. \ No newline at end of file diff --git a/knowledgeBase.md b/knowledgeBase.md index 0f95342..0b8347d 100644 --- a/knowledgeBase.md +++ b/knowledgeBase.md @@ -564,4 +564,7 @@ Total: 5926, missed was: 29 real 2m0.766s user 1m41.482s sys 0m0.851s -``` \ No newline at end of file +``` + + +Keep nfo files? \ No newline at end of file From bfd8a2a1f5acd0faa34484f9cdc5aee1d4efc6de Mon Sep 17 00:00:00 2001 From: KevinMidboe Date: Fri, 14 Sep 2018 18:51:40 +0200 Subject: [PATCH 02/60] Snapshot before refactor. --- seasonedParser/core.py | 51 ++++++---- seasonedParser/pirateSearch.py | 48 ++------- seasonedParser/scandir.py | 175 +++++++++++++++++++++++++++++++++ seasonedParser/video.py | 4 +- seasonedParser/walk.py | 25 +++++ 5 files changed, 241 insertions(+), 62 deletions(-) create mode 100755 seasonedParser/scandir.py create mode 100755 seasonedParser/walk.py diff --git a/seasonedParser/core.py b/seasonedParser/core.py index dab6ba6..bd184da 100755 --- a/seasonedParser/core.py +++ b/seasonedParser/core.py @@ -3,10 +3,10 @@ # @Author: KevinMidboe # @Date: 2017-08-25 23:22:27 # @Last Modified by: KevinMidboe -# @Last Modified time: 2017-09-29 12:35:24 +# @Last Modified time: 2018-05-13 20:54:17 from guessit import guessit -import os, errno +import os, errno,sys import logging import tvdb_api from pprint import pprint @@ -19,6 +19,7 @@ from utils import sanitize logging.basicConfig(filename=env.logfile, level=logging.INFO) +from datetime import datetime #: Supported archive extensions ARCHIVE_EXTENSIONS = ('.rar',) @@ -45,7 +46,7 @@ def scan_video(path): # guess parent_path = path.strip(filename) video = Video.fromguess(filename, parent_path, guessit(path)) - # video = Video('test') + # video = Video(filename) # guessit(path) return video @@ -86,6 +87,8 @@ def scan_files(path, age=None, archives=True): if not os.path.isdir(path): raise ValueError('Path is not a directory') + name_dict = {} + # walk the path mediafiles = [] for dirpath, dirnames, filenames in os.walk(path): @@ -99,11 +102,9 @@ def scan_files(path, age=None, archives=True): # scan for videos for filename in filenames: - # filter on videos and archives if not (filename.endswith(VIDEO_EXTENSIONS) or filename.endswith(SUBTITLE_EXTENSIONS) or archives and filename.endswith(ARCHIVE_EXTENSIONS)): continue - # skip hidden files if filename.startswith('.'): logging.debug('Skipping hidden filename %r in %r', filename, dirpath) continue @@ -116,16 +117,19 @@ def scan_files(path, age=None, archives=True): logging.debug('Skipping link %r in %r', filename, dirpath) continue - # skip old files - if age and datetime.utcnow() - datetime.utcfromtimestamp(os.path.getmtime(filepath)) > age: - logging.debug('Skipping old file %r in %r', filename, dirpath) - continue - # scan if filename.endswith(VIDEO_EXTENSIONS): # video try: video = scan_video(filepath) + # try: + # name_dict[video.series] += 1 + # except KeyError: + # name_dict[video.series] = 0 + # except: + # print('video did not have attrib series') + # pass mediafiles.append(video) + except ValueError: # pragma: no cover logging.exception('Error scanning video') continue @@ -138,24 +142,26 @@ def scan_files(path, age=None, archives=True): # except (NotRarFile, RarCannotExec, ValueError): # pragma: no cover # logging.exception('Error scanning archive') # continue - elif filename.endswith(SUBTITLE_EXTENSIONS): # subtitle - try: - subtitle = scan_subtitle(filepath) - mediafiles.append(subtitle) - except ValueError: - logging.exception('Error scanning subtitle') - continue + # elif filename.endswith(SUBTITLE_EXTENSIONS): # subtitle + # try: + # subtitle = scan_subtitle(filepath) + # mediafiles.append(subtitle) + # except ValueError: + # logging.exception('Error scanning subtitle') + # continue else: # pragma: no cover - raise ValueError('Unsupported file %r' % filename) + print('Skipping unsupported file {}'.format(filename)) + # raise ValueError('Unsupported file %r' % filename) + pprint(name_dict) return mediafiles def organize_files(path): hashList = {} mediafiles = scan_files(path) - # print(mediafiles) + print(mediafiles) for file in mediafiles: hashList.setdefault(file.__hash__(),[]).append(file) @@ -251,10 +257,15 @@ def save_subtitles(files, single=False, directory=None, encoding=None): # return saved_subtitles +def stringTime(): + return str(datetime.now().strftime("%Y-%m-%d %H:%M:%S:%f")) + def main(): # episodePath = '/Volumes/media/tv/Black Mirror/Black Mirror Season 01/' - episodePath = '/media/hdd1/tv/' + episodePath = '/Volumes/mainframe/shows/Black Mirror/Black Mirror Season 01/' + episodePath = '/Volumes/mainframe/shows/The.Voice.S14E24.720p.WEB.x264-TBS[rarbg]' + episodePath = '/Volumes/mainframe/incomplete' t = tvdb_api.Tvdb() diff --git a/seasonedParser/pirateSearch.py b/seasonedParser/pirateSearch.py index 02b2b18..397aca4 100755 --- a/seasonedParser/pirateSearch.py +++ b/seasonedParser/pirateSearch.py @@ -3,7 +3,7 @@ # @Author: KevinMidboe # @Date: 2017-10-12 11:55:03 # @Last Modified by: KevinMidboe -# @Last Modified time: 2017-10-17 00:58:24 +# @Last Modified time: 2017-11-01 16:11:30 import sys, logging, re from urllib import parse, request @@ -28,39 +28,6 @@ RELEASE_TYPES = ('bdremux', 'brremux', 'remux', 'camrip', 'cam') -def sanitize(string, ignore_characters=None, replace_characters=None): - """Sanitize a string to strip special characters. - - :param str string: the string to sanitize. - :param set ignore_characters: characters to ignore. - :return: the sanitized string. - :rtype: str - - """ - # only deal with strings - if string is None: - return - - replace_characters = replace_characters or '' - - ignore_characters = ignore_characters or set() - - characters = ignore_characters - if characters: - string = re.sub(r'[%s]' % re.escape(''.join(characters)), replace_characters, string) - - return string - -def return_re_match(string, re_statement): - if string is None: - return - - m = re.search(re_statement, string) - if 'Y-day' in m.group(): - return datetime.datetime.now().strftime('%m-%d %Y') - return sanitize(m.group(), '\xa0', ' ') - - # Should maybe not be able to set values without checking if they are valid? class piratebay(object): def __init__(self, query=None, page=0, sort=None, category=None): @@ -157,7 +124,7 @@ class piratebay(object): print(self.page) # Fetch in parallel - n = self.total_pages + n = pagesToCount(multiple_pages, self.total_pages) while n > 1: torrents_found.extend(self.next_page()) n -= 1 @@ -276,7 +243,7 @@ def chooseCandidate(torrent_list): size, _, size_id = torrent.size.partition(' ') if intersecting_release_types and int(torrent.seed_count) > 0 and float(size) > 4 and size_id == 'GiB': - print('{} : {} : {}'.format(torrent.name, torrent.size, torrent.seed_count)) + print('{} : {} : {} {}'.format(torrent.name, torrent.size, torrent.seed_count, torrent.magnet)) interesting_torrents.append(torrent) # else: # print('Denied match! %s : %s : %s' % (torrent.name, torrent.size, torrent.seed_count)) @@ -286,10 +253,11 @@ def chooseCandidate(torrent_list): def searchTorrentSite(query, site='piratebay'): pirate = piratebay() - torrents_found = pirate.search(query, page=0, multiple_pages=0, sort='size') - # pprint(torrents_found) + torrents_found = pirate.search(query, page=0, multiple_pages=5, sort='size') + pprint(torrents_found) candidates = chooseCandidate(torrents_found) - + pprint(candidates) + exit(0) torrents_found = pirate.search(query, page=0, multiple_pages=0, sort='size', category='movies') movie_candidates = chooseCandidate(torrents_found) @@ -308,4 +276,4 @@ def main(): searchTorrentSite(query) if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/seasonedParser/scandir.py b/seasonedParser/scandir.py new file mode 100755 index 0000000..9c64eb7 --- /dev/null +++ b/seasonedParser/scandir.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python3.6 +# -*- coding: utf-8 -*- +# @Author: KevinMidboe +# @Date: 2017-10-02 16:29:25 +# @Last Modified by: KevinMidboe +# @Last Modified time: 2018-01-15 17:18:36 + +try: + from os import scandir +except ImportError: + from scandir import scandir # use scandir PyPI module on Python < 3.5 + +import env_variables as env +import multiprocessing as mp +import logging, re, datetime +from guessit import guessit + +from video import VIDEO_EXTENSIONS, Episode, Movie, Video +from subtitle import SUBTITLE_EXTENSIONS, Subtitle, get_subtitle_path + +logging.basicConfig(filename=env.logfile, level=logging.INFO) + +""" Move to utils file """ +def removeLeadingZero(number): + stringedNumber = str(number) + if (len(stringedNumber) > 1 and stringedNumber[0] == '0'): + return int(stringedNumber[1:]) + return int(number) + +class movie(object): + def __init__(self, path, title=None, year=None): + self.path = path + self.title = title + self.year = year + +class Episode(object): + def __init__(self, path, name, title=None, season=None, episode=None): + super(Episode, self).__init__() + self.path = path + self.name = name + self.title = title + self.season = season + self.episode = episode + + @classmethod + def fromname(cls, path, name): + title = cls.findTitle(cls, name) + season = cls.findSeasonNumber(cls, name) + episode = cls.findEpisodeNumber(cls, name) + + return cls(path, name, title, season, episode) + + def findTitle(self, name): + m = re.search("([a-zA-Z0-9\'\.\-\ ])+([sS][0-9]{1,3})", name) + if m: + return re.sub('[\ \.]*[sS][0-9]{1,2}', '', m.group(0)) + + def findSeasonNumber(self, name): + m = re.search('[sS][0-9]{1,2}', name) + if m: + seasonNumber = re.sub('[sS]', '', m.group(0)) + return removeLeadingZero(seasonNumber) + + def findEpisodeNumber(self, name): + m = re.search('[eE][0-9]{1,3}', name) + if m: + episodeNumber = re.sub('[eE]', '', m.group(0)) + return removeLeadingZero(episodeNumber) + +def get_tree_size(path): + """Return total size of files in given path and subdirs.""" + total = 0 + for entry in scandir(path): + if not ('.DS_Store' in entry.path or 'lost+found' in entry.path): + if entry.is_dir(follow_symlinks=False): + total += get_tree_size(entry.path) + else: + total += entry.stat(follow_symlinks=False).st_size + return int(total) + +def scantree(path): + """Recursively yield DirEntry objects for given directory.""" + for entry in scandir(path): + # Skip .DS_Store and lost+found + # TODO have a blacklist here + if not ('.DS_Store' in entry.path or 'lost+found' in entry.path): + if entry.is_dir(follow_symlinks=False): + yield from scantree(entry.path) + else: + yield entry + +# Find all the mediaobjects for a given path +# TODO handle list of path's +def get_objects_for_path(path, archives=None, match=False): + # Declare list to save the media objects found in the given path + hashList = {} + mediaFiles = [] + # All entries given from scantree functoin + for entry in scantree(path): + logging.debug('Looking at file %s', str(entry.name)) + name = entry.name # Pull out name for faster index + + # Skip if not corrent media extension + if not (name.endswith(VIDEO_EXTENSIONS) or name.endswith(SUBTITLE_EXTENSIONS) or archives and name.endswith(ARCHIVE_EXTENSIONS)): + continue + + # Skip if the file is a dotfile + if name.startswith('.'): + logging.debug('Skipping hidden file %s' % str(name)) + continue + + # If we have a video, create a class and append to mediaFiles + if name.endswith(VIDEO_EXTENSIONS): # video + episode = Episode.fromname(entry.path, entry.name) + if (episode.title is None): + logging.debug('None found for %s' % name) + continue + + title = re.sub('[\.]', ' ', episode.title) + mediaFiles.append(episode) + + return mediaFiles + +if __name__ == '__main__': + logging.info('Started: %s' % str(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S:%f"))) + import sys + from pprint import pprint + total = 0 + missed = 0 + + # print(get_tree_size(sys.argv[1] if len(sys.argv) > 1 else '.')) + # print(str(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S:%f"))) + path = sys.argv[1] if len(sys.argv) > 1 else '.' + mediaFiles = get_objects_for_path(path) + getTitle = lambda ep: ep.title + + for ep in mediaFiles: + print(getTitle(ep)) + + + mediaList = [] + for entry in scantree(sys.argv[1] if len(sys.argv) > 1 else '.'): + name = entry.name + manual = Episode.fromname(entry.path, entry.name) + size = int(entry.stat(follow_symlinks=False).st_size) / 1024 / 1024 / 1024 + # print(name + ' : ' + str(round(size, 2)) + 'GB') + + title = manual.title + if title is None: + logging.debug('None found for %s' % (name)) + continue + + title = re.sub('[\.]', ' ', manual.title) + + # try: + # print(name + ' : ' + "%s S%iE%i" % (str(title), manual.season, manual.episode)) + # except TypeError: + # logging.error('Unexpected error: ' + name) + + mediaList.append(manual) + if ('-m' in sys.argv): + guess = guessit(name) + + logging.info('Manual is: {} and guess is {}'.format(title, guess['title'])) + # # if not (guess['season'] == manual.season and guess['episode'] == manual.episode): + if (guess['title'].lower() != title.lower()): + logging.info('Missmatch: %s by manual guess: %s : %s' % (name, guess['title'], title)) + missed += 1 + + total += 1 + + + print('Total: %i, missed was: %i' % (total, missed)) + logging.info('Ended: %s' % str(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S:%f"))) + logging.info(' - - - - - - - - - ') diff --git a/seasonedParser/video.py b/seasonedParser/video.py index 68be105..f724598 100644 --- a/seasonedParser/video.py +++ b/seasonedParser/video.py @@ -3,7 +3,7 @@ # @Author: KevinMidboe # @Date: 2017-08-26 08:23:18 # @Last Modified by: KevinMidboe -# @Last Modified time: 2017-09-29 13:56:21 +# @Last Modified time: 2018-05-13 20:50:00 from guessit import guessit import os @@ -12,7 +12,7 @@ import hashlib, tvdb_api #: Video extensions VIDEO_EXTENSIONS = ('.3g2', '.3gp', '.3gp2', '.3gpp', '.60d', '.ajp', '.asf', '.asx', '.avchd', '.avi', '.bik', '.bix', '.box', '.cam', '.dat', '.divx', '.dmf', '.dv', '.dvr-ms', '.evo', '.flc', '.fli', - '.flic', '.flv', '.flx', '.gvi', '.gvp', '.h264', '.m1v', '.m2p', '.m2ts', '.m2v', '.m4e', + '.flic', '.flv', '.flx', '.gvi', '.gvp', '.h264', '.m1v', '.m2p', '.m2v', '.m4e', '.m4v', '.mjp', '.mjpeg', '.mjpg', '.mkv', '.moov', '.mov', '.movhd', '.movie', '.movx', '.mp4', '.mpe', '.mpeg', '.mpg', '.mpv', '.mpv2', '.mxf', '.nsv', '.nut', '.ogg', '.ogm' '.ogv', '.omf', '.ps', '.qt', '.ram', '.rm', '.rmvb', '.swf', '.ts', '.vfw', '.vid', '.video', '.viv', '.vivo', diff --git a/seasonedParser/walk.py b/seasonedParser/walk.py new file mode 100755 index 0000000..b922802 --- /dev/null +++ b/seasonedParser/walk.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3.6 +# -*- coding: utf-8 -*- +# @Author: KevinMidboe +# @Date: 2017-10-02 16:29:25 +# @Last Modified by: KevinMidboe +# @Last Modified time: 2017-10-02 18:07:26 + +import itertools, os +import multiprocessing + +def worker(filename): + print(filename) + +def main(): + with multiprocessing.Pool(48) as Pool: # pool of 48 processes + + walk = os.walk("/Volumes/mainframe/shows/") + fn_gen = itertools.chain.from_iterable((os.path.join(root, file) + for file in files) + for root, dirs, files in walk) + + results_of_work = Pool.map(worker, fn_gen) # this does the parallel processing + +if __name__ == '__main__': + main() \ No newline at end of file From 7ea82d4b33cffbceb265080be2a4fa4c85a34df2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Fri, 14 Sep 2018 19:18:59 +0200 Subject: [PATCH 03/60] Renamed seasonedParser subfolder to src --- {seasonedParser => src}/core.py | 0 {seasonedParser => src}/env_variables.py | 0 {seasonedParser => src}/pirateSearch.py | 0 {seasonedParser => src}/scandir.py | 0 {seasonedParser => src}/seasonGuesser.py | 0 {seasonedParser => src}/seasonMover.py | 0 {seasonedParser => src}/subtitle.py | 0 {seasonedParser => src}/tvdb.py | 0 {seasonedParser => src}/utils.py | 0 {seasonedParser => src}/video.py | 0 {seasonedParser => src}/walk.py | 0 {seasonedParser => src}/watcher.py | 0 12 files changed, 0 insertions(+), 0 deletions(-) rename {seasonedParser => src}/core.py (100%) rename {seasonedParser => src}/env_variables.py (100%) rename {seasonedParser => src}/pirateSearch.py (100%) rename {seasonedParser => src}/scandir.py (100%) rename {seasonedParser => src}/seasonGuesser.py (100%) rename {seasonedParser => src}/seasonMover.py (100%) rename {seasonedParser => src}/subtitle.py (100%) rename {seasonedParser => src}/tvdb.py (100%) rename {seasonedParser => src}/utils.py (100%) rename {seasonedParser => src}/video.py (100%) rename {seasonedParser => src}/walk.py (100%) rename {seasonedParser => src}/watcher.py (100%) diff --git a/seasonedParser/core.py b/src/core.py similarity index 100% rename from seasonedParser/core.py rename to src/core.py diff --git a/seasonedParser/env_variables.py b/src/env_variables.py similarity index 100% rename from seasonedParser/env_variables.py rename to src/env_variables.py diff --git a/seasonedParser/pirateSearch.py b/src/pirateSearch.py similarity index 100% rename from seasonedParser/pirateSearch.py rename to src/pirateSearch.py diff --git a/seasonedParser/scandir.py b/src/scandir.py similarity index 100% rename from seasonedParser/scandir.py rename to src/scandir.py diff --git a/seasonedParser/seasonGuesser.py b/src/seasonGuesser.py similarity index 100% rename from seasonedParser/seasonGuesser.py rename to src/seasonGuesser.py diff --git a/seasonedParser/seasonMover.py b/src/seasonMover.py similarity index 100% rename from seasonedParser/seasonMover.py rename to src/seasonMover.py diff --git a/seasonedParser/subtitle.py b/src/subtitle.py similarity index 100% rename from seasonedParser/subtitle.py rename to src/subtitle.py diff --git a/seasonedParser/tvdb.py b/src/tvdb.py similarity index 100% rename from seasonedParser/tvdb.py rename to src/tvdb.py diff --git a/seasonedParser/utils.py b/src/utils.py similarity index 100% rename from seasonedParser/utils.py rename to src/utils.py diff --git a/seasonedParser/video.py b/src/video.py similarity index 100% rename from seasonedParser/video.py rename to src/video.py diff --git a/seasonedParser/walk.py b/src/walk.py similarity index 100% rename from seasonedParser/walk.py rename to src/walk.py diff --git a/seasonedParser/watcher.py b/src/watcher.py similarity index 100% rename from seasonedParser/watcher.py rename to src/watcher.py From 1c81a0165818b0a5722607efbf270902c14c94ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Fri, 14 Sep 2018 19:25:17 +0200 Subject: [PATCH 04/60] Scans for files and subtitles in same folder as video files. --- src/core.py | 100 +++++++++++++++++++++++----------------------------- 1 file changed, 44 insertions(+), 56 deletions(-) diff --git a/src/core.py b/src/core.py index bd184da..d1a7071 100755 --- a/src/core.py +++ b/src/core.py @@ -3,10 +3,10 @@ # @Author: KevinMidboe # @Date: 2017-08-25 23:22:27 # @Last Modified by: KevinMidboe -# @Last Modified time: 2018-05-13 20:54:17 +# @Last Modified time: 2017-09-29 12:35:24 from guessit import guessit -import os, errno,sys +import os, errno import logging import tvdb_api from pprint import pprint @@ -19,11 +19,33 @@ from utils import sanitize logging.basicConfig(filename=env.logfile, level=logging.INFO) -from datetime import datetime #: Supported archive extensions ARCHIVE_EXTENSIONS = ('.rar',) +def search_external_subtitles(path, directory=None): + dirpath, filename = os.path.split(path) + dirpath = dirpath or '.' + fileroot, fileext = os.path.splitext(filename) + + subtitles = {} + for p in os.listdir(directory or dirpath): + if not p.endswith(SUBTITLE_EXTENSIONS): + continue + + language = Language('und') + language_code = p[len(fileroot):-len(os.path.splitext(p)[1])].replace(fileext, '').replace('_','-')[1:] + if language_code: + try: + language = Language.fromietf(language_code) + except (ValueError, LanguageReverseError): + logger.error('Cannot parse language code %r', language_code) + + subtitles[p] = language + logger.debug('Found subtitles %r', subtitles) + + return subtitles + def scan_video(path): """Scan a video from a `path`. @@ -36,18 +58,15 @@ def scan_video(path): if not os.path.exists(path): raise ValueError('Path does not exist') - # check video extension - # if not path.endswith(VIDEO_EXTENSIONS): - # raise ValueError('%r is not a valid video extension' % os.path.splitext(path)[1]) + check video extension + if not path.endswith(VIDEO_EXTENSIONS): + raise ValueError('%r is not a valid video extension' % os.path.splitext(path)[1]) dirpath, filename = os.path.split(path) logging.info('Scanning video %r in %r', filename, dirpath) # guess - parent_path = path.strip(filename) - video = Video.fromguess(filename, parent_path, guessit(path)) - # video = Video(filename) - # guessit(path) + video = Video.fromguess(path, guessit(path)) return video @@ -67,14 +86,12 @@ def scan_subtitle(path): return subtitle -def scan_files(path, age=None, archives=True): +def scan_videos(path): """Scan `path` for videos and their subtitles. See :func:`refine` to find additional information for the video. :param str path: existing directory path to scan. - :param datetime.timedelta age: maximum age of the video or archive. - :param bool archives: scan videos in archives. :return: the scanned videos. :rtype: list of :class:`~subliminal.video.Video` @@ -87,10 +104,8 @@ def scan_files(path, age=None, archives=True): if not os.path.isdir(path): raise ValueError('Path is not a directory') - name_dict = {} - # walk the path - mediafiles = [] + videos = [] for dirpath, dirnames, filenames in os.walk(path): logging.debug('Walking directory %r', dirpath) @@ -102,9 +117,11 @@ def scan_files(path, age=None, archives=True): # scan for videos for filename in filenames: - if not (filename.endswith(VIDEO_EXTENSIONS) or filename.endswith(SUBTITLE_EXTENSIONS) or archives and filename.endswith(ARCHIVE_EXTENSIONS)): + # filter on videos and archives + if not (filename.endswith(VIDEO_EXTENSIONS) or archives and filename.endswith(ARCHIVE_EXTENSIONS)): continue + # skip hidden files if filename.startswith('.'): logging.debug('Skipping hidden filename %r in %r', filename, dirpath) continue @@ -121,47 +138,21 @@ def scan_files(path, age=None, archives=True): if filename.endswith(VIDEO_EXTENSIONS): # video try: video = scan_video(filepath) - # try: - # name_dict[video.series] += 1 - # except KeyError: - # name_dict[video.series] = 0 - # except: - # print('video did not have attrib series') - # pass - mediafiles.append(video) - except ValueError: # pragma: no cover logging.exception('Error scanning video') continue - elif archives and filename.endswith(ARCHIVE_EXTENSIONS): # archive - print('archive') - pass - # try: - # video = scan_archive(filepath) - # mediafiles.append(video) - # except (NotRarFile, RarCannotExec, ValueError): # pragma: no cover - # logging.exception('Error scanning archive') - # continue - # elif filename.endswith(SUBTITLE_EXTENSIONS): # subtitle - # try: - # subtitle = scan_subtitle(filepath) - # mediafiles.append(subtitle) - # except ValueError: - # logging.exception('Error scanning subtitle') - # continue else: # pragma: no cover - print('Skipping unsupported file {}'.format(filename)) - # raise ValueError('Unsupported file %r' % filename) + raise ValueError('Unsupported file %r' % filename) + videos.append(video) - pprint(name_dict) - return mediafiles + return videos def organize_files(path): hashList = {} mediafiles = scan_files(path) - print(mediafiles) + # print(mediafiles) for file in mediafiles: hashList.setdefault(file.__hash__(),[]).append(file) @@ -257,21 +248,18 @@ def save_subtitles(files, single=False, directory=None, encoding=None): # return saved_subtitles -def stringTime(): - return str(datetime.now().strftime("%Y-%m-%d %H:%M:%S:%f")) - def main(): # episodePath = '/Volumes/media/tv/Black Mirror/Black Mirror Season 01/' - episodePath = '/Volumes/mainframe/shows/Black Mirror/Black Mirror Season 01/' - episodePath = '/Volumes/mainframe/shows/The.Voice.S14E24.720p.WEB.x264-TBS[rarbg]' - episodePath = '/Volumes/mainframe/incomplete' + path = '/mnt/rescue/' - t = tvdb_api.Tvdb() + # t = tvdb_api.Tvdb() - hashList = organize_files(episodePath) - pprint(hashList) + # hashList = organize_files(episodePath) + # pprint(hashList) + videos = scan_videos(path) + pprint(videos) if __name__ == '__main__': From 61a1307275a3cf4dbceb72d4bcb3a8e968404e3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Fri, 14 Sep 2018 19:31:16 +0200 Subject: [PATCH 05/60] Removed custom paramenters and class functions for subtitles and video. --- src/subtitle.py | 223 +++++++++++++++++++++++++++++++++++++++++++----- src/video.py | 4 +- 2 files changed, 206 insertions(+), 21 deletions(-) diff --git a/src/subtitle.py b/src/subtitle.py index ee36ff1..465c49b 100644 --- a/src/subtitle.py +++ b/src/subtitle.py @@ -15,7 +15,7 @@ from langdetect import detect logger = logging.getLogger(__name__) #: Subtitle extensions -SUBTITLE_EXTENSIONS = ('.srt', '.sub') +SUBTITLE_EXTENSIONS = ('.srt') class Subtitle(object): @@ -33,28 +33,214 @@ class Subtitle(object): #: Name of the provider that returns that class of subtitle provider_name = '' - def __init__(self, name, parent_path, series, season, episode, language=None, hash=None, container=None, format=None, sdh=False): - #: Language of the subtitle - - self.name = name + def __init__(self, language, hearing_impaired=False, page_link=None, encoding=None): + #: Language of the subtitle + self.language = language - self.parent_path = parent_path - - self.series = series - - self.season = season + #: Whether or not the subtitle is hearing impaired + self.hearing_impaired = hearing_impaired - self.episode = episode + #: URL of the web page from which the subtitle can be downloaded + self.page_link = page_link - self.language=language + #: Content as bytes + self.content = None - self.hash = hash + #: Encoding to decode with when accessing :attr:`text` + self.encoding = None - self.container = container - - self.format = format + # validate the encoding + if encoding: + try: + self.encoding = codecs.lookup(encoding).name + except (TypeError, LookupError): + logger.debug('Unsupported encoding %s', encoding) - self.sdh = sdh + @property + def id(self): + """Unique identifier of the subtitle""" + raise NotImplementedError + + @property + def text(self): + """Content as string + If :attr:`encoding` is None, the encoding is guessed with :meth:`guess_encoding` + """ + if not self.content: + return + + if self.encoding: + return self.content.decode(self.encoding, errors='replace') + + return self.content.decode(self.guess_encoding(), errors='replace') + + def is_valid(self): + """Check if a :attr:`text` is a valid SubRip format. + :return: whether or not the subtitle is valid. + :rtype: bool + """ + if not self.text: + return False + + try: + pysrt.from_string(self.text, error_handling=pysrt.ERROR_RAISE) + except pysrt.Error as e: + if e.args[0] < 80: + return False + + return True + + def guess_encoding(self): + """Guess encoding using the language, falling back on chardet. + :return: the guessed encoding. + :rtype: str + """ + logger.info('Guessing encoding for language %s', self.language) + + # always try utf-8 first + encodings = ['utf-8'] + + # add language-specific encodings + if self.language.alpha3 == 'zho': + encodings.extend(['gb18030', 'big5']) + elif self.language.alpha3 == 'jpn': + encodings.append('shift-jis') + elif self.language.alpha3 == 'ara': + encodings.append('windows-1256') + elif self.language.alpha3 == 'heb': + encodings.append('windows-1255') + elif self.language.alpha3 == 'tur': + encodings.extend(['iso-8859-9', 'windows-1254']) + elif self.language.alpha3 == 'pol': + # Eastern European Group 1 + encodings.extend(['windows-1250']) + elif self.language.alpha3 == 'bul': + # Eastern European Group 2 + encodings.extend(['windows-1251']) + else: + # Western European (windows-1252) + encodings.append('latin-1') + + # try to decode + logger.debug('Trying encodings %r', encodings) + for encoding in encodings: + try: + self.content.decode(encoding) + except UnicodeDecodeError: + pass + else: + logger.info('Guessed encoding %s', encoding) + return encoding + + logger.warning('Could not guess encoding from language') + + # fallback on chardet + encoding = chardet.detect(self.content)['encoding'] + logger.info('Chardet found encoding %s', encoding) + + return encoding + + def get_matches(self, video): + """Get the matches against the `video`. + :param video: the video to get the matches with. + :type video: :class:`~subliminal.video.Video` + :return: matches of the subtitle. + :rtype: set + """ + raise NotImplementedError + + def __hash__(self): + return hash(self.provider_name + '-' + self.id) + + def __repr__(self): + return '<%s %r [%s]>' % (self.__class__.__name__, self.id, self.language) + + +def get_subtitle_path(video_path, language=None, extension='.srt'): + """Get the subtitle path using the `video_path` and `language`. + :param str video_path: path to the video. + :param language: language of the subtitle to put in the path. + :type language: :class:`~babelfish.language.Language` + :param str extension: extension of the subtitle. + :return: path of the subtitle. + :rtype: str + """ + subtitle_root = os.path.splitext(video_path)[0] + + if language: + subtitle_root += '.' + str(language) + + return subtitle_root + extension + + +def guess_matches(video, guess, partial=False): + """Get matches between a `video` and a `guess`. + If a guess is `partial`, the absence information won't be counted as a match. + :param video: the video. + :type video: :class:`~subliminal.video.Video` + :param guess: the guess. + :type guess: dict + :param bool partial: whether or not the guess is partial. + :return: matches between the `video` and the `guess`. + :rtype: set + """ + matches = set() + if isinstance(video, Episode): + # series + if video.series and 'title' in guess and sanitize(guess['title']) == sanitize(video.series): + matches.add('series') + # title + if video.title and 'episode_title' in guess and sanitize(guess['episode_title']) == sanitize(video.title): + matches.add('title') + # season + if video.season and 'season' in guess and guess['season'] == video.season: + matches.add('season') + # episode + if video.episode and 'episode' in guess and guess['episode'] == video.episode: + matches.add('episode') + # year + if video.year and 'year' in guess and guess['year'] == video.year: + matches.add('year') + # count "no year" as an information + if not partial and video.original_series and 'year' not in guess: + matches.add('year') + elif isinstance(video, Movie): + # year + if video.year and 'year' in guess and guess['year'] == video.year: + matches.add('year') + # title + if video.title and 'title' in guess and sanitize(guess['title']) == sanitize(video.title): + matches.add('title') + # release_group + if (video.release_group and 'release_group' in guess and + sanitize_release_group(guess['release_group']) in + get_equivalent_release_groups(sanitize_release_group(video.release_group))): + matches.add('release_group') + # resolution + if video.resolution and 'screen_size' in guess and guess['screen_size'] == video.resolution: + matches.add('resolution') + # format + if video.format and 'format' in guess and guess['format'].lower() == video.format.lower(): + matches.add('format') + # video_codec + if video.video_codec and 'video_codec' in guess and guess['video_codec'] == video.video_codec: + matches.add('video_codec') + # audio_codec + if video.audio_codec and 'audio_codec' in guess and guess['audio_codec'] == video.audio_codec: + matches.add('audio_codec') + + return matches + + +def fix_line_ending(content): + """Fix line ending of `content` by changing it to \n. + :param bytes content: content of the subtitle. + :return: the content with fixed line endings. + :rtype: bytes + """ + return content.replace(b'\r\n', b'\n').replace(b'\r', b'\n') + +''' @classmethod def fromguess(cls, name, parent_path, guess): @@ -107,5 +293,4 @@ def get_subtitle_path(subtitles_path, language=None, extension='.srt'): return subtitle_root + extension - - +''' \ No newline at end of file diff --git a/src/video.py b/src/video.py index f724598..f2043a0 100644 --- a/src/video.py +++ b/src/video.py @@ -78,14 +78,14 @@ class Video(object): return timedelta() @classmethod - def fromguess(cls, name, parent_path, guess): + def fromguess(cls, name, guess): """Create an :class:`Episode` or a :class:`Movie` with the given `name` based on the `guess`. :param str name: name of the video. :param dict guess: guessed data. :raise: :class:`ValueError` if the `type` of the `guess` is invalid """ if guess['type'] == 'episode': - return Episode.fromguess(name, parent_path, guess) + return Episode.fromguess(name, guess) if guess['type'] == 'movie': return Movie.fromguess(name, guess) From 541a18ac9f79086201c03f93e06c78cfb385c969 Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Fri, 14 Sep 2018 19:32:24 +0200 Subject: [PATCH 06/60] Disabled import of langdetect library --- src/subtitle.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/subtitle.py b/src/subtitle.py index 465c49b..f377ebb 100644 --- a/src/subtitle.py +++ b/src/subtitle.py @@ -9,7 +9,7 @@ import hashlib from video import Episode, Movie from utils import sanitize -from langdetect import detect +# from langdetect import detect logger = logging.getLogger(__name__) @@ -293,4 +293,4 @@ def get_subtitle_path(subtitles_path, language=None, extension='.srt'): return subtitle_root + extension -''' \ No newline at end of file +''' From 1075942c5232ca36970c9fa6ce3bd9e13564048d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Fri, 14 Sep 2018 19:49:35 +0200 Subject: [PATCH 07/60] Crawls path and checks file if path is not directory. Subtitles for file is also collected. --- src/core.py | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/src/core.py b/src/core.py index d1a7071..49ab53a 100755 --- a/src/core.py +++ b/src/core.py @@ -258,8 +258,56 @@ def main(): # hashList = organize_files(episodePath) # pprint(hashList) - videos = scan_videos(path) - pprint(videos) + # videos = scan_videos(path) + # pprint(videos) + + force = False + + videos = [] + ignored_videos = [] + errored_paths = [] + logger.debug('Collecting path %s', path) + + # non-existing + if not os.path.exists(path): + try: + video = Video.fromname(path) + except: + logger.exception('Unexpected error while collecting non-existing path %s', path) + errored_paths.append(path) + continue + if not force: + video.subtitle_languages |= set(search_external_subtitles(video.name, directory=path).values()) + # refine(video, episode_refiners=refiner, movie_refiners=refiner, embedded_subtitles=not force) + videos.append(video) + continue + + # directories + if os.path.isdir(path): + try: + scanned_videos = scan_videos(path) + except: + logger.exception('Unexpected error while collecting directory path %s', path) + errored_paths.append(path) + continue + for video in scanned_videos: + if not force: + video.subtitle_languages |= set(search_external_subtitles(video.name, + directory=path).values()) + videos.append(video) + continue + + # other inputs + try: + video = scan_video(path) + except: + logger.exception('Unexpected error while collecting path %s', path) + errored_paths.append(path) + continue + if not force: + video.subtitle_languages |= set(search_external_subtitles(video.name, directory=path).values()) + + videos.append(video) if __name__ == '__main__': From 4321f967f8d5d5a31dd7c0423b6c20524deb634f Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Fri, 14 Sep 2018 19:52:02 +0200 Subject: [PATCH 08/60] Syntax error for comment resolved. More exact path and removed option for archive --- src/core.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core.py b/src/core.py index d1a7071..54e9e12 100755 --- a/src/core.py +++ b/src/core.py @@ -58,7 +58,7 @@ def scan_video(path): if not os.path.exists(path): raise ValueError('Path does not exist') - check video extension + # check video extension if not path.endswith(VIDEO_EXTENSIONS): raise ValueError('%r is not a valid video extension' % os.path.splitext(path)[1]) @@ -118,7 +118,7 @@ def scan_videos(path): # scan for videos for filename in filenames: # filter on videos and archives - if not (filename.endswith(VIDEO_EXTENSIONS) or archives and filename.endswith(ARCHIVE_EXTENSIONS)): + if not (filename.endswith(VIDEO_EXTENSIONS) or filename.endswith(ARCHIVE_EXTENSIONS)): continue # skip hidden files @@ -251,7 +251,7 @@ def save_subtitles(files, single=False, directory=None, encoding=None): def main(): # episodePath = '/Volumes/media/tv/Black Mirror/Black Mirror Season 01/' - path = '/mnt/rescue/' + path = '/mnt/rescue/#137101383' # t = tvdb_api.Tvdb() From 9c7a44b027aceb96e9dd9fa751748d0e8876ea69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Fri, 14 Sep 2018 19:54:42 +0200 Subject: [PATCH 09/60] Removed continues when not in loop. --- src/core.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/core.py b/src/core.py index 49ab53a..98c6078 100755 --- a/src/core.py +++ b/src/core.py @@ -275,12 +275,10 @@ def main(): except: logger.exception('Unexpected error while collecting non-existing path %s', path) errored_paths.append(path) - continue if not force: video.subtitle_languages |= set(search_external_subtitles(video.name, directory=path).values()) # refine(video, episode_refiners=refiner, movie_refiners=refiner, embedded_subtitles=not force) videos.append(video) - continue # directories if os.path.isdir(path): @@ -289,13 +287,11 @@ def main(): except: logger.exception('Unexpected error while collecting directory path %s', path) errored_paths.append(path) - continue for video in scanned_videos: if not force: video.subtitle_languages |= set(search_external_subtitles(video.name, directory=path).values()) videos.append(video) - continue # other inputs try: @@ -303,7 +299,6 @@ def main(): except: logger.exception('Unexpected error while collecting path %s', path) errored_paths.append(path) - continue if not force: video.subtitle_languages |= set(search_external_subtitles(video.name, directory=path).values()) From 417e6dda8fdc5c6a99d7a931e382910a858cde85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Fri, 14 Sep 2018 19:56:13 +0200 Subject: [PATCH 10/60] Renamed all loggers to logging --- src/core.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/core.py b/src/core.py index 98c6078..fe5fce0 100755 --- a/src/core.py +++ b/src/core.py @@ -39,10 +39,10 @@ def search_external_subtitles(path, directory=None): try: language = Language.fromietf(language_code) except (ValueError, LanguageReverseError): - logger.error('Cannot parse language code %r', language_code) + logging.error('Cannot parse language code %r', language_code) subtitles[p] = language - logger.debug('Found subtitles %r', subtitles) + logging.debug('Found subtitles %r', subtitles) return subtitles @@ -266,14 +266,14 @@ def main(): videos = [] ignored_videos = [] errored_paths = [] - logger.debug('Collecting path %s', path) + logging.debug('Collecting path %s', path) # non-existing if not os.path.exists(path): try: video = Video.fromname(path) except: - logger.exception('Unexpected error while collecting non-existing path %s', path) + logging.exception('Unexpected error while collecting non-existing path %s', path) errored_paths.append(path) if not force: video.subtitle_languages |= set(search_external_subtitles(video.name, directory=path).values()) @@ -285,7 +285,7 @@ def main(): try: scanned_videos = scan_videos(path) except: - logger.exception('Unexpected error while collecting directory path %s', path) + logging.exception('Unexpected error while collecting directory path %s', path) errored_paths.append(path) for video in scanned_videos: if not force: @@ -297,7 +297,7 @@ def main(): try: video = scan_video(path) except: - logger.exception('Unexpected error while collecting path %s', path) + logging.exception('Unexpected error while collecting path %s', path) errored_paths.append(path) if not force: video.subtitle_languages |= set(search_external_subtitles(video.name, directory=path).values()) From fb7f6238818e8e368b88d2fb826883a755a9aee3 Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Fri, 14 Sep 2018 21:58:51 +0200 Subject: [PATCH 11/60] Imported default conf for video. --- src/video.py | 106 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) diff --git a/src/video.py b/src/video.py index f2043a0..a6919c9 100644 --- a/src/video.py +++ b/src/video.py @@ -106,6 +106,111 @@ class Video(object): return hash(self.name) +class Episode(Video): + """Episode :class:`Video`. + :param str series: series of the episode. + :param int season: season number of the episode. + :param int episode: episode number of the episode. + :param str title: title of the episode. + :param int year: year of the series. + :param bool original_series: whether the series is the first with this name. + :param int tvdb_id: TVDB id of the episode. + :param \*\*kwargs: additional parameters for the :class:`Video` constructor. + """ + def __init__(self, name, series, season, episode, title=None, year=None, original_series=True, tvdb_id=None, + series_tvdb_id=None, series_imdb_id=None, **kwargs): + super(Episode, self).__init__(name, **kwargs) + + #: Series of the episode + self.series = series + + #: Season number of the episode + self.season = season + + #: Episode number of the episode + self.episode = episode + + #: Title of the episode + self.title = title + + #: Year of series + self.year = year + + #: The series is the first with this name + self.original_series = original_series + + #: TVDB id of the episode + self.tvdb_id = tvdb_id + + #: TVDB id of the series + self.series_tvdb_id = series_tvdb_id + + #: IMDb id of the series + self.series_imdb_id = series_imdb_id + + @classmethod + def fromguess(cls, name, guess): + if guess['type'] != 'episode': + raise ValueError('The guess must be an episode guess') + + if 'title' not in guess or 'episode' not in guess: + raise ValueError('Insufficient data to process the guess') + + return cls(name, guess['title'], guess.get('season', 1), guess['episode'], title=guess.get('episode_title'), + year=guess.get('year'), format=guess.get('format'), original_series='year' not in guess, + release_group=guess.get('release_group'), resolution=guess.get('screen_size'), + video_codec=guess.get('video_codec'), audio_codec=guess.get('audio_codec')) + + @classmethod + def fromname(cls, name): + return cls.fromguess(name, guessit(name, {'type': 'episode'})) + + def __repr__(self): + if self.year is None: + return '<%s [%r, %dx%s]>' % (self.__class__.__name__, self.series, self.season, self.episode) + if self.subtitle_languages is not None: + return '<%s [%r, %dx%s] %s>' % (self.__class__.__name__, self.series, self.season, self.episode, self.subtitle_languages) + + return '<%s [%r, %d, %dx%d]>' % (self.__class__.__name__, self.series, self.year, self.season, self.episode) + + +class Movie(Video): + """Movie :class:`Video`. + :param str title: title of the movie. + :param int year: year of the movie. + :param \*\*kwargs: additional parameters for the :class:`Video` constructor. + """ + def __init__(self, name, title, year=None, **kwargs): + super(Movie, self).__init__(name, **kwargs) + + #: Title of the movie + self.title = title + + #: Year of the movie + self.year = year + + @classmethod + def fromguess(cls, name, guess): + if guess['type'] != 'movie': + raise ValueError('The guess must be a movie guess') + + if 'title' not in guess: + raise ValueError('Insufficient data to process the guess') + + return cls(name, guess['title'], format=guess.get('format'), release_group=guess.get('release_group'), + resolution=guess.get('screen_size'), video_codec=guess.get('video_codec'), + audio_codec=guess.get('audio_codec'), year=guess.get('year')) + + @classmethod + def fromname(cls, name): + return cls.fromguess(name, guessit(name, {'type': 'movie'})) + + def __repr__(self): + if self.year is None: + return '<%s [%r]>' % (self.__class__.__name__, self.title) + + return '<%s [%r, %d]>' % (self.__class__.__name__, self.title, self.year) +''' class Episode(): """Episode :class:`Video`. :param str series: series of the episode. @@ -231,3 +336,4 @@ class Movie(): return '<%s [%r]>' % (self.__class__.__name__, self.title) return '<%s [%r, %d]>' % (self.__class__.__name__, self.title, self.year) +''' From eb055db2fd1643aaa9a0552b4d08d20c313713f6 Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Fri, 14 Sep 2018 22:00:33 +0200 Subject: [PATCH 12/60] Added input for hash folders --- src/core.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/core.py b/src/core.py index 49ffbb8..4e920cb 100755 --- a/src/core.py +++ b/src/core.py @@ -6,6 +6,7 @@ # @Last Modified time: 2017-09-29 12:35:24 from guessit import guessit +from babelfish import Language, LanguageReverseError import os, errno import logging import tvdb_api @@ -251,7 +252,9 @@ def save_subtitles(files, single=False, directory=None, encoding=None): def main(): # episodePath = '/Volumes/media/tv/Black Mirror/Black Mirror Season 01/' - path = '/mnt/rescue/#137101383' + path = '/mnt/rescue/' + # hash_path = input('Hash: ') + # path += hash_path # t = tvdb_api.Tvdb() @@ -304,6 +307,9 @@ def main(): videos.append(video) + for video in videos: + pprint(video) + if __name__ == '__main__': main() From 6b9de3a9750e0e576a5355db4a58d2a2ab2a9b3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sat, 15 Sep 2018 12:12:07 +0200 Subject: [PATCH 13/60] Included enzyme package version 0.4.1 for mkv parsing. --- requirements.txt | 2 + src/core.py | 153 +++++++++++++++++++++++++++-------------------- 2 files changed, 89 insertions(+), 66 deletions(-) diff --git a/requirements.txt b/requirements.txt index 0ed1f9c..dd58b86 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,4 @@ guessit==2.1.4 tvdb_api==2.0 +hashids==1.2.0 +enzyme>=0.4.1 diff --git a/src/core.py b/src/core.py index 4e920cb..179ee1d 100755 --- a/src/core.py +++ b/src/core.py @@ -7,6 +7,7 @@ from guessit import guessit from babelfish import Language, LanguageReverseError +from hashids import Hashids import os, errno import logging import tvdb_api @@ -67,7 +68,15 @@ def scan_video(path): logging.info('Scanning video %r in %r', filename, dirpath) # guess - video = Video.fromguess(path, guessit(path)) + video = Video.fromguess(path, filename, guessit(path)) + + # size + video.size = os.path.getsize(path) + + # hash of name + hashids = Hashids(min_length=16) + hashid = hashids.encode(path) + video.name_hash = name_hash return video @@ -210,6 +219,83 @@ def save_subtitles(files, single=False, directory=None, encoding=None): print() + def refine(video, episode_refiners=None, movie_refiners=None, **kwargs): + """Refine a video using :ref:`refiners`. + .. note:: + Exceptions raised in refiners are silently passed and logged. + :param video: the video to refine. + :type video: :class:`~subliminal.video.Video` + :param tuple episode_refiners: refiners to use for episodes. + :param tuple movie_refiners: refiners to use for movies. + :param \*\*kwargs: additional parameters for the :func:`~subliminal.refiners.refine` functions. + """ + refiners = () + if isinstance(video, Episode): + refiners = episode_refiners or ('metadata') + elif isinstance(video, Movie): + refiners = movie_refiners or ('metadata') + for refiner in refiners: + logger.info('Refining video with %s', refiner) + try: + print(refiner) + refiner_manager[refiner].plugin(video, **kwargs) + except: + logger.exception('Failed to refine video') + +def scan_folder(path): + videos = [] + ignored_videos = [] + errored_paths = [] + logging.debug('Collecting path %s', path) + + # non-existing + if not os.path.exists(path): + try: + video = Video.fromname(path) + except: + logging.exception('Unexpected error while collecting non-existing path %s', path) + errored_paths.append(path) + + video.subtitle_languages |= set(search_external_subtitles(video.name, directory=path).values()) + + refine(video, episode_refiners=refiner, movie_refiners=refiner, embedded_subtitles=not force) + videos.append(video) + + # directories + if os.path.isdir(path): + try: + scanned_videos = scan_videos(path) + except: + logging.exception('Unexpected error while collecting directory path %s', path) + errored_paths.append(path) + + for video in scanned_videos: + video.subtitle_languages |= set(search_external_subtitles(video.name, + directory=path).values()) + refine(video, episode_refiners=refiner, movie_refiners=refiner, embedded_subtitles=not force) + videos.append(video) + + return videos + +def main(): + path = '/mnt/rescue/' + # hash_path = input('Hash: ') + # path += hash_path + + # t = tvdb_api.Tvdb() + + # hashList = organize_files(episodePath) + # pprint(hashList) + + videos = scan_folder() + for video in videos: + pprint(video) + + +if __name__ == '__main__': + main() + + # for hash in files: # hashIndex = [files[hash]] # for hashItems in hashIndex: @@ -248,68 +334,3 @@ def save_subtitles(files, single=False, directory=None, encoding=None): # break # return saved_subtitles - - -def main(): - # episodePath = '/Volumes/media/tv/Black Mirror/Black Mirror Season 01/' - path = '/mnt/rescue/' - # hash_path = input('Hash: ') - # path += hash_path - - # t = tvdb_api.Tvdb() - - # hashList = organize_files(episodePath) - # pprint(hashList) - - # videos = scan_videos(path) - # pprint(videos) - - force = False - - videos = [] - ignored_videos = [] - errored_paths = [] - logging.debug('Collecting path %s', path) - - # non-existing - if not os.path.exists(path): - try: - video = Video.fromname(path) - except: - logging.exception('Unexpected error while collecting non-existing path %s', path) - errored_paths.append(path) - if not force: - video.subtitle_languages |= set(search_external_subtitles(video.name, directory=path).values()) - # refine(video, episode_refiners=refiner, movie_refiners=refiner, embedded_subtitles=not force) - videos.append(video) - - # directories - if os.path.isdir(path): - try: - scanned_videos = scan_videos(path) - except: - logging.exception('Unexpected error while collecting directory path %s', path) - errored_paths.append(path) - for video in scanned_videos: - if not force: - video.subtitle_languages |= set(search_external_subtitles(video.name, - directory=path).values()) - videos.append(video) - - # other inputs - try: - video = scan_video(path) - except: - logging.exception('Unexpected error while collecting path %s', path) - errored_paths.append(path) - if not force: - video.subtitle_languages |= set(search_external_subtitles(video.name, directory=path).values()) - - videos.append(video) - - for video in videos: - pprint(video) - - -if __name__ == '__main__': - main() From 077ac940b190a8a1e20fd58a83068359f0d6fcb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sat, 15 Sep 2018 12:15:00 +0200 Subject: [PATCH 14/60] Refine function added to utils to extract metadata from mkv files. --- src/utils.py | 92 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/src/utils.py b/src/utils.py index 2cd6ea7..9495dd3 100644 --- a/src/utils.py +++ b/src/utils.py @@ -2,9 +2,15 @@ from datetime import datetime import hashlib import os +import logging import re import struct +from babelfish import Error as BabelfishError, Language +from enzyme import MKV + +logger = logging.getLogger(__name__) + def sanitize(string, ignore_characters=None): """Sanitize a string to strip special characters. @@ -36,3 +42,89 @@ def sanitize(string, ignore_characters=None): # strip and lower case return string.strip().lower() + +def refine(video, embedded_subtitles=True, **kwargs): + """Refine a video by searching its metadata. + Several :class:`~subliminal.video.Video` attributes can be found: + * :attr:`~subliminal.video.Video.resolution` + * :attr:`~subliminal.video.Video.video_codec` + * :attr:`~subliminal.video.Video.audio_codec` + * :attr:`~subliminal.video.Video.subtitle_languages` + :param bool embedded_subtitles: search for embedded subtitles. + """ + # skip non existing videos + if not video.exists: + return + + # check extensions + extension = os.path.splitext(video.name)[1] + if extension == '.mkv': + with open(video.name, 'rb') as f: + mkv = MKV(f) + + # main video track + if mkv.video_tracks: + video_track = mkv.video_tracks[0] + + # resolution + if video_track.height in (480, 720, 1080): + if video_track.interlaced: + video.resolution = '%di' % video_track.height + else: + video.resolution = '%dp' % video_track.height + logger.debug('Found resolution %s', video.resolution) + + # video codec + if video_track.codec_id == 'V_MPEG4/ISO/AVC': + video.video_codec = 'h264' + logger.debug('Found video_codec %s', video.video_codec) + elif video_track.codec_id == 'V_MPEG4/ISO/SP': + video.video_codec = 'DivX' + logger.debug('Found video_codec %s', video.video_codec) + elif video_track.codec_id == 'V_MPEG4/ISO/ASP': + video.video_codec = 'XviD' + logger.debug('Found video_codec %s', video.video_codec) + else: + logger.warning('MKV has no video track') + + # main audio track + if mkv.audio_tracks: + audio_track = mkv.audio_tracks[0] + # audio codec + if audio_track.codec_id == 'A_AC3': + video.audio_codec = 'AC3' + logger.debug('Found audio_codec %s', video.audio_codec) + elif audio_track.codec_id == 'A_DTS': + video.audio_codec = 'DTS' + logger.debug('Found audio_codec %s', video.audio_codec) + elif audio_track.codec_id == 'A_AAC': + video.audio_codec = 'AAC' + logger.debug('Found audio_codec %s', video.audio_codec) + else: + logger.warning('MKV has no audio track') + + # subtitle tracks + if mkv.subtitle_tracks: + if embedded_subtitles: + embedded_subtitle_languages = set() + for st in mkv.subtitle_tracks: + if st.language: + try: + embedded_subtitle_languages.add(Language.fromalpha3b(st.language)) + except BabelfishError: + logger.error('Embedded subtitle track language %r is not a valid language', st.language) + embedded_subtitle_languages.add(Language('und')) + elif st.name: + try: + embedded_subtitle_languages.add(Language.fromname(st.name)) + except BabelfishError: + logger.debug('Embedded subtitle track name %r is not a valid language', st.name) + embedded_subtitle_languages.add(Language('und')) + else: + embedded_subtitle_languages.add(Language('und')) + logger.debug('Found embedded subtitle %r', embedded_subtitle_languages) + video.subtitle_languages |= embedded_subtitle_languages + else: + logger.debug('MKV has no subtitle track') + else: + logger.debug('Unsupported video extension %s', extension) \ No newline at end of file From 934aeeb37f0baec99966fd692c16dd101cab9ce1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sat, 15 Sep 2018 12:15:56 +0200 Subject: [PATCH 15/60] Refine imported from utils and all calls to Video class no longer include path. --- src/core.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/core.py b/src/core.py index 179ee1d..aecdd72 100755 --- a/src/core.py +++ b/src/core.py @@ -17,7 +17,7 @@ import env_variables as env from video import VIDEO_EXTENSIONS, Episode, Movie, Video from subtitle import SUBTITLE_EXTENSIONS, Subtitle, get_subtitle_path -from utils import sanitize +from utils import sanitize, refine logging.basicConfig(filename=env.logfile, level=logging.INFO) @@ -68,7 +68,7 @@ def scan_video(path): logging.info('Scanning video %r in %r', filename, dirpath) # guess - video = Video.fromguess(path, filename, guessit(path)) + video = Video.fromguess(path, guessit(path)) # size video.size = os.path.getsize(path) @@ -90,7 +90,7 @@ def scan_subtitle(path): # guess parent_path = path.strip(filename) - subtitle = Subtitle.fromguess(filename, parent_path, guessit(path)) + subtitle = Subtitle.fromguess(parent_path, guessit(path)) return subtitle @@ -238,6 +238,7 @@ def save_subtitles(files, single=False, directory=None, encoding=None): logger.info('Refining video with %s', refiner) try: print(refiner) + exit(0) refiner_manager[refiner].plugin(video, **kwargs) except: logger.exception('Failed to refine video') From 24047a2b1d08b59b3f2e0e5652e5953f5532034d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sat, 15 Sep 2018 12:16:48 +0200 Subject: [PATCH 16/60] Hash_name variable added to Video class and reordered varaibles in class declaration. --- src/video.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/video.py b/src/video.py index a6919c9..3859257 100644 --- a/src/video.py +++ b/src/video.py @@ -28,15 +28,21 @@ class Video(object): :param str video_codec: codec of the video stream. :param str audio_codec: codec of the main audio stream. :param str imdb_id: IMDb id of the video. - :param dict hashes: hashes of the video file by provider names. + :param dict name_hash: hashes of the video file by provider names. :param int size: size of the video file in bytes. :param set subtitle_languages: existing subtitle languages. """ - def __init__(self, name, format=None, release_group=None, resolution=None, video_codec=None, audio_codec=None, - imdb_id=None, hashes=None, size=None, subtitle_languages=None): + def __init__(self, name, name_hash=None, size=None, format=None, release_group=None, resolution=None, video_codec=None, audio_codec=None, + imdb_id=None, subtitle_languages=None): #: Name or path of the video self.name = name + #: Hashes of the video file by provider names + self.name_hash = name_hash + + #: Size of the video file in bytes + self.size = size + #: Format of the video (HDTV, WEB-DL, BluRay, ...) self.format = format @@ -55,12 +61,6 @@ class Video(object): #: IMDb id of the video self.imdb_id = imdb_id - #: Hashes of the video file by provider names - self.hashes = hashes or {} - - #: Size of the video file in bytes - self.size = size - #: Existing subtitle languages self.subtitle_languages = subtitle_languages or set() From 3eb3609a3894c308957ab246bbb44745c8211367 Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Sat, 15 Sep 2018 12:32:54 +0200 Subject: [PATCH 17/60] Fixed indentation of refine function and added path parameter the scan_folder function call --- src/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core.py b/src/core.py index aecdd72..5d6e0a0 100755 --- a/src/core.py +++ b/src/core.py @@ -219,7 +219,7 @@ def save_subtitles(files, single=False, directory=None, encoding=None): print() - def refine(video, episode_refiners=None, movie_refiners=None, **kwargs): +def refine(video, episode_refiners=None, movie_refiners=None, **kwargs): """Refine a video using :ref:`refiners`. .. note:: Exceptions raised in refiners are silently passed and logged. @@ -288,7 +288,7 @@ def main(): # hashList = organize_files(episodePath) # pprint(hashList) - videos = scan_folder() + videos = scan_folder(path) for video in videos: pprint(video) From 0badb4e9888f88922c06b211012d02d46c514e30 Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Sat, 15 Sep 2018 12:34:18 +0200 Subject: [PATCH 18/60] Sets name_hash to correct variable hashid not name_hash.wq --- src/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core.py b/src/core.py index 5d6e0a0..3839775 100755 --- a/src/core.py +++ b/src/core.py @@ -76,7 +76,7 @@ def scan_video(path): # hash of name hashids = Hashids(min_length=16) hashid = hashids.encode(path) - video.name_hash = name_hash + video.name_hash = hashid return video From 565921830d31be1854244fce2559c24ac4aac99f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sat, 15 Sep 2018 12:37:25 +0200 Subject: [PATCH 19/60] Refiner is not defined on first refine so pass None. --- src/core.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/core.py b/src/core.py index aecdd72..4a78876 100755 --- a/src/core.py +++ b/src/core.py @@ -259,7 +259,7 @@ def scan_folder(path): video.subtitle_languages |= set(search_external_subtitles(video.name, directory=path).values()) - refine(video, episode_refiners=refiner, movie_refiners=refiner, embedded_subtitles=not force) + refine(video, episode_refiners=None, movie_refiners=None, embedded_subtitles=not force) videos.append(video) # directories @@ -267,13 +267,14 @@ def scan_folder(path): try: scanned_videos = scan_videos(path) except: + print('Unexpected error while collecting directory path %s', path) logging.exception('Unexpected error while collecting directory path %s', path) errored_paths.append(path) for video in scanned_videos: video.subtitle_languages |= set(search_external_subtitles(video.name, directory=path).values()) - refine(video, episode_refiners=refiner, movie_refiners=refiner, embedded_subtitles=not force) + refine(video, episode_refiners=None, movie_refiners=None, embedded_subtitles=not force) videos.append(video) return videos From f11bb3339c51b489785841176398228c96c9b179 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sat, 15 Sep 2018 12:40:54 +0200 Subject: [PATCH 20/60] Removed all optional params for refine function. --- src/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core.py b/src/core.py index 180d137..9bf1c83 100755 --- a/src/core.py +++ b/src/core.py @@ -259,7 +259,7 @@ def scan_folder(path): video.subtitle_languages |= set(search_external_subtitles(video.name, directory=path).values()) - refine(video, episode_refiners=None, movie_refiners=None, embedded_subtitles=not force) + refine(video) videos.append(video) # directories @@ -274,7 +274,7 @@ def scan_folder(path): for video in scanned_videos: video.subtitle_languages |= set(search_external_subtitles(video.name, directory=path).values()) - refine(video, episode_refiners=None, movie_refiners=None, embedded_subtitles=not force) + refine(video) videos.append(video) return videos From d50b97daa3e4744277f963b992b5e08a30b37b2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sat, 15 Sep 2018 12:44:15 +0200 Subject: [PATCH 21/60] Changed all logger instances with the default logging. --- src/utils.py | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/src/utils.py b/src/utils.py index 9495dd3..f30b6fa 100644 --- a/src/utils.py +++ b/src/utils.py @@ -9,8 +9,6 @@ import struct from babelfish import Error as BabelfishError, Language from enzyme import MKV -logger = logging.getLogger(__name__) - def sanitize(string, ignore_characters=None): """Sanitize a string to strip special characters. @@ -72,20 +70,20 @@ def refine(video, embedded_subtitles=True, **kwargs): video.resolution = '%di' % video_track.height else: video.resolution = '%dp' % video_track.height - logger.debug('Found resolution %s', video.resolution) + logging.debug('Found resolution %s', video.resolution) # video codec if video_track.codec_id == 'V_MPEG4/ISO/AVC': video.video_codec = 'h264' - logger.debug('Found video_codec %s', video.video_codec) + logging.debug('Found video_codec %s', video.video_codec) elif video_track.codec_id == 'V_MPEG4/ISO/SP': video.video_codec = 'DivX' - logger.debug('Found video_codec %s', video.video_codec) + logging.debug('Found video_codec %s', video.video_codec) elif video_track.codec_id == 'V_MPEG4/ISO/ASP': video.video_codec = 'XviD' - logger.debug('Found video_codec %s', video.video_codec) + logging.debug('Found video_codec %s', video.video_codec) else: - logger.warning('MKV has no video track') + logging.warning('MKV has no video track') # main audio track if mkv.audio_tracks: @@ -93,15 +91,15 @@ def refine(video, embedded_subtitles=True, **kwargs): # audio codec if audio_track.codec_id == 'A_AC3': video.audio_codec = 'AC3' - logger.debug('Found audio_codec %s', video.audio_codec) + logging.debug('Found audio_codec %s', video.audio_codec) elif audio_track.codec_id == 'A_DTS': video.audio_codec = 'DTS' - logger.debug('Found audio_codec %s', video.audio_codec) + logging.debug('Found audio_codec %s', video.audio_codec) elif audio_track.codec_id == 'A_AAC': video.audio_codec = 'AAC' - logger.debug('Found audio_codec %s', video.audio_codec) + logging.debug('Found audio_codec %s', video.audio_codec) else: - logger.warning('MKV has no audio track') + logging.warning('MKV has no audio track') # subtitle tracks if mkv.subtitle_tracks: @@ -112,19 +110,19 @@ def refine(video, embedded_subtitles=True, **kwargs): try: embedded_subtitle_languages.add(Language.fromalpha3b(st.language)) except BabelfishError: - logger.error('Embedded subtitle track language %r is not a valid language', st.language) + logging.error('Embedded subtitle track language %r is not a valid language', st.language) embedded_subtitle_languages.add(Language('und')) elif st.name: try: embedded_subtitle_languages.add(Language.fromname(st.name)) except BabelfishError: - logger.debug('Embedded subtitle track name %r is not a valid language', st.name) + logging.debug('Embedded subtitle track name %r is not a valid language', st.name) embedded_subtitle_languages.add(Language('und')) else: embedded_subtitle_languages.add(Language('und')) - logger.debug('Found embedded subtitle %r', embedded_subtitle_languages) + logging.debug('Found embedded subtitle %r', embedded_subtitle_languages) video.subtitle_languages |= embedded_subtitle_languages else: - logger.debug('MKV has no subtitle track') + logging.debug('MKV has no subtitle track') else: - logger.debug('Unsupported video extension %s', extension) \ No newline at end of file + logging.debug('Unsupported video extension %s', extension) \ No newline at end of file From 388d1a927edfd083e396ccf9fec371e28af43d83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sat, 15 Sep 2018 12:47:09 +0200 Subject: [PATCH 22/60] Removed refined functions in core. Use the utils one instead. --- src/core.py | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/src/core.py b/src/core.py index 9bf1c83..a87ea46 100755 --- a/src/core.py +++ b/src/core.py @@ -219,29 +219,29 @@ def save_subtitles(files, single=False, directory=None, encoding=None): print() -def refine(video, episode_refiners=None, movie_refiners=None, **kwargs): - """Refine a video using :ref:`refiners`. - .. note:: - Exceptions raised in refiners are silently passed and logged. - :param video: the video to refine. - :type video: :class:`~subliminal.video.Video` - :param tuple episode_refiners: refiners to use for episodes. - :param tuple movie_refiners: refiners to use for movies. - :param \*\*kwargs: additional parameters for the :func:`~subliminal.refiners.refine` functions. - """ - refiners = () - if isinstance(video, Episode): - refiners = episode_refiners or ('metadata') - elif isinstance(video, Movie): - refiners = movie_refiners or ('metadata') - for refiner in refiners: - logger.info('Refining video with %s', refiner) - try: - print(refiner) - exit(0) - refiner_manager[refiner].plugin(video, **kwargs) - except: - logger.exception('Failed to refine video') +# def refine(video, episode_refiners=None, movie_refiners=None, **kwargs): +# """Refine a video using :ref:`refiners`. +# .. note:: +# Exceptions raised in refiners are silently passed and logged. +# :param video: the video to refine. +# :type video: :class:`~subliminal.video.Video` +# :param tuple episode_refiners: refiners to use for episodes. +# :param tuple movie_refiners: refiners to use for movies. +# :param \*\*kwargs: additional parameters for the :func:`~subliminal.refiners.refine` functions. +# """ +# refiners = () +# if isinstance(video, Episode): +# refiners = episode_refiners or ('metadata') +# elif isinstance(video, Movie): +# refiners = movie_refiners or ('metadata') +# for refiner in refiners: +# logger.info('Refining video with %s', refiner) +# try: +# print(refiner) +# exit(0) +# refiner_manager[refiner].plugin(video, **kwargs) +# except: +# logger.exception('Failed to refine video') def scan_folder(path): videos = [] From c0621f41d0ad2f045c2e717705ec2ac1ed248620 Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Sun, 16 Sep 2018 15:15:12 +0200 Subject: [PATCH 23/60] Removed archives from accepted filetypes --- src/core.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/core.py b/src/core.py index a87ea46..14cbbf2 100755 --- a/src/core.py +++ b/src/core.py @@ -22,9 +22,6 @@ from utils import sanitize, refine logging.basicConfig(filename=env.logfile, level=logging.INFO) -#: Supported archive extensions -ARCHIVE_EXTENSIONS = ('.rar',) - def search_external_subtitles(path, directory=None): dirpath, filename = os.path.split(path) dirpath = dirpath or '.' @@ -128,7 +125,8 @@ def scan_videos(path): # scan for videos for filename in filenames: # filter on videos and archives - if not (filename.endswith(VIDEO_EXTENSIONS) or filename.endswith(ARCHIVE_EXTENSIONS)): + if not (filename.endswith(VIDEO_EXTENSIONS)): + logging.debug('Skipping non-video file %s', filename) continue # skip hidden files From 3d63a3bf0816ae2e3a06987b79624e7a07b19dd7 Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Sun, 16 Sep 2018 15:21:42 +0200 Subject: [PATCH 24/60] Catches exceptions when reading mkv containers and return without crashing program. --- src/utils.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/utils.py b/src/utils.py index f30b6fa..af53255 100644 --- a/src/utils.py +++ b/src/utils.py @@ -7,7 +7,7 @@ import re import struct from babelfish import Error as BabelfishError, Language -from enzyme import MKV +from enzyme import MalformedMKVError, MKV def sanitize(string, ignore_characters=None): """Sanitize a string to strip special characters. @@ -58,7 +58,14 @@ def refine(video, embedded_subtitles=True, **kwargs): extension = os.path.splitext(video.name)[1] if extension == '.mkv': with open(video.name, 'rb') as f: - mkv = MKV(f) + try: + mkv = MKV(f) + except MalformedMKVError: + logging.error('Failed to parse mkv, malformed file') + return + except KeyError: + logging.error('Key error while opening file, uncompatible mkv container') + return # main video track if mkv.video_tracks: @@ -125,4 +132,4 @@ def refine(video, embedded_subtitles=True, **kwargs): else: logging.debug('MKV has no subtitle track') else: - logging.debug('Unsupported video extension %s', extension) \ No newline at end of file + logging.debug('Unsupported video extension %s', extension) From 5ce0e614650550c2b298b284589e6cefe237a9a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sun, 16 Sep 2018 15:25:12 +0200 Subject: [PATCH 25/60] Allow 4k metadata collection for mkv containers. --- src/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils.py b/src/utils.py index f30b6fa..fc5ffd4 100644 --- a/src/utils.py +++ b/src/utils.py @@ -65,7 +65,7 @@ def refine(video, embedded_subtitles=True, **kwargs): video_track = mkv.video_tracks[0] # resolution - if video_track.height in (480, 720, 1080): + if video_track.height in (480, 720, 1080, 2160): if video_track.interlaced: video.resolution = '%di' % video_track.height else: From 1b373087d27a61643ad9c2fa5d895ddc35c396fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sun, 16 Sep 2018 15:45:26 +0200 Subject: [PATCH 26/60] Added progressbar based on iterations of number of files in directory. --- src/core.py | 82 ++++++++++++++++++++--------------------------------- 1 file changed, 31 insertions(+), 51 deletions(-) diff --git a/src/core.py b/src/core.py index 14cbbf2..8c616ee 100755 --- a/src/core.py +++ b/src/core.py @@ -214,68 +214,48 @@ def save_subtitles(files, single=False, directory=None, encoding=None): print('Moved: %s ---> %s' % (old, newname)) os.rename(old, newname) - print() - - -# def refine(video, episode_refiners=None, movie_refiners=None, **kwargs): -# """Refine a video using :ref:`refiners`. -# .. note:: -# Exceptions raised in refiners are silently passed and logged. -# :param video: the video to refine. -# :type video: :class:`~subliminal.video.Video` -# :param tuple episode_refiners: refiners to use for episodes. -# :param tuple movie_refiners: refiners to use for movies. -# :param \*\*kwargs: additional parameters for the :func:`~subliminal.refiners.refine` functions. -# """ -# refiners = () -# if isinstance(video, Episode): -# refiners = episode_refiners or ('metadata') -# elif isinstance(video, Movie): -# refiners = movie_refiners or ('metadata') -# for refiner in refiners: -# logger.info('Refining video with %s', refiner) -# try: -# print(refiner) -# exit(0) -# refiner_manager[refiner].plugin(video, **kwargs) -# except: -# logger.exception('Failed to refine video') - def scan_folder(path): videos = [] ignored_videos = [] errored_paths = [] logging.debug('Collecting path %s', path) - # non-existing - if not os.path.exists(path): - try: - video = Video.fromname(path) - except: - logging.exception('Unexpected error while collecting non-existing path %s', path) - errored_paths.append(path) + content_count = 0 + for _ in os.listdir(path): + content_count += 1 - video.subtitle_languages |= set(search_external_subtitles(video.name, directory=path).values()) - - refine(video) - videos.append(video) + with click.progressbar(length=content_count, label='Collecting videos') as bar: + # non-existing + if not os.path.exists(path): + try: + video = Video.fromname(path) + except: + logging.exception('Unexpected error while collecting non-existing path %s', path) + errored_paths.append(path) - # directories - if os.path.isdir(path): - try: - scanned_videos = scan_videos(path) - except: - print('Unexpected error while collecting directory path %s', path) - logging.exception('Unexpected error while collecting directory path %s', path) - errored_paths.append(path) - - for video in scanned_videos: - video.subtitle_languages |= set(search_external_subtitles(video.name, - directory=path).values()) + video.subtitle_languages |= set(search_external_subtitles(video.name, directory=path).values()) + refine(video) videos.append(video) + # Increment bar to full ? - return videos + # directories + if os.path.isdir(path): + try: + scanned_videos = scan_videos(path) + except: + print('Unexpected error while collecting directory path %s', path) + logging.exception('Unexpected error while collecting directory path %s', path) + errored_paths.append(path) + + for video in scanned_videos: + video.subtitle_languages |= set(search_external_subtitles(video.name, + directory=path).values()) + refine(video) + videos.append(video) + bar.update(1) + + return videos def main(): path = '/mnt/rescue/' From 964a446a9bac8f1458575047a6f746615f74b33d Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Sun, 16 Sep 2018 15:47:42 +0200 Subject: [PATCH 27/60] Added click library --- requirements.txt | 1 + src/core.py | 1 + 2 files changed, 2 insertions(+) diff --git a/requirements.txt b/requirements.txt index dd58b86..79e1ed8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ guessit==2.1.4 tvdb_api==2.0 hashids==1.2.0 enzyme>=0.4.1 +click>=6.7 diff --git a/src/core.py b/src/core.py index 8c616ee..2f496fa 100755 --- a/src/core.py +++ b/src/core.py @@ -11,6 +11,7 @@ from hashids import Hashids import os, errno import logging import tvdb_api +import click from pprint import pprint import env_variables as env From 1ea94faa8eeb4c3d7a3afddc0f1d1543477c45ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sun, 16 Sep 2018 16:03:47 +0200 Subject: [PATCH 28/60] Moved progress bar from indexing folder content to parsing found videos. --- src/core.py | 53 ++++++++++++++++++++++++++--------------------------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/src/core.py b/src/core.py index 8c616ee..c72163c 100755 --- a/src/core.py +++ b/src/core.py @@ -224,38 +224,37 @@ def scan_folder(path): for _ in os.listdir(path): content_count += 1 - with click.progressbar(length=content_count, label='Collecting videos') as bar: - # non-existing - if not os.path.exists(path): - try: - video = Video.fromname(path) - except: - logging.exception('Unexpected error while collecting non-existing path %s', path) - errored_paths.append(path) + # non-existing + if not os.path.exists(path): + try: + video = Video.fromname(path) + except: + logging.exception('Unexpected error while collecting non-existing path %s', path) + errored_paths.append(path) - video.subtitle_languages |= set(search_external_subtitles(video.name, directory=path).values()) - - refine(video) - videos.append(video) - # Increment bar to full ? + video.subtitle_languages |= set(search_external_subtitles(video.name, directory=path).values()) + + refine(video) + videos.append(video) + # Increment bar to full ? - # directories - if os.path.isdir(path): - try: - scanned_videos = scan_videos(path) - except: - print('Unexpected error while collecting directory path %s', path) - logging.exception('Unexpected error while collecting directory path %s', path) - errored_paths.append(path) + # directories + if os.path.isdir(path): + try: + scanned_videos = scan_videos(path) + except: + print('Unexpected error while collecting directory path %s', path) + logging.exception('Unexpected error while collecting directory path %s', path) + errored_paths.append(path) - for video in scanned_videos: - video.subtitle_languages |= set(search_external_subtitles(video.name, + with click.progressbar(scanned_videos, label='Parsing found videos', item_show_function=lambda v: os.path.split(v.name)[1] if v is not None else '') as bar: + for v in bar: + v.subtitle_languages |= set(search_external_subtitles(v.name, directory=path).values()) - refine(video) - videos.append(video) - bar.update(1) + refine(v) + videos.append(v) - return videos + return videos def main(): path = '/mnt/rescue/' From 2eac7187491f4e3d0d5a46c079f47d5d2b45a1c5 Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Sun, 16 Sep 2018 16:04:22 +0200 Subject: [PATCH 29/60] Changed lading path --- src/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core.py b/src/core.py index 2f496fa..414d848 100755 --- a/src/core.py +++ b/src/core.py @@ -259,7 +259,7 @@ def scan_folder(path): return videos def main(): - path = '/mnt/rescue/' + path = '/mnt/mainframe/' # hash_path = input('Hash: ') # path += hash_path From 175b66b120d0fda93011b9786e0c5c9436a94834 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sun, 16 Sep 2018 16:28:16 +0200 Subject: [PATCH 30/60] Progressbar during parsing of scanned videos no longer shows the filename because it created newline per element. Scanning videos for a given path shows progressbar based on the number of outer folders walked throught --- src/core.py | 87 +++++++++++++++++++++++++++-------------------------- 1 file changed, 44 insertions(+), 43 deletions(-) diff --git a/src/core.py b/src/core.py index c72163c..ce36db7 100755 --- a/src/core.py +++ b/src/core.py @@ -111,50 +111,55 @@ def scan_videos(path): if not os.path.isdir(path): raise ValueError('Path is not a directory') - # walk the path - videos = [] - for dirpath, dirnames, filenames in os.walk(path): - logging.debug('Walking directory %r', dirpath) + # setup progress bar + with click.progressbar(length=len(os.listdir(path)), label='Searching for videos') as bar: - # remove badly encoded and hidden dirnames - for dirname in list(dirnames): - if dirname.startswith('.'): - logging.debug('Skipping hidden dirname %r in %r', dirname, dirpath) - dirnames.remove(dirname) + # walk the path + videos = [] + for dirpath, dirnames, filenames in os.walk(path): + logging.debug('Walking directory %r', dirpath) - # scan for videos - for filename in filenames: - # filter on videos and archives - if not (filename.endswith(VIDEO_EXTENSIONS)): - logging.debug('Skipping non-video file %s', filename) - continue + # remove badly encoded and hidden dirnames + for dirname in list(dirnames): + if dirname.startswith('.'): + logging.debug('Skipping hidden dirname %r in %r', dirname, dirpath) + dirnames.remove(dirname) - # skip hidden files - if filename.startswith('.'): - logging.debug('Skipping hidden filename %r in %r', filename, dirpath) - continue - - # reconstruct the file path - filepath = os.path.join(dirpath, filename) - - # skip links - if os.path.islink(filepath): - logging.debug('Skipping link %r in %r', filename, dirpath) - continue - - # scan - if filename.endswith(VIDEO_EXTENSIONS): # video - try: - video = scan_video(filepath) - except ValueError: # pragma: no cover - logging.exception('Error scanning video') + # scan for videos + for filename in filenames: + # filter on videos and archives + if not (filename.endswith(VIDEO_EXTENSIONS)): + logging.debug('Skipping non-video file %s', filename) continue - else: # pragma: no cover - raise ValueError('Unsupported file %r' % filename) - videos.append(video) + # skip hidden files + if filename.startswith('.'): + logging.debug('Skipping hidden filename %r in %r', filename, dirpath) + continue - return videos + # reconstruct the file path + filepath = os.path.join(dirpath, filename) + + # skip links + if os.path.islink(filepath): + logging.debug('Skipping link %r in %r', filename, dirpath) + continue + + # scan + if filename.endswith(VIDEO_EXTENSIONS): # video + try: + video = scan_video(filepath) + except ValueError: # pragma: no cover + logging.exception('Error scanning video') + continue + else: # pragma: no cover + raise ValueError('Unsupported file %r' % filename) + + videos.append(video) + + bar.update(1) + + return videos def organize_files(path): @@ -220,10 +225,6 @@ def scan_folder(path): errored_paths = [] logging.debug('Collecting path %s', path) - content_count = 0 - for _ in os.listdir(path): - content_count += 1 - # non-existing if not os.path.exists(path): try: @@ -247,7 +248,7 @@ def scan_folder(path): logging.exception('Unexpected error while collecting directory path %s', path) errored_paths.append(path) - with click.progressbar(scanned_videos, label='Parsing found videos', item_show_function=lambda v: os.path.split(v.name)[1] if v is not None else '') as bar: + with click.progressbar(scanned_videos, label='Parsing found videos') as bar: for v in bar: v.subtitle_languages |= set(search_external_subtitles(v.name, directory=path).values()) From f62b834cf9785d34cdbb31e524ce9ab2e0da6340 Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Sun, 16 Sep 2018 16:48:43 +0200 Subject: [PATCH 31/60] Progressbar when searching folders for videos now uses correct length parameter and changed the label text. --- src/core.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/core.py b/src/core.py index f2d4531..5a1ae2f 100755 --- a/src/core.py +++ b/src/core.py @@ -113,7 +113,9 @@ def scan_videos(path): raise ValueError('Path is not a directory') # setup progress bar - with click.progressbar(length=len(os.listdir(path)), label='Searching for videos') as bar: + path_children = 0 + for _ in os.walk(path): path_children += 1 + with click.progressbar(length=path_children, label='Searching folders for videos') as bar: # walk the path videos = [] From 0c1b153e03d8e104a4dd673704424e64e2b00324 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sun, 16 Sep 2018 16:55:05 +0200 Subject: [PATCH 32/60] Uses click.echo to display the number of found videos and erroros paths. --- src/core.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/core.py b/src/core.py index 5a1ae2f..7ab9fbb 100755 --- a/src/core.py +++ b/src/core.py @@ -251,6 +251,7 @@ def scan_folder(path): logging.exception('Unexpected error while collecting directory path %s', path) errored_paths.append(path) + # Iterates over our scanned videos with click.progressbar(scanned_videos, label='Parsing found videos') as bar: for v in bar: v.subtitle_languages |= set(search_external_subtitles(v.name, @@ -258,6 +259,13 @@ def scan_folder(path): refine(v) videos.append(v) + click.echo('%s video%s collected / %s error%s' % ( + click.style(str(len(videos)), bold=True, fg='green' if videos else None), + 's' if len(videos) > 1 else '', + click.style(str(len(errored_paths)), bold=True, fg='red' if errored_paths else None), + 's' if len(errored_paths) > 1 else '', + )) + return videos def main(): From 2541a2171505bd9adbef182ac2601795da1d116d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sun, 16 Sep 2018 17:26:58 +0200 Subject: [PATCH 33/60] Changed class var from name_hash to just be hash. --- src/video.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/video.py b/src/video.py index 3859257..d0c09e3 100644 --- a/src/video.py +++ b/src/video.py @@ -32,13 +32,13 @@ class Video(object): :param int size: size of the video file in bytes. :param set subtitle_languages: existing subtitle languages. """ - def __init__(self, name, name_hash=None, size=None, format=None, release_group=None, resolution=None, video_codec=None, audio_codec=None, + def __init__(self, name, hash=None, size=None, format=None, release_group=None, resolution=None, video_codec=None, audio_codec=None, imdb_id=None, subtitle_languages=None): #: Name or path of the video self.name = name #: Hashes of the video file by provider names - self.name_hash = name_hash + self.hash = hash #: Size of the video file in bytes self.size = size From d9880c77fb562dfbfdac3572af024fe4148acabd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sun, 16 Sep 2018 17:27:38 +0200 Subject: [PATCH 34/60] Sets hash value based on if movie or episode instance. --- src/core.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/core.py b/src/core.py index 7ab9fbb..3d6af28 100755 --- a/src/core.py +++ b/src/core.py @@ -73,8 +73,11 @@ def scan_video(path): # hash of name hashids = Hashids(min_length=16) - hashid = hashids.encode(path) - video.name_hash = hashid + if isinstance(v, Episode): + videoHash = hashids.encode(''.join(self.title, self.year or '')) + elif isinstance(v, Movie): + videoHash = hashids.encode(''.join(self.series, self.season, self.episode)) + video.hash = videoHash return video From ba5011d3a4f0a9450a735b9672af25eef5daa76d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sun, 16 Sep 2018 18:19:25 +0200 Subject: [PATCH 35/60] Updated progressbars to have more info and changed from procent to absolut value when searching for videos. --- src/core.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/core.py b/src/core.py index 3d6af28..fe2ab01 100755 --- a/src/core.py +++ b/src/core.py @@ -118,7 +118,7 @@ def scan_videos(path): # setup progress bar path_children = 0 for _ in os.walk(path): path_children += 1 - with click.progressbar(length=path_children, label='Searching folders for videos') as bar: + with click.progressbar(length=path_children, show_pos=True, label='Searching folders for videos') as bar: # walk the path videos = [] @@ -250,12 +250,11 @@ def scan_folder(path): try: scanned_videos = scan_videos(path) except: - print('Unexpected error while collecting directory path %s', path) logging.exception('Unexpected error while collecting directory path %s', path) errored_paths.append(path) # Iterates over our scanned videos - with click.progressbar(scanned_videos, label='Parsing found videos') as bar: + with click.progressbar(scanned_videos, label='Parsing videos') as bar: for v in bar: v.subtitle_languages |= set(search_external_subtitles(v.name, directory=path).values()) From 5916843e9538327c5f46094aa38f1e34885e86ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sun, 16 Sep 2018 21:45:23 +0200 Subject: [PATCH 36/60] Videos are picked for girl scouts and printed if become scout or remain civilian. --- src/core.py | 75 +++++++++++++++++++---------------------------------- 1 file changed, 27 insertions(+), 48 deletions(-) diff --git a/src/core.py b/src/core.py index fe2ab01..2c4518c 100755 --- a/src/core.py +++ b/src/core.py @@ -270,60 +270,39 @@ def scan_folder(path): return videos +def pickforgirlscouts(video): + if isinstance(video, Movie): + if video.title != None and video.year != None: + return True + + elif isinstance(video, Episode): + if video.series != None and video.season != None and video.episode != None and type(video.episode) != list: + return True + + return False + def main(): - path = '/mnt/mainframe/' - # hash_path = input('Hash: ') - # path += hash_path - - # t = tvdb_api.Tvdb() - - # hashList = organize_files(episodePath) - # pprint(hashList) + path = '/mnt/rescue/' videos = scan_folder(path) + + scout = [] + civilan = [] for video in videos: - pprint(video) + girl = pickforgirlscouts(video) + if girl: + scout.append(girl) + else: + civilan.append(girl) + + click.echo('%s scouts%s collected / %s civilans%s' % ( + click.style(str(len(scout)), bold=True, fg='green' if scout else None), + 's' if len(scout) > 1 else '', + click.style(str(len(civilan)), bold=True, fg='red' if civilan else None), + 's' if len(civilan) > 1 else '', + )) if __name__ == '__main__': main() - - # for hash in files: - # hashIndex = [files[hash]] - # for hashItems in hashIndex: - # for file in hashItems: - # print(file.series) - - # saved_subtitles = [] - # for subtitle in files: - # # check content - # if subtitle.name is None: - # logging.error('Skipping subtitle %r: no content', subtitle) - # continue - - # # check language - # if subtitle.language in set(s.language for s in saved_subtitles): - # logging.debug('Skipping subtitle %r: language already saved', subtitle) - # continue - - # # create subtitle path - # subtitle_path = get_subtitle_path(video.name, None if single else subtitle.language) - # if directory is not None: - # subtitle_path = os.path.join(directory, os.path.split(subtitle_path)[1]) - - # # save content as is or in the specified encoding - # logging.info('Saving %r to %r', subtitle, subtitle_path) - # if encoding is None: - # with io.open(subtitle_path, 'wb') as f: - # f.write(subtitle.content) - # else: - # with io.open(subtitle_path, 'w', encoding=encoding) as f: - # f.write(subtitle.text) - # saved_subtitles.append(subtitle) - - # # check single - # if single: - # break - - # return saved_subtitles From 5ed1019a4670d0af442e1195c184ab1a864920c3 Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Sun, 16 Sep 2018 21:47:15 +0200 Subject: [PATCH 37/60] Redid hash function. Hashids only supports ints and our hash text is string. Moved over to hashlib and md5 function. --- src/core.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/core.py b/src/core.py index fe2ab01..26d7239 100755 --- a/src/core.py +++ b/src/core.py @@ -7,7 +7,7 @@ from guessit import guessit from babelfish import Language, LanguageReverseError -from hashids import Hashids +import hashlib import os, errno import logging import tvdb_api @@ -72,11 +72,11 @@ def scan_video(path): video.size = os.path.getsize(path) # hash of name - hashids = Hashids(min_length=16) - if isinstance(v, Episode): - videoHash = hashids.encode(''.join(self.title, self.year or '')) - elif isinstance(v, Movie): - videoHash = hashids.encode(''.join(self.series, self.season, self.episode)) + if isinstance(video, Movie): + hash_str = ''.join([video.title, str(video.year) or '']) + elif isinstance(video, Episode): + hash_str = ''.join([video.series, str(video.season), str(video.episode)]) + videoHash = hashlib.md5(hash_str.encode()).hexdigest() video.hash = videoHash return video From 96ca355c50309507f3d43c4d264f9b44d3781840 Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Sun, 16 Sep 2018 22:26:07 +0200 Subject: [PATCH 38/60] Fixed naming error with civilian and girl. Also prints total found videos. --- src/core.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/core.py b/src/core.py index 3c4ae44..7cb286c 100755 --- a/src/core.py +++ b/src/core.py @@ -287,21 +287,25 @@ def main(): videos = scan_folder(path) scout = [] - civilan = [] + civilian = [] for video in videos: - girl = pickforgirlscouts(video) - if girl: - scout.append(girl) + sortingHat = pickforgirlscouts(video) + if sortingHat: + scout.append(video) else: - civilan.append(girl) + civilian.append(video) - click.echo('%s scouts%s collected / %s civilans%s' % ( + click.echo('%s scout%s collected / %s civilan%s / %s candidate%s' % ( click.style(str(len(scout)), bold=True, fg='green' if scout else None), 's' if len(scout) > 1 else '', - click.style(str(len(civilan)), bold=True, fg='red' if civilan else None), - 's' if len(civilan) > 1 else '', + click.style(str(len(civilian)), bold=True, fg='red' if civilian else None), + 's' if len(civilian) > 1 else '', + click.style(str(len(videos)), bold=True, fg='blue' if videos else None), + 's' if len(videos) > 1 else '' )) + for video in civilian: + print(video) if __name__ == '__main__': main() From 2108f2f25a20656680b520556e0025e0d25ec69e Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Sun, 16 Sep 2018 23:01:49 +0200 Subject: [PATCH 39/60] Removed directory path from function call. --- src/core.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/core.py b/src/core.py index 7cb286c..e37917b 100755 --- a/src/core.py +++ b/src/core.py @@ -256,8 +256,7 @@ def scan_folder(path): # Iterates over our scanned videos with click.progressbar(scanned_videos, label='Parsing videos') as bar: for v in bar: - v.subtitle_languages |= set(search_external_subtitles(v.name, - directory=path).values()) + v.subtitle_languages |= set(search_external_subtitles(v.name).values()) refine(v) videos.append(v) From 1ff34630ac8432bcd62a9172890060399bc9194a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sun, 16 Sep 2018 23:05:09 +0200 Subject: [PATCH 40/60] Renamed subtitle_languages to subtitles to save the subtitles and not only the subtitle languages available. --- src/core.py | 4 ++-- src/video.py | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/core.py b/src/core.py index e37917b..ca8856e 100755 --- a/src/core.py +++ b/src/core.py @@ -239,7 +239,7 @@ def scan_folder(path): logging.exception('Unexpected error while collecting non-existing path %s', path) errored_paths.append(path) - video.subtitle_languages |= set(search_external_subtitles(video.name, directory=path).values()) + video.subtitles |= set(search_external_subtitles(video.name, directory=path)) refine(video) videos.append(video) @@ -256,7 +256,7 @@ def scan_folder(path): # Iterates over our scanned videos with click.progressbar(scanned_videos, label='Parsing videos') as bar: for v in bar: - v.subtitle_languages |= set(search_external_subtitles(v.name).values()) + v.subtitles |= set(search_external_subtitles(v.name)) refine(v) videos.append(v) diff --git a/src/video.py b/src/video.py index d0c09e3..edfb777 100644 --- a/src/video.py +++ b/src/video.py @@ -30,10 +30,10 @@ class Video(object): :param str imdb_id: IMDb id of the video. :param dict name_hash: hashes of the video file by provider names. :param int size: size of the video file in bytes. - :param set subtitle_languages: existing subtitle languages. + :param set subtitles: existing subtitle languages. """ def __init__(self, name, hash=None, size=None, format=None, release_group=None, resolution=None, video_codec=None, audio_codec=None, - imdb_id=None, subtitle_languages=None): + imdb_id=None, subtitles=None): #: Name or path of the video self.name = name @@ -62,7 +62,7 @@ class Video(object): self.imdb_id = imdb_id #: Existing subtitle languages - self.subtitle_languages = subtitle_languages or set() + self.subtitles = subtitles or set() @property def exists(self): @@ -168,12 +168,11 @@ class Episode(Video): def __repr__(self): if self.year is None: return '<%s [%r, %dx%s]>' % (self.__class__.__name__, self.series, self.season, self.episode) - if self.subtitle_languages is not None: - return '<%s [%r, %dx%s] %s>' % (self.__class__.__name__, self.series, self.season, self.episode, self.subtitle_languages) + if self.subtitles is not None: + return '<%s [%r, %dx%s] %s>' % (self.__class__.__name__, self.series, self.season, self.episode, self.subtitles) return '<%s [%r, %d, %dx%d]>' % (self.__class__.__name__, self.series, self.year, self.season, self.episode) - class Movie(Video): """Movie :class:`Video`. :param str title: title of the movie. @@ -210,6 +209,7 @@ class Movie(Video): return '<%s [%r]>' % (self.__class__.__name__, self.title) return '<%s [%r, %d]>' % (self.__class__.__name__, self.title, self.year) + ''' class Episode(): """Episode :class:`Video`. From 3e2faadc2ffc42748441ebdcc0f3aa0c1d3c5f15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sun, 16 Sep 2018 23:26:06 +0200 Subject: [PATCH 41/60] Video gets a new parameter embeded_subtitles to represent the subtitle files found in the mkv container. --- src/utils.py | 18 +++++++++--------- src/video.py | 5 ++++- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/utils.py b/src/utils.py index 91caf4e..b9534fe 100644 --- a/src/utils.py +++ b/src/utils.py @@ -47,7 +47,7 @@ def refine(video, embedded_subtitles=True, **kwargs): * :attr:`~subliminal.video.Video.resolution` * :attr:`~subliminal.video.Video.video_codec` * :attr:`~subliminal.video.Video.audio_codec` - * :attr:`~subliminal.video.Video.subtitle_languages` + * :attr:`~subliminal.video.Video.embeded_subtitles` :param bool embedded_subtitles: search for embedded subtitles. """ # skip non existing videos @@ -111,24 +111,24 @@ def refine(video, embedded_subtitles=True, **kwargs): # subtitle tracks if mkv.subtitle_tracks: if embedded_subtitles: - embedded_subtitle_languages = set() + embeded_subtitles = set() for st in mkv.subtitle_tracks: if st.language: try: - embedded_subtitle_languages.add(Language.fromalpha3b(st.language)) + embeded_subtitles.add(Language.fromalpha3b(st.language)) except BabelfishError: logging.error('Embedded subtitle track language %r is not a valid language', st.language) - embedded_subtitle_languages.add(Language('und')) + embeded_subtitles.add(Language('und')) elif st.name: try: - embedded_subtitle_languages.add(Language.fromname(st.name)) + embeded_subtitles.add(Language.fromname(st.name)) except BabelfishError: logging.debug('Embedded subtitle track name %r is not a valid language', st.name) - embedded_subtitle_languages.add(Language('und')) + embeded_subtitles.add(Language('und')) else: - embedded_subtitle_languages.add(Language('und')) - logging.debug('Found embedded subtitle %r', embedded_subtitle_languages) - video.subtitle_languages |= embedded_subtitle_languages + embeded_subtitles.add(Language('und')) + logging.debug('Found embedded subtitle %r', embeded_subtitles) + video.embeded_subtitles |= embeded_subtitles else: logging.debug('MKV has no subtitle track') else: diff --git a/src/video.py b/src/video.py index edfb777..d78e614 100644 --- a/src/video.py +++ b/src/video.py @@ -33,7 +33,7 @@ class Video(object): :param set subtitles: existing subtitle languages. """ def __init__(self, name, hash=None, size=None, format=None, release_group=None, resolution=None, video_codec=None, audio_codec=None, - imdb_id=None, subtitles=None): + imdb_id=None, subtitles=None, embeded_subtitles=None): #: Name or path of the video self.name = name @@ -64,6 +64,9 @@ class Video(object): #: Existing subtitle languages self.subtitles = subtitles or set() + #: Embeded subtitle languages + self.embeded_subtitles = embeded_subtitles or set() + @property def exists(self): """Test whether the video exists""" From 16afdb4cd8fdc8ca4473118600f2acc1ceef2d33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sun, 16 Sep 2018 23:38:19 +0200 Subject: [PATCH 42/60] Video has a new attribute home that is the optimal parent folder for this element. --- src/core.py | 7 +++++++ src/video.py | 8 ++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/core.py b/src/core.py index ca8856e..c46637d 100755 --- a/src/core.py +++ b/src/core.py @@ -269,13 +269,20 @@ def scan_folder(path): return videos +def movingToCollege(video, home_path): + video.home = home_path + def pickforgirlscouts(video): if isinstance(video, Movie): if video.title != None and video.year != None: + home_path = '{} ({})'.format(video.title, video.year) + movingToCollege(video, home_path) return True elif isinstance(video, Episode): if video.series != None and video.season != None and video.episode != None and type(video.episode) != list: + home_path = '{} S{:02d}E{:02d}'.format(video.series, video.season, video.episode) + movingToCollege(video, home_path) return True return False diff --git a/src/video.py b/src/video.py index d78e614..fb9a918 100644 --- a/src/video.py +++ b/src/video.py @@ -23,8 +23,8 @@ class Video(object): Represent a video, existing or not. :param str name: name or path of the video. :param str format: format of the video (HDTV, WEB-DL, BluRay, ...). - :param str release_group: release group of the video. - :param str resolution: resolution of the video stream (480p, 720p, 1080p or 1080i). + :param str home: home is the optimal parent folder. + :param str resolution: resolution of the video stream (480p, 720p, 1080p or 1080i, 4K). :param str video_codec: codec of the video stream. :param str audio_codec: codec of the main audio stream. :param str imdb_id: IMDb id of the video. @@ -32,7 +32,7 @@ class Video(object): :param int size: size of the video file in bytes. :param set subtitles: existing subtitle languages. """ - def __init__(self, name, hash=None, size=None, format=None, release_group=None, resolution=None, video_codec=None, audio_codec=None, + def __init__(self, name, hash=None, size=None, format=None, home=None, resolution=None, video_codec=None, audio_codec=None, imdb_id=None, subtitles=None, embeded_subtitles=None): #: Name or path of the video self.name = name @@ -47,7 +47,7 @@ class Video(object): self.format = format #: Release group of the video - self.release_group = release_group + self.home = home #: Resolution of the video stream (480p, 720p, 1080p or 1080i) self.resolution = resolution From 1fc62479be4143ee92cbd6454b553bbb8c57ebda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sun, 16 Sep 2018 23:46:39 +0200 Subject: [PATCH 43/60] Conflict resolved by re-adding release_group --- src/video.py | 145 +++------------------------------------------------ 1 file changed, 7 insertions(+), 138 deletions(-) diff --git a/src/video.py b/src/video.py index fb9a918..3c46eda 100644 --- a/src/video.py +++ b/src/video.py @@ -23,17 +23,17 @@ class Video(object): Represent a video, existing or not. :param str name: name or path of the video. :param str format: format of the video (HDTV, WEB-DL, BluRay, ...). - :param str home: home is the optimal parent folder. + :param str release_group: release group of the video. :param str resolution: resolution of the video stream (480p, 720p, 1080p or 1080i, 4K). :param str video_codec: codec of the video stream. :param str audio_codec: codec of the main audio stream. - :param str imdb_id: IMDb id of the video. + :param str home: optimal parent folder. :param dict name_hash: hashes of the video file by provider names. :param int size: size of the video file in bytes. :param set subtitles: existing subtitle languages. """ - def __init__(self, name, hash=None, size=None, format=None, home=None, resolution=None, video_codec=None, audio_codec=None, - imdb_id=None, subtitles=None, embeded_subtitles=None): + def __init__(self, name, hash=None, size=None, format=None, release_group=None, resolution=None, video_codec=None, audio_codec=None, + home=None, subtitles=None, embeded_subtitles=None): #: Name or path of the video self.name = name @@ -58,8 +58,8 @@ class Video(object): #: Codec of the main audio stream self.audio_codec = audio_codec - #: IMDb id of the video - self.imdb_id = imdb_id + #: optimal home path; parent folder. + self.home = home #: Existing subtitle languages self.subtitles = subtitles or set() @@ -121,7 +121,7 @@ class Episode(Video): :param \*\*kwargs: additional parameters for the :class:`Video` constructor. """ def __init__(self, name, series, season, episode, title=None, year=None, original_series=True, tvdb_id=None, - series_tvdb_id=None, series_imdb_id=None, **kwargs): + series_tvdb_id=None, **kwargs): super(Episode, self).__init__(name, **kwargs) #: Series of the episode @@ -148,9 +148,6 @@ class Episode(Video): #: TVDB id of the series self.series_tvdb_id = series_tvdb_id - #: IMDb id of the series - self.series_imdb_id = series_imdb_id - @classmethod def fromguess(cls, name, guess): if guess['type'] != 'episode': @@ -212,131 +209,3 @@ class Movie(Video): return '<%s [%r]>' % (self.__class__.__name__, self.title) return '<%s [%r, %d]>' % (self.__class__.__name__, self.title, self.year) - -''' -class Episode(): - """Episode :class:`Video`. - :param str series: series of the episode. - :param int season: season number of the episode. - :param int episode: episode number of the episode. - :param str title: title of the episode. - :param int year: year of the series. - :param bool original_series: whether the series is the first with this name. - :param int tvdb_id: TVDB id of the episode. - :param \*\*kwargs: additional parameters for the :class:`Video` constructor. - """ - def __init__(self, name, parent_path, series, season, episode, year=None, original_series=True, tvdb_id=None, - series_tvdb_id=None, series_imdb_id=None, release_group=None, video_codec=None, container=None, - format=None, screen_size=None, **kwargs): - super(Episode, self).__init__() - - self.name = name - - self.parent_path = parent_path - - #: Series of the episode - self.series = series - - #: Season number of the episode - self.season = season - - #: Episode number of the episode - self.episode = episode - - #: Year of series - self.year = year - - #: The series is the first with this name - self.original_series = original_series - - #: TVDB id of the episode - self.tvdb_id = tvdb_id - - #: TVDB id of the series - self.series_tvdb_id = series_tvdb_id - - #: IMDb id of the series - self.series_imdb_id = series_imdb_id - - # The release group of the episode - self.release_group = release_group - - # The video vodec of the series - self.video_codec = video_codec - - # The Video container of the episode - self.container = container - - # The Video format of the episode - self.format = format - - # The Video screen_size of the episode - self.screen_size = screen_size - - @classmethod - def fromguess(cls, name, parent_path, guess): - if guess['type'] != 'episode': - raise ValueError('The guess must be an episode guess') - - if 'title' not in guess or 'episode' not in guess: - raise ValueError('Insufficient data to process the guess') - - return cls(name, parent_path, guess['title'], guess.get('season', 1), guess['episode'], - year=guess.get('year'), original_series='year' not in guess, release_group=guess.get('release_group'), - video_codec=guess.get('video_codec'), audio_codec=guess.get('audio_codec'), container=guess.get('container'), - format=guess.get('format'), screen_size=guess.get('screen_size')) - - @classmethod - def fromname(cls, name): - return cls.fromguess(name, guessit(name, {'type': 'episode'})) - - def __hash__(self): - return hashlib.md5("b'{}'".format(str(self.series) + str(self.season) + str(self.episode)).encode()).hexdigest() - - # THE EP NUMBER IS CONVERTED TO STRING AS A QUICK FIX FOR MULTIPLE NUMBERS IN ONE - def __repr__(self): - if self.year is None: - return '<%s [%r, %sx%s]>' % (self.__class__.__name__, self.series, self.season, str(self.episode)) - - return '<%s [%r, %d, %sx%s]>' % (self.__class__.__name__, self.series, self.year, self.season, str(self.episode)) - - - -class Movie(): - """Movie :class:`Video`. - :param str title: title of the movie. - :param int year: year of the movie. - :param \*\*kwargs: additional parameters for the :class:`Video` constructor. - """ - def __init__(self, name, title, year=None, format=None, **kwargs): - super(Movie, self).__init__() - - #: Title of the movie - self.title = title - - #: Year of the movie - self.year = year - self.format = format - - @classmethod - def fromguess(cls, name, guess): - if guess['type'] != 'movie': - raise ValueError('The guess must be a movie guess') - - if 'title' not in guess: - raise ValueError('Insufficient data to process the guess') - - return cls(name, guess['title'], format=guess.get('format'), release_group=guess.get('release_group'), - resolution=guess.get('screen_size'), video_codec=guess.get('video_codec'), - audio_codec=guess.get('audio_codec'), year=guess.get('year')) - - @classmethod - def fromname(cls, name): - return cls.fromguess(name, guessit(name, {'type': 'movie'})) - - def __repr__(self): - if self.year is None: - return '<%s [%r]>' % (self.__class__.__name__, self.title) - - return '<%s [%r, %d]>' % (self.__class__.__name__, self.title, self.year) -''' From 916ce45fecb6d2b9edbed0a408ea1a16b7d85d0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Thu, 20 Sep 2018 23:24:17 +0200 Subject: [PATCH 44/60] If any error return false when picking for girlscouts --- src/core.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/core.py b/src/core.py index c46637d..7d4c140 100755 --- a/src/core.py +++ b/src/core.py @@ -133,7 +133,6 @@ def scan_videos(path): # scan for videos for filename in filenames: - # filter on videos and archives if not (filename.endswith(VIDEO_EXTENSIONS)): logging.debug('Skipping non-video file %s', filename) continue @@ -146,7 +145,6 @@ def scan_videos(path): # reconstruct the file path filepath = os.path.join(dirpath, filename) - # skip links if os.path.islink(filepath): logging.debug('Skipping link %r in %r', filename, dirpath) continue @@ -270,20 +268,27 @@ def scan_folder(path): return videos def movingToCollege(video, home_path): - video.home = home_path + video.home = path.join(home_path) def pickforgirlscouts(video): if isinstance(video, Movie): if video.title != None and video.year != None: home_path = '{} ({})'.format(video.title, video.year) - movingToCollege(video, home_path) - return True + try: + movingToCollege(video, home_path) + return True + except: + return False elif isinstance(video, Episode): if video.series != None and video.season != None and video.episode != None and type(video.episode) != list: - home_path = '{} S{:02d}E{:02d}'.format(video.series, video.season, video.episode) - movingToCollege(video, home_path) - return True + # Handle the list problems + home_path = '{} S{:02d}E{:02d}'.format(str(video.series), str(video.season), str(video.episode)) + try: + movingToCollege(video, home_path) + return True + except: + return False return False @@ -295,8 +300,7 @@ def main(): scout = [] civilian = [] for video in videos: - sortingHat = pickforgirlscouts(video) - if sortingHat: + if pickforgirlscouts(video): scout.append(video) else: civilian.append(video) From 4b09d6cd2ce6ba02fb5f15d9f204acb8c2afc4cc Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Thu, 20 Sep 2018 23:25:47 +0200 Subject: [PATCH 45/60] Titlecase the homepath and try except creating hash for filename. --- src/core.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/src/core.py b/src/core.py index c46637d..42bd89f 100755 --- a/src/core.py +++ b/src/core.py @@ -13,6 +13,7 @@ import logging import tvdb_api import click from pprint import pprint +from titlecase import titlecase import env_variables as env @@ -72,12 +73,15 @@ def scan_video(path): video.size = os.path.getsize(path) # hash of name - if isinstance(video, Movie): - hash_str = ''.join([video.title, str(video.year) or '']) - elif isinstance(video, Episode): - hash_str = ''.join([video.series, str(video.season), str(video.episode)]) - videoHash = hashlib.md5(hash_str.encode()).hexdigest() - video.hash = videoHash + try: + if isinstance(video, Movie): + hash_str = ''.join([video.title, str(video.year) or '']) + elif isinstance(video, Episode): + hash_str = ''.join([video.series, str(video.season), str(video.episode)]) + videoHash = hashlib.md5(hash_str.encode()).hexdigest() + video.hash = videoHash + except: + print(video) return video @@ -96,7 +100,6 @@ def scan_subtitle(path): return subtitle - def scan_videos(path): """Scan `path` for videos and their subtitles. @@ -118,7 +121,7 @@ def scan_videos(path): # setup progress bar path_children = 0 for _ in os.walk(path): path_children += 1 - with click.progressbar(length=path_children, show_pos=True, label='Searching folders for videos') as bar: + with click.progressbar(length=path_children, show_pos=True, label='Collecting videos') as bar: # walk the path videos = [] @@ -270,7 +273,7 @@ def scan_folder(path): return videos def movingToCollege(video, home_path): - video.home = home_path + video.home = titlecase(home_path) def pickforgirlscouts(video): if isinstance(video, Movie): @@ -281,14 +284,14 @@ def pickforgirlscouts(video): elif isinstance(video, Episode): if video.series != None and video.season != None and video.episode != None and type(video.episode) != list: - home_path = '{} S{:02d}E{:02d}'.format(video.series, video.season, video.episode) + home_path = '{} S{}E{}'.format(str(video.series), str(video.season), str(video.episode)) movingToCollege(video, home_path) return True return False def main(): - path = '/mnt/rescue/' + path = '/mnt/mainframe/' videos = scan_folder(path) @@ -310,8 +313,8 @@ def main(): 's' if len(videos) > 1 else '' )) - for video in civilian: - print(video) + for video in scout: + print('{} lives: {}'.format(video, video.home)) if __name__ == '__main__': main() From 8fab27a95f0a197335669ee835075d9d5c5fca38 Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Thu, 20 Sep 2018 23:28:47 +0200 Subject: [PATCH 46/60] And the other conflict --- src/core.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/core.py b/src/core.py index 011a56f..9cf7c11 100755 --- a/src/core.py +++ b/src/core.py @@ -285,11 +285,6 @@ def pickforgirlscouts(video): elif isinstance(video, Episode): if video.series != None and video.season != None and video.episode != None and type(video.episode) != list: -<<<<<<< HEAD - home_path = '{} S{}E{}'.format(str(video.series), str(video.season), str(video.episode)) - movingToCollege(video, home_path) - return True -======= # Handle the list problems home_path = '{} S{:02d}E{:02d}'.format(str(video.series), str(video.season), str(video.episode)) try: @@ -297,7 +292,6 @@ def pickforgirlscouts(video): return True except: return False ->>>>>>> 916ce45fecb6d2b9edbed0a408ea1a16b7d85d0b return False From 379d789adc3b18e5fb712dee5eb02dfce4cb76a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sun, 23 Sep 2018 16:28:44 +0200 Subject: [PATCH 47/60] Disbled hashing of filenames for now --- src/core.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/core.py b/src/core.py index 9cf7c11..8d06822 100755 --- a/src/core.py +++ b/src/core.py @@ -73,15 +73,17 @@ def scan_video(path): video.size = os.path.getsize(path) # hash of name - try: - if isinstance(video, Movie): - hash_str = ''.join([video.title, str(video.year) or '']) - elif isinstance(video, Episode): - hash_str = ''.join([video.series, str(video.season), str(video.episode)]) - videoHash = hashlib.md5(hash_str.encode()).hexdigest() - video.hash = videoHash - except: - print(video) + # if isinstance(video, Movie): + # if type(video.title) is str and type(video.year) is int: + # home_path = '{} ({})'.format(video.title, video.year) + # hash_str = ''.join([video.title, str(video.year) or '']) + # elif isinstance(video, Episode): + # if type(video.series) is str and type(video.season) is int and type(video.episode) is int: + # home_path = '{} ({})'.format(video.title, video.year) + # hash_str = ''.join([video.series, str(video.season), str(video.episode)]) + # video.hash = hashlib.md5(hash_str.encode()).hexdigest() + # except: + # print(video) return video From a106a35d42219a04340ddb362ce93cad87540a77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sun, 23 Sep 2018 16:29:41 +0200 Subject: [PATCH 48/60] Moved checks for sufficient info in order to move series or movie to the class. Homelocation in now a class function for each movie and series class. --- src/core.py | 25 ++++--------------------- src/video.py | 23 +++++++++++++++++++++++ 2 files changed, 27 insertions(+), 21 deletions(-) diff --git a/src/core.py b/src/core.py index 8d06822..387a8b5 100755 --- a/src/core.py +++ b/src/core.py @@ -272,28 +272,11 @@ def scan_folder(path): return videos -def movingToCollege(video, home_path): - video.home = titlecase(home_path) - def pickforgirlscouts(video): - if isinstance(video, Movie): - if video.title != None and video.year != None: - home_path = '{} ({})'.format(video.title, video.year) - try: - movingToCollege(video, home_path) - return True - except: - return False - - elif isinstance(video, Episode): - if video.series != None and video.season != None and video.episode != None and type(video.episode) != list: - # Handle the list problems - home_path = '{} S{:02d}E{:02d}'.format(str(video.series), str(video.season), str(video.episode)) - try: - movingToCollege(video, home_path) - return True - except: - return False + if video.sufficientInfo(): + video.moveLocation() + print(video.home) + return True return False diff --git a/src/video.py b/src/video.py index 3c46eda..c75b297 100644 --- a/src/video.py +++ b/src/video.py @@ -7,6 +7,7 @@ from guessit import guessit import os +from titlecase import titlecase import hashlib, tvdb_api #: Video extensions @@ -165,6 +166,17 @@ class Episode(Video): def fromname(cls, name): return cls.fromguess(name, guessit(name, {'type': 'episode'})) + @classmethod + def sufficientInfo(self): + return None not in [self.series, self.season, self.episode] and list not in [type(self.series), type(self.episode)] + + @classmethod + def moveLocation(self): + series = titlecase(self.series) + grandParent = '{}/{} {:02d}'.format(series, series, self.season) + parent = '{} S{:02d}E{:02d}'.format(series, self.season, self.episode) + self.home = os.path.join(grandParent, parent, self.name) + def __repr__(self): if self.year is None: return '<%s [%r, %dx%s]>' % (self.__class__.__name__, self.series, self.season, self.episode) @@ -204,6 +216,17 @@ class Movie(Video): def fromname(cls, name): return cls.fromguess(name, guessit(name, {'type': 'movie'})) + @classmethod + def sufficientInfo(self): + return None not in [self.title, self.year] + + @classmethod + def moveLocation(self): + title = titlecase(self.title) + parent = '{} ({})'.format(title, self.year) + self.home = os.path.join(parent, self.name) + + def __repr__(self): if self.year is None: return '<%s [%r]>' % (self.__class__.__name__, self.title) From bfab59d49c64e6a184f13db7da483cb4a776ac21 Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Sun, 23 Sep 2018 17:26:43 +0200 Subject: [PATCH 49/60] Methods should not have been tagged with classmethods and now the basename of class name is attributed to the final home path. --- src/video.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/video.py b/src/video.py index c75b297..a761d06 100644 --- a/src/video.py +++ b/src/video.py @@ -48,7 +48,7 @@ class Video(object): self.format = format #: Release group of the video - self.home = home + self.release_group = release_group #: Resolution of the video stream (480p, 720p, 1080p or 1080i) self.resolution = resolution @@ -166,11 +166,9 @@ class Episode(Video): def fromname(cls, name): return cls.fromguess(name, guessit(name, {'type': 'episode'})) - @classmethod def sufficientInfo(self): return None not in [self.series, self.season, self.episode] and list not in [type(self.series), type(self.episode)] - @classmethod def moveLocation(self): series = titlecase(self.series) grandParent = '{}/{} {:02d}'.format(series, series, self.season) @@ -216,16 +214,15 @@ class Movie(Video): def fromname(cls, name): return cls.fromguess(name, guessit(name, {'type': 'movie'})) - @classmethod def sufficientInfo(self): - return None not in [self.title, self.year] - - @classmethod + t = hasattr(self, "title") + y = hasattr(self, "year") + return None not in [t, y] + def moveLocation(self): title = titlecase(self.title) parent = '{} ({})'.format(title, self.year) - self.home = os.path.join(parent, self.name) - + self.home = os.path.join(parent, os.path.basename(self.name)) def __repr__(self): if self.year is None: From 81a53465e7cbde846601a41589b84fa939241d97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sun, 23 Sep 2018 17:42:42 +0200 Subject: [PATCH 50/60] If a movie or series does not have sufficient info it is logged before returning value. --- src/video.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/src/video.py b/src/video.py index a761d06..141cd1e 100644 --- a/src/video.py +++ b/src/video.py @@ -167,7 +167,19 @@ class Episode(Video): return cls.fromguess(name, guessit(name, {'type': 'episode'})) def sufficientInfo(self): - return None not in [self.series, self.season, self.episode] and list not in [type(self.series), type(self.episode)] + ser = hasattr(self, 'series') + sea = hasattr(self, 'season') + ep = hasattr(self, 'episode') + + if False in [ser, sea, ep]: + logger.error('{}, {} or {} found to have none value, manual correction required'.format(self.series, self.season, self.episode)) + return False + + if list in [type(self.series), type(self.season), type(self.episode)]: + logger.error('{}, {} or {} found to have list values, manual correction required'.format(self.series, self.season, self.episode)) + return False + + return True def moveLocation(self): series = titlecase(self.series) @@ -217,7 +229,15 @@ class Movie(Video): def sufficientInfo(self): t = hasattr(self, "title") y = hasattr(self, "year") - return None not in [t, y] + + if None in [t, y]: + logger.error('{} or {} found to have none value, manual correction required'.format(self.title, self.year)) + return False + if list in [type(self.title), type(self.year)]: + logger.error('{} or {} found to have list value, manual correction required'.format(self.title, self.year)) + return False + + return True def moveLocation(self): title = titlecase(self.title) From f92a3fc2a2ecb38d2880884f1df782c683dd420f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sun, 23 Sep 2018 18:00:24 +0200 Subject: [PATCH 51/60] Added custom logger instance to core and import in video. --- src/core.py | 37 ++++++++++++++++++++++++------------- src/video.py | 5 ++++- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/src/core.py b/src/core.py index 387a8b5..b1b1376 100755 --- a/src/core.py +++ b/src/core.py @@ -22,6 +22,17 @@ from subtitle import SUBTITLE_EXTENSIONS, Subtitle, get_subtitle_path from utils import sanitize, refine logging.basicConfig(filename=env.logfile, level=logging.INFO) +logger = logging.getLogger('seasonedParser_core') +fh = logging.FileHandler(env.logfile) +fh.setLevel(logging.INFO) +ch = logging.StreamHandler() +ch.setLevel(logging.ERROR) +formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +fh.setFormatter(formatter) +ch.setFormatter(formatter) + +logger.addHandler(fh) +logger.addHandler(ch) def search_external_subtitles(path, directory=None): @@ -40,10 +51,10 @@ def search_external_subtitles(path, directory=None): try: language = Language.fromietf(language_code) except (ValueError, LanguageReverseError): - logging.error('Cannot parse language code %r', language_code) + logger.error('Cannot parse language code %r', language_code) subtitles[p] = language - logging.debug('Found subtitles %r', subtitles) + logger.debug('Found subtitles %r', subtitles) return subtitles @@ -64,7 +75,7 @@ def scan_video(path): raise ValueError('%r is not a valid video extension' % os.path.splitext(path)[1]) dirpath, filename = os.path.split(path) - logging.info('Scanning video %r in %r', filename, dirpath) + logger.info('Scanning video %r in %r', filename, dirpath) # guess video = Video.fromguess(path, guessit(path)) @@ -93,7 +104,7 @@ def scan_subtitle(path): raise ValueError('Path does not exist') dirpath, filename = os.path.split(path) - logging.info('Scanning subtitle %r in %r', filename, dirpath) + logger.info('Scanning subtitle %r in %r', filename, dirpath) # guess parent_path = path.strip(filename) @@ -128,30 +139,30 @@ def scan_videos(path): # walk the path videos = [] for dirpath, dirnames, filenames in os.walk(path): - logging.debug('Walking directory %r', dirpath) + logger.debug('Walking directory %r', dirpath) # remove badly encoded and hidden dirnames for dirname in list(dirnames): if dirname.startswith('.'): - logging.debug('Skipping hidden dirname %r in %r', dirname, dirpath) + logger.debug('Skipping hidden dirname %r in %r', dirname, dirpath) dirnames.remove(dirname) # scan for videos for filename in filenames: if not (filename.endswith(VIDEO_EXTENSIONS)): - logging.debug('Skipping non-video file %s', filename) + logger.debug('Skipping non-video file %s', filename) continue # skip hidden files if filename.startswith('.'): - logging.debug('Skipping hidden filename %r in %r', filename, dirpath) + logger.debug('Skipping hidden filename %r in %r', filename, dirpath) continue # reconstruct the file path filepath = os.path.join(dirpath, filename) if os.path.islink(filepath): - logging.debug('Skipping link %r in %r', filename, dirpath) + logger.debug('Skipping link %r in %r', filename, dirpath) continue # scan @@ -159,7 +170,7 @@ def scan_videos(path): try: video = scan_video(filepath) except ValueError: # pragma: no cover - logging.exception('Error scanning video') + logger.exception('Error scanning video') continue else: # pragma: no cover raise ValueError('Unsupported file %r' % filename) @@ -232,14 +243,14 @@ def scan_folder(path): videos = [] ignored_videos = [] errored_paths = [] - logging.debug('Collecting path %s', path) + logger.debug('Collecting path %s', path) # non-existing if not os.path.exists(path): try: video = Video.fromname(path) except: - logging.exception('Unexpected error while collecting non-existing path %s', path) + logger.exception('Unexpected error while collecting non-existing path %s', path) errored_paths.append(path) video.subtitles |= set(search_external_subtitles(video.name, directory=path)) @@ -253,7 +264,7 @@ def scan_folder(path): try: scanned_videos = scan_videos(path) except: - logging.exception('Unexpected error while collecting directory path %s', path) + logger.exception('Unexpected error while collecting directory path %s', path) errored_paths.append(path) # Iterates over our scanned videos diff --git a/src/video.py b/src/video.py index 141cd1e..9019b91 100644 --- a/src/video.py +++ b/src/video.py @@ -7,9 +7,12 @@ from guessit import guessit import os +import logging from titlecase import titlecase import hashlib, tvdb_api +logger = logging.getLogger('seasonedParser_core') + #: Video extensions VIDEO_EXTENSIONS = ('.3g2', '.3gp', '.3gp2', '.3gpp', '.60d', '.ajp', '.asf', '.asx', '.avchd', '.avi', '.bik', '.bix', '.box', '.cam', '.dat', '.divx', '.dmf', '.dv', '.dvr-ms', '.evo', '.flc', '.fli', @@ -190,7 +193,7 @@ class Episode(Video): def __repr__(self): if self.year is None: return '<%s [%r, %dx%s]>' % (self.__class__.__name__, self.series, self.season, self.episode) - if self.subtitles is not None: + if self.subtitles is not (None or set): return '<%s [%r, %dx%s] %s>' % (self.__class__.__name__, self.series, self.season, self.episode, self.subtitles) return '<%s [%r, %d, %dx%d]>' % (self.__class__.__name__, self.series, self.year, self.season, self.episode) From abb9ddbf09ce65e1eda74171b1f8beb8300f9c31 Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Sun, 23 Sep 2018 18:01:55 +0200 Subject: [PATCH 52/60] Removed unused print and fixed show path and sanity check. --- src/core.py | 1 - src/video.py | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/core.py b/src/core.py index 387a8b5..3fd927c 100755 --- a/src/core.py +++ b/src/core.py @@ -275,7 +275,6 @@ def scan_folder(path): def pickforgirlscouts(video): if video.sufficientInfo(): video.moveLocation() - print(video.home) return True return False diff --git a/src/video.py b/src/video.py index 141cd1e..4eae332 100644 --- a/src/video.py +++ b/src/video.py @@ -183,14 +183,14 @@ class Episode(Video): def moveLocation(self): series = titlecase(self.series) - grandParent = '{}/{} {:02d}'.format(series, series, self.season) + grandParent = '{}/{} Season {:02d}'.format(series, series, self.season) parent = '{} S{:02d}E{:02d}'.format(series, self.season, self.episode) - self.home = os.path.join(grandParent, parent, self.name) + self.home = os.path.join(grandParent, parent, os.path.basename(self.name)) def __repr__(self): if self.year is None: return '<%s [%r, %dx%s]>' % (self.__class__.__name__, self.series, self.season, self.episode) - if self.subtitles is not None: + if self.subtitles is not None and len(self.subtitles) > 0: return '<%s [%r, %dx%s] %s>' % (self.__class__.__name__, self.series, self.season, self.episode, self.subtitles) return '<%s [%r, %d, %dx%d]>' % (self.__class__.__name__, self.series, self.year, self.season, self.episode) From 11f5c16336e4e5732dc085d52d2852c9d52a6e3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20Midb=C3=B8e?= Date: Sun, 23 Sep 2018 20:13:30 +0200 Subject: [PATCH 53/60] Added moviebase and showbase that represents the absolute base for the movie and show folders on disk. --- src/env_variables.py | 2 ++ src/video.py | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/env_variables.py b/src/env_variables.py index 4fd250a..0bb275a 100644 --- a/src/env_variables.py +++ b/src/env_variables.py @@ -1 +1,3 @@ logfile = 'conf/output.log' +MOVIEBASE = '/mnt/mainframe/movies' +SHOWBASE = '/mnt/mainframe/shows' \ No newline at end of file diff --git a/src/video.py b/src/video.py index baeae98..baf2348 100644 --- a/src/video.py +++ b/src/video.py @@ -11,6 +11,8 @@ import logging from titlecase import titlecase import hashlib, tvdb_api +import env_variables as env + logger = logging.getLogger('seasonedParser_core') #: Video extensions @@ -188,7 +190,7 @@ class Episode(Video): series = titlecase(self.series) grandParent = '{}/{} Season {:02d}'.format(series, series, self.season) parent = '{} S{:02d}E{:02d}'.format(series, self.season, self.episode) - self.home = os.path.join(grandParent, parent, os.path.basename(self.name)) + self.home = os.path.join(env.SHOWBASE, grandParent, parent, os.path.basename(self.name)) def __repr__(self): if self.year is None: @@ -245,7 +247,7 @@ class Movie(Video): def moveLocation(self): title = titlecase(self.title) parent = '{} ({})'.format(title, self.year) - self.home = os.path.join(parent, os.path.basename(self.name)) + self.home = os.path.join(env.MOVIEBASE, parent, os.path.basename(self.name)) def __repr__(self): if self.year is None: From 55f435109f0d298e945d13847cae73c8b8ce3979 Mon Sep 17 00:00:00 2001 From: KevinMidboe Date: Sun, 23 Sep 2018 20:37:26 +0200 Subject: [PATCH 54/60] Added langdetect for last resort when trying to find the subtitle language. It reads the first 1000 lines of the file and guesses the locale of the text. --- requirements.txt | 1 + src/core.py | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/requirements.txt b/requirements.txt index 79e1ed8..01e0530 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ tvdb_api==2.0 hashids==1.2.0 enzyme>=0.4.1 click>=6.7 +langdetect>=1.0.7 diff --git a/src/core.py b/src/core.py index 3b0521e..8457b54 100755 --- a/src/core.py +++ b/src/core.py @@ -14,6 +14,7 @@ import tvdb_api import click from pprint import pprint from titlecase import titlecase +import langdetect import env_variables as env @@ -49,6 +50,16 @@ def search_external_subtitles(path, directory=None): except (ValueError, LanguageReverseError): logger.error('Cannot parse language code %r', language_code) + f = open(p, 'r', encoding='ISO-8859-15') + + pattern = re.compile('[0-9:\,-<>]+') + # head = list(islice(f.read(), 10)) + filecontent = pattern.sub('', f.read()) + filecontent = filecontent[0:1000] + language = langdetect.detect(filecontent) + print(language) + f.close() + subtitles[p] = language logger.debug('Found subtitles %r', subtitles) From 4e055403d1a104278ad1630736d0b342fcfdbeb4 Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Sun, 23 Sep 2018 21:00:14 +0200 Subject: [PATCH 55/60] Imported re (regex lib) and fixed pathing issue when reading subtitle --- src/core.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core.py b/src/core.py index 8457b54..b7da6ca 100755 --- a/src/core.py +++ b/src/core.py @@ -10,6 +10,7 @@ from babelfish import Language, LanguageReverseError import hashlib import os, errno import logging +import re import tvdb_api import click from pprint import pprint @@ -50,7 +51,7 @@ def search_external_subtitles(path, directory=None): except (ValueError, LanguageReverseError): logger.error('Cannot parse language code %r', language_code) - f = open(p, 'r', encoding='ISO-8859-15') + f = open(os.path.join(dirpath, p), 'r', encoding='ISO-8859-15') pattern = re.compile('[0-9:\,-<>]+') # head = list(islice(f.read(), 10)) From b40f0f91ef672183d19bf7f0bbc6b5306ce68a04 Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Sun, 23 Sep 2018 21:26:57 +0200 Subject: [PATCH 56/60] Added subtitle_path function which finds the dirpath based on video sibling and subtitle name. --- src/core.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/core.py b/src/core.py index b7da6ca..014cdb5 100755 --- a/src/core.py +++ b/src/core.py @@ -58,7 +58,6 @@ def search_external_subtitles(path, directory=None): filecontent = pattern.sub('', f.read()) filecontent = filecontent[0:1000] language = langdetect.detect(filecontent) - print(language) f.close() subtitles[p] = language @@ -121,6 +120,10 @@ def scan_subtitle(path): return subtitle +def subtitle_path(sibling, subtitle): + parent_path = os.path.dirname(sibling) + return os.path.join(parent_path, subtitle) + def scan_videos(path): """Scan `path` for videos and their subtitles. From 03902eeecbab20d319e0cc47c11382a58e276a1c Mon Sep 17 00:00:00 2001 From: KevinMidboe Date: Sun, 23 Sep 2018 21:36:03 +0200 Subject: [PATCH 57/60] Created the big move home function which will create the parent folders and move all videos and subtitles. --- src/core.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/core.py b/src/core.py index 014cdb5..7a1aa5f 100755 --- a/src/core.py +++ b/src/core.py @@ -301,6 +301,20 @@ def pickforgirlscouts(video): return False +def moveHome(video): + dir = os.path.dirname(video.home) + + if not os.path.exists(dir): + logger.info('Creating directory {}'.format(dir)) + os.makedirs(dir) + + logger.info("Moving video file from: '{}' to: '{}'".format(video.name, video.home)) + shutil.move(video.name, video.home) + for sub in video.subtitles: + sub_home = subtitle_path(sub) + logger.info("Moving subtitle file from: '{}' to: '{}'".format(sub, sub_home)) + shutil.move(sub, sub_home) + def main(): path = '/mnt/mainframe/' @@ -324,7 +338,7 @@ def main(): )) for video in scout: - print('{} lives: {}'.format(video, video.home)) + moveHome(video) if __name__ == '__main__': main() From 66f7ea3e24b61e2719677e86252ade9e347737e5 Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Sun, 23 Sep 2018 22:38:26 +0200 Subject: [PATCH 58/60] Because we read subtitles from mkv containers a check for if subtitle file exists was added so not to try move a sub file within a mkv container. --- src/core.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/core.py b/src/core.py index 7a1aa5f..6eb7fb4 100755 --- a/src/core.py +++ b/src/core.py @@ -9,6 +9,7 @@ from guessit import guessit from babelfish import Language, LanguageReverseError import hashlib import os, errno +import shutil import logging import re import tvdb_api @@ -60,7 +61,7 @@ def search_external_subtitles(path, directory=None): language = langdetect.detect(filecontent) f.close() - subtitles[p] = language + subtitles[os.path.join(dirpath, p)] = language logger.debug('Found subtitles %r', subtitles) return subtitles @@ -122,7 +123,7 @@ def scan_subtitle(path): def subtitle_path(sibling, subtitle): parent_path = os.path.dirname(sibling) - return os.path.join(parent_path, subtitle) + return os.path.join(parent_path, os.path.basename(subtitle)) def scan_videos(path): """Scan `path` for videos and their subtitles. @@ -311,9 +312,12 @@ def moveHome(video): logger.info("Moving video file from: '{}' to: '{}'".format(video.name, video.home)) shutil.move(video.name, video.home) for sub in video.subtitles: - sub_home = subtitle_path(sub) - logger.info("Moving subtitle file from: '{}' to: '{}'".format(sub, sub_home)) - shutil.move(sub, sub_home) + if not os.path.isfile(sub): + continue + oldpath = sub + newpath = subtitle_path(video.home, sub) + logger.info("Moving subtitle file from: '{}' to: '{}'".format(oldpath, newpath)) + shutil.move(oldpath, newpath) def main(): path = '/mnt/mainframe/' From aeacd8a5b67986c560eb96b869d3a518c18b4105 Mon Sep 17 00:00:00 2001 From: KevinMidboe Date: Tue, 16 Oct 2018 23:06:45 +0200 Subject: [PATCH 59/60] Started cli for seasonedparser --- src/cli.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100755 src/cli.py diff --git a/src/cli.py b/src/cli.py new file mode 100755 index 0000000..074eb68 --- /dev/null +++ b/src/cli.py @@ -0,0 +1,39 @@ +#/usr/local/bin/python3 +import click +import os +import logging + +import env_variables as env + +logging.basicConfig(filename=env.logfile, level=logging.INFO) +logger = logging.getLogger('seasonedParser') +fh = logging.FileHandler(env.logfile) +fh.setLevel(logging.INFO) + +formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +fh.setFormatter(formatter) + +logger.addHandler(fh) + +def listPath(path): + if (os.path.isdir(path)): + print('Contents of path:') + print(os.listdir(path)) + + elif os.path.isfile(path): + print('File to parse:') + print(path) + + else: + print('Path does not exists') + +@click.command() +@click.argument('path') +@click.option('--greeting', '-g') +def main(path, greeting): + logger.info('Received cli variables: \n\t path: {}'.format(path)) + listPath(path) + + +if __name__ == '__main__': + main() From 6d67ce121423f48d36cbb51d3aa1e2aedacaa59c Mon Sep 17 00:00:00 2001 From: KevinMidboe Date: Tue, 16 Oct 2018 23:07:05 +0200 Subject: [PATCH 60/60] Cli roadmap --- README.md | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 79dd1da..1244e8c 100644 --- a/README.md +++ b/README.md @@ -26,4 +26,19 @@ There are many run commands for this, but here is a list of the current working ``` Here the first parameter is our move command, which in turn calls motherMover. The second parameter is what we want the filenames to be called. Notice the (num1..num2), this is to create a range for all the episodes we want to move. The last parameter is the path we want to move our content. - > This will be done automatically by the parser based on the info in the media items name, but it is nice to have a manual command. \ No newline at end of file + > This will be done automatically by the parser based on the info in the media items name, but it is nice to have a manual command. + + +## Cli + +Arguments +* Dry run with --dry +* Path variable +* daemon with -d option + * Still need the path variable + * Daemon sends confirmation and on missing asks tweetf or correction + +Functions +* Should ask for input when missing info, always when cli + +