69 Commits

Author SHA1 Message Date
Ali Parlakci
d685860c22 Update version 2018-07-26 12:25:54 +03:00
Ali Parlakci
dcf9f35273 Merge branch 'master' of https://github.com/aliparlakci/bulk-downloader-for-reddit 2018-07-26 12:25:50 +03:00
Ali Parlakci
7fdf03aa24 Added new line after 'GETTING POSTS' 2018-07-26 12:25:23 +03:00
Ali Parlakçı
25d61a4c78 Update README.md 2018-07-26 12:23:08 +03:00
Ali Parlakci
558eb107f4 Update changelog 2018-07-26 12:01:00 +03:00
Ali Parlakci
6e74630050 Typo fix 2018-07-26 11:59:29 +03:00
Ali Parlakci
2fd9248715 Added quit after finish option 2018-07-26 11:15:13 +03:00
Ali Parlakci
457b8cd21c Added remaining credits to log file 2018-07-26 11:08:37 +03:00
Ali Parlakci
e953456ead Merge branch 'master' of https://github.com/aliparlakci/bulk-downloader-for-reddit 2018-07-26 10:10:48 +03:00
Ali Parlakci
ed0564fba0 Improve verbose mode 2018-07-26 10:08:57 +03:00
Ali Parlakçı
5378555f74 Update COMPILE_FROM_SOURCE.md 2018-07-26 09:24:50 +03:00
Ali Parlakçı
95ef308915 Update COMPILE_FROM_SOURCE.md 2018-07-26 09:22:14 +03:00
Ali Parlakçı
436f867f2e Update COMPILE_FROM_SOURCE.md 2018-07-26 09:22:03 +03:00
Ali Parlakçı
91d71565cc Update script.py 2018-07-25 18:30:48 +03:00
Ali Parlakci
c7b7361ded Update version 2018-07-25 18:27:58 +03:00
Ali Parlakci
cd81a6c38b Update changelog 2018-07-25 13:53:33 +03:00
Ali Parlakçı
1623722138 Merge pull request #42 from aliparlakci/verboseMode
Added Verbose mode
2018-07-25 13:52:34 +03:00
Ali Parlakci
dad5669441 Typo fix 2018-07-25 13:50:11 +03:00
Ali Parlakci
35d54d1eb1 Stylize 2018-07-25 13:48:30 +03:00
Ali Parlakci
394b864d86 Updated FAQ 2018-07-25 13:48:02 +03:00
Ali Parlakci
837281c3c6 Added verbose mode 2018-07-25 13:40:06 +03:00
Ali Parlakci
e6b648d8b3 Update changelog 2018-07-25 12:25:39 +03:00
Ali Parlakci
cfaf2de7db Stylize the console output 2018-07-25 12:24:50 +03:00
Ali Parlakci
80546d7094 Update version 2018-07-25 11:36:58 +03:00
Ali Parlakci
139a81a0e7 Merge branch 'master' of https://github.com/aliparlakci/bulk-downloader-for-reddit 2018-07-25 09:29:48 +03:00
Ali Parlakci
9bb0a5da7f Added delays for imgur rate limit 2018-07-25 09:27:41 +03:00
Ali Parlakçı
6f2273f182 Add file name formatting doc 2018-07-24 23:41:02 +03:00
Ali Parlakci
b5d6165802 Update version 2018-07-24 22:13:38 +03:00
Ali Parlakci
b98815376f Bug fix 2018-07-24 22:13:11 +03:00
Ali Parlakci
d9586f99b8 Use else in try blocks 2018-07-24 22:11:12 +03:00
Ali Parlakci
76711892a2 Merge branch 'master' of https://github.com/aliparlakci/bulk-downloader-for-reddit 2018-07-24 22:10:28 +03:00
Ali Parlakci
bfea548eab Print credits in the same line 2018-07-24 22:10:19 +03:00
Ali Parlakçı
2e852db4c3 Typo fix 2018-07-24 19:45:37 +03:00
Ali Parlakci
8ac02e7aff Update version 2018-07-24 19:42:38 +03:00
Ali Parlakci
5eccf4dd3d Update changelog 2018-07-24 19:35:49 +03:00
Ali Parlakçı
7a68ff3efa Merge pull request #41 from aliparlakci/changePostNames
Add submitter to file names
2018-07-24 19:34:02 +03:00
Ali Parlakçı
3ea2e16b62 Merge branch 'master' into changePostNames 2018-07-24 19:33:38 +03:00
Ali Parlakci
fc6787aa28 Update changelog 2018-07-24 19:28:48 +03:00
Ali Parlakci
21533bb78c Improved exception handling 2018-07-24 19:27:52 +03:00
Ali Parlakci
1781ab8ffe Update changelog 2018-07-24 19:10:34 +03:00
Ali Parlakci
821383c465 Deleted # char from file names 2018-07-24 19:09:45 +03:00
Ali Parlakci
9d0fdc7521 Add OP's name first 2018-07-24 18:55:33 +03:00
Ali Parlakci
0387dd5243 Tweaked 'What it can do' 2018-07-24 14:16:46 +03:00
Ali Parlakci
93732b0367 Little refactoring 2018-07-24 13:17:37 +03:00
Ali Parlakci
400ce01918 Added older version support 2018-07-24 13:17:14 +03:00
Ali Parlakci
ccedac4bdc Add submitter to file name 2018-07-24 12:44:53 +03:00
Ali Parlakci
a6997898ce Update version 2018-07-23 23:37:27 +03:00
Ali Parlakci
61632c7143 Improve error handling 2018-07-23 23:33:11 +03:00
Ali Parlakçı
9bff3399a8 Typo fix 2018-07-23 23:19:29 +03:00
Ali Parlakçı
b00d185f67 Update changelog 2018-07-23 23:18:10 +03:00
Ali Parlakçı
7314e17125 Added erome support 2018-07-23 23:16:56 +03:00
Ali Parlakci
2d334d56bf remove exclude mode 2018-07-23 22:57:54 +03:00
Ali Parlakçı
974517928f Update README.md 2018-07-23 22:07:28 +03:00
Ali Parlakçı
bcae177b1e Split download function 2018-07-23 22:06:33 +03:00
Ali Parlakci
229def6578 Merge branch 'master' of https://github.com/aliparlakci/bulk-downloader-for-reddit 2018-07-23 18:49:39 +03:00
Ali Parlakci
59b0376d6e Added directions for frontpage 2018-07-23 18:49:28 +03:00
Ali Parlakçı
cf1dc7d08c Rename changelog section 2018-07-22 18:16:11 +03:00
Ali Parlakci
27532408c1 Update version 2018-07-22 18:06:46 +03:00
Ali Parlakci
32647beee9 Update changelog 2018-07-22 18:06:24 +03:00
Ali Parlakci
a67da461d2 Fixed the bug that makes multireddit mode unusable 2018-07-22 18:05:20 +03:00
Ali Parlakci
8c6f593496 Update changelog 2018-07-22 17:39:30 +03:00
Ali Parlakci
b60ce8a71e Put log files in a folder 2018-07-22 17:38:35 +03:00
Ali Parlakci
49920cc457 Bug fix 2018-07-22 17:25:30 +03:00
Ali Parlakci
c70e7c2ebb Update version 2018-07-22 14:39:09 +03:00
Ali Parlakci
3931dfff54 Update links 2018-07-21 22:03:16 +03:00
Ali Parlakci
4a8c2377f9 Updated --help page 2018-07-21 21:55:01 +03:00
Ali Parlakci
8a18a42a9a Updated changelog 2018-07-21 21:54:23 +03:00
Ali Parlakçı
6c2d748fbc Exclude post types (#38)
* Added argument for excluded links

* Added exclude in PromptUser()

* Added functionality for exclude and bug fix
2018-07-21 21:52:28 +03:00
Ali Parlakci
8c966df105 Improved traceback 2018-07-21 21:50:54 +03:00
8 changed files with 568 additions and 297 deletions

View File

@@ -6,13 +6,12 @@ This program downloads imgur, gfycat and direct image and video links of saved p
## What it can do
- Can get posts from: frontpage, subreddits, multireddits, redditor's submissions, upvoted and saved posts; search results or just plain reddit links
- Sorts posts by hot, top, new and so on
- Downloads imgur albums, gfycat links, [self posts](#how-do-i-open-self-post-files) and any link to a direct image
- Downloads **REDDIT** images and videos, **IMGUR** images and albums, **GFYCAT** links, **EROME** images and albums, **SELF POSTS** and any link to a **DIRECT IMAGE**
- Skips the existing ones
- Puts post titles to file's name
- Puts post title and OP's name in file's name
- Puts every post to its subreddit's folder
- Saves a reusable copy of posts' details that are found so that they can be re-downloaded again
- Logs failed ones in a file to so that you can try to download them later
- Can run with double-clicking on Windows
## [Download the latest release](https://github.com/aliparlakci/bulk-downloader-for-reddit/releases/latest)
@@ -24,7 +23,8 @@ This program downloads imgur, gfycat and direct image and video links of saved p
- **MacOS** users have to **[compile it from source code](docs/COMPILE_FROM_SOURCE.md)**.
Script also accepts **command-line arguments**, get further information from **[`--help`](docs/COMMAND_LINE_ARGUMENTS.md)**
### Additional options
Script also accepts additional options via command-line arguments, get further information from **[`--help`](docs/COMMAND_LINE_ARGUMENTS.md)**
## Setting up the script
Because this is not a commercial app, you need to create an imgur developer app in order API to work.
@@ -37,11 +37,24 @@ Because this is not a commercial app, you need to create an imgur developer app
* Correct CHAPTCHA
* Click **submit** button
It should redirect to a page which shows your **imgur_client_id** and **imgur_client_secret**
It should redirect you to a page which shows your **imgur_client_id** and **imgur_client_secret**
\* Select **OAuth 2 authorization without a callback URL** first then select **Anonymous usage without user authorization** if it says *Authorization callback URL: required*
\* Select **OAuth 2 authorization without a callback URL** first then select **Anonymous usage without user authorization** if it says *Authorization callback URL: required*. If this does not work, it is safe to proceed with **OAuth 2 authorization without a callback URL**.
## FAQ
### What do the dots resemble when getting posts?
- Each dot means that 100 posts are scanned.
### Getting posts is taking too long.
- You can press Ctrl+C to interrupt it and start downloading.
### How downloaded files' names are formatted?
- Self posts and images that are not belong to an album are formatted as **`[SUBMITTER NAME]_[POST TITLE]_[REDDIT ID]`**.
You can use *reddit id* to go to post's reddit page by going to link **reddit.com/[REDDIT ID]**
- An image in an imgur album is formatted as **`[ITEM NUMBER]_[IMAGE TITLE]_[IMGUR ID]`**
Similarly, you can use *imgur id* to go to image's imgur page by going to link **imgur.com/[IMGUR ID]**.
### How do I open self post files?
- Self posts are held at reddit as styled with markdown. So, the script downloads them as they are in order not to lose their stylings.
However, there is a [great Chrome extension](https://chrome.google.com/webstore/detail/markdown-viewer/ckkdlimhmcjmikdlpkmbgfkaikojcbjk) for viewing Markdown files with its styling. Install it and open the files with [Chrome](https://www.google.com/intl/tr/chrome/).
@@ -52,8 +65,35 @@ It should redirect to a page which shows your **imgur_client_id** and **imgur_cl
- All of the user data is held in **config.json** file which is in a folder named "Bulk Downloader for Reddit" in your **Home** directory. You can edit
them, there.
## Changelog
### [20/07/2018](https://github.com/aliparlakci/bulk-downloader-for-reddit/commit/7548a010198fb693841ca03654d2c9bdf5742139)
## Changes on *master*
### [26/07/2018](https://github.com/aliparlakci/bulk-downloader-for-reddit/tree/1623722138bad80ae39ffcd5fb38baf80680deac)
- Improved verbose mode
- Minimalized the console output
- Added quit option for auto quitting the program after process finished
### [25/07/2018](https://github.com/aliparlakci/bulk-downloader-for-reddit/tree/1623722138bad80ae39ffcd5fb38baf80680deac)
- Added verbose mode
- Stylized the console output
### [24/07/2018](https://github.com/aliparlakci/bulk-downloader-for-reddit/tree/7a68ff3efac9939f9574c2cef6184b92edb135f4)
- Added OP's name to file names (backwards compatible)
- Deleted # char from file names (backwards compatible)
- Improved exception handling
### [23/07/2018](https://github.com/aliparlakci/bulk-downloader-for-reddit/tree/7314e17125aa78fd4e6b28e26fda7ec7db7e0147)
- Splited download() function
- Added erome support
- Removed exclude feature
- Bug fixes
### [22/07/2018](https://github.com/aliparlakci/bulk-downloader-for-reddit/tree/6e7463005051026ad64006a8580b0b5dc9536b8c)
- Put log files in a folder named "LOG_FILES"
- Fixed the bug that makes multireddit mode unusable
### [21/07/2018](https://github.com/aliparlakci/bulk-downloader-for-reddit/tree/4a8c2377f9fb4d60ed7eeb8d50aaf9a26492462a)
- Added exclude mode
### [20/07/2018](https://github.com/aliparlakci/bulk-downloader-for-reddit/tree/7548a010198fb693841ca03654d2c9bdf5742139)
- "0" input for no limit
- Fixed the bug that recognizes none image direct links as image links
@@ -63,7 +103,7 @@ It should redirect to a page which shows your **imgur_client_id** and **imgur_cl
- Fixed the bug that prevents downloading some gfycat URLs
### [13/07/2018](https://github.com/aliparlakci/bulk-downloader-for-reddit/tree/9f831e1b784a770c82252e909462871401a05c11)
- Change config.json file's path to home directory
- Changed config.json file's path to home directory
### [12/07/2018](https://github.com/aliparlakci/bulk-downloader-for-reddit/tree/50a77f6ba54c24f5647d5ea4e177400b71ff04a7)
- Added binaries for Windows and Linux

View File

@@ -5,25 +5,30 @@ See **[compiling from source](COMPILE_FROM_SOURCE.md)** page first unless you ar
***Use*** `.\bulk-downloader-for-reddit.exe` ***or*** `./bulk-downloader-for-reddit` ***if you are using the executable***.
```console
$ python script.py --help
usage: script.py [-h] [--directory DIRECTORY] [--link link] [--saved]
[--submitted] [--upvoted] [--log LOG FILE]
[--subreddit SUBREDDIT [SUBREDDIT ...]]
usage: script.py [-h] [--directory DIRECTORY] [--NoDownload] [--verbose]
[--quit] [--link link] [--saved] [--submitted] [--upvoted]
[--log LOG FILE] [--subreddit SUBREDDIT [SUBREDDIT ...]]
[--multireddit MULTIREDDIT] [--user redditor]
[--search query] [--sort SORT TYPE] [--limit Limit]
[--time TIME_LIMIT] [--NoDownload]
[--time TIME_LIMIT]
This program downloads media from reddit posts
optional arguments:
-h, --help show this help message and exit
--directory DIRECTORY
--directory DIRECTORY, -d DIRECTORY
Specifies the directory where posts will be downloaded
to
--NoDownload Just gets the posts and stores them in a file for
downloading later
--verbose, -v Verbose Mode
--quit, -q Auto quit afer the process finishes
--link link, -l link Get posts from link
--saved Triggers saved mode
--submitted Gets posts of --user
--upvoted Gets upvoted posts of --user
--log LOG FILE Triggers log read mode and takes a log file
--log LOG FILE Takes a log file which created by itself (json files),
reads posts and tries downloading them again.
--subreddit SUBREDDIT [SUBREDDIT ...]
Triggers subreddit mode and takes subreddit's name
without r/. use "frontpage" for frontpage
@@ -37,8 +42,6 @@ optional arguments:
--limit Limit default: unlimited
--time TIME_LIMIT Either hour, day, week, month, year or all. default:
all
--NoDownload Just gets the posts and store them in a file for
downloading later
```
# Examples

View File

@@ -1,16 +1,14 @@
# Compiling from source code
## Requirements
### Python 3 Interpreter
Latest* version of **Python 3** is needed. See if it is already installed [here](#finding-the-correct-keyword-for-python). If not, download the matching release for your platform [here](https://www.python.org/downloads/) and install it. If you are a *Windows* user, selecting **Add Python 3 to PATH** option is mandatory.
Latest* version of **Python 3** is needed. See if it is already installed [here](#finding-the-correct-keyword-for-python). If not, download the matching release for your platform [here](https://www.python.org/downloads/) and install it. If you are a *Windows* user, selecting **Add Python 3 to PATH** option when installing the software is mandatory.
\* *Use Python 3.6.5 if you encounter an issue*
## Using terminal
### To open it...
- **On Windows 8/8.1/10**: Press the File tab on **Windows Explorer**, click on **Open Windows PowerShell** or **Open Windows Command Prompt** or look for *Command Prompt* or *PowerShell* in *Start Menu*.
- **On Windows**: Press **Shift+Right Click**, select **Open Powershell window here** or **Open Command Prompt window here**
- **On Windows 7**: Press **WindowsKey+R**, type **cmd** and hit Enter or look for *Command Prompt* or *PowerShell* in *Start Menu*.
- **On Linux**: Right-click in a folder and select **Open Terminal** or press **Ctrl+Alt+T** or look for **Terminal** in the programs.
- **On Linux**: Right-click in a folder and select **Open Terminal** or press **Ctrl+Alt+T**.
- **On MacOS**: Look for an app called **Terminal**.
@@ -39,4 +37,4 @@ python -m pip install -r requirements.txt
---
Now, you can go to [Using command-line arguments](COMMAND_LINE_ARGUMENTS.md)
Now, you can go to [Using command-line arguments](COMMAND_LINE_ARGUMENTS.md)

301
script.py
View File

@@ -13,7 +13,7 @@ import time
from io import StringIO
from pathlib import Path, PurePath
from src.downloader import Direct, Gfycat, Imgur, Self
from src.downloader import Direct, Gfycat, Imgur, Self, Erome
from src.errors import *
from src.parser import LinkDesigner
from src.searcher import getPosts
@@ -22,7 +22,7 @@ from src.tools import (GLOBAL, createLogFile, jsonFile, nameCorrector,
__author__ = "Ali Parlakci"
__license__ = "GPL"
__version__ = "1.2.1"
__version__ = "1.6.0"
__maintainer__ = "Ali Parlakci"
__email__ = "parlakciali@gmail.com"
@@ -62,10 +62,26 @@ def parseArguments(arguments=[]):
description="This program downloads " \
"media from reddit " \
"posts")
parser.add_argument("--directory",
parser.add_argument("--directory","-d",
help="Specifies the directory where posts will be " \
"downloaded to",
metavar="DIRECTORY")
parser.add_argument("--NoDownload",
help="Just gets the posts and stores them in a file" \
" for downloading later",
action="store_true",
default=False)
parser.add_argument("--verbose","-v",
help="Verbose Mode",
action="store_true",
default=False)
parser.add_argument("--quit","-q",
help="Auto quit afer the process finishes",
action="store_true",
default=False)
parser.add_argument("--link","-l",
help="Get posts from link",
@@ -137,12 +153,6 @@ def parseArguments(arguments=[]):
choices=["all","hour","day","week","month","year"],
metavar="TIME_LIMIT",
type=str)
parser.add_argument("--NoDownload",
help="Just gets the posts and store them in a file" \
" for downloading later",
action="store_true",
default=False)
if arguments == []:
return parser.parse_args()
@@ -159,7 +169,10 @@ def checkConflicts():
else:
user = 1
modes = ["saved","subreddit","submitted","search","log","link","upvoted"]
modes = [
"saved","subreddit","submitted","search","log","link","upvoted",
"multireddit"
]
values = {
x: 0 if getattr(GLOBAL.arguments,x) is None or \
@@ -233,10 +246,10 @@ class PromptUser:
if programMode == "subreddit":
subredditInput = input("subreddit: ")
subredditInput = input("subreddit (enter frontpage for frontpage): ")
GLOBAL.arguments.subreddit = subredditInput
while not subredditInput == "":
while not (subredditInput == "" or subredditInput.lower() == "frontpage"):
subredditInput = input("subreddit: ")
GLOBAL.arguments.subreddit += "+" + subredditInput
@@ -244,7 +257,8 @@ class PromptUser:
GLOBAL.arguments.subreddit = "+".join(GLOBAL.arguments.subreddit.split())
# DELETE THE PLUS (+) AT THE END
GLOBAL.arguments.subreddit = GLOBAL.arguments.subreddit[:-1]
if not subredditInput.lower() == "frontpage":
GLOBAL.arguments.subreddit = GLOBAL.arguments.subreddit[:-1]
print("\nselect sort type:")
sortTypes = [
@@ -265,7 +279,7 @@ class PromptUser:
elif programMode == "multireddit":
GLOBAL.arguments.user = input("\nredditor: ")
GLOBAL.arguments.subreddit = input("\nmultireddit: ")
GLOBAL.arguments.multireddit = input("\nmultireddit: ")
print("\nselect sort type:")
sortTypes = [
@@ -317,7 +331,6 @@ class PromptUser:
GLOBAL.arguments.log = input("\nlog file directory:")
if Path(GLOBAL.arguments.log ).is_file():
break
while True:
try:
GLOBAL.arguments.limit = int(input("\nlimit (0 for none): "))
@@ -377,6 +390,9 @@ def prepareAttributes():
ATTRIBUTES["subreddit"] = GLOBAL.arguments.subreddit
elif GLOBAL.arguments.multireddit is not None:
ATTRIBUTES["multireddit"] = GLOBAL.arguments.multireddit
elif GLOBAL.arguments.saved is True:
ATTRIBUTES["saved"] = True
@@ -416,75 +432,76 @@ def postFromLog(fileName):
return posts
def postExists(POST):
def isPostExists(POST):
"""Figure out a file's name and checks if the file already exists"""
title = nameCorrector(POST['postTitle'])
FILENAME = title + "_" + POST['postId']
PATH = GLOBAL.directory / POST["postSubreddit"]
possibleExtensions = [".jpg",".png",".mp4",".gif",".webm",".md"]
for i in range(2):
for extension in possibleExtensions:
FILE_PATH = PATH / (FILENAME+extension)
if FILE_PATH.exists():
return True
else:
FILENAME = POST['postId']
for extension in possibleExtensions:
OLD_FILE_PATH = PATH / (
title
+ "_" + POST['postId']
+ extension
)
FILE_PATH = PATH / (
POST["postSubmitter"]
+ "_" + title
+ "_" + POST['postId']
+ extension
)
SHORT_FILE_PATH = PATH / (POST['postId']+extension)
if OLD_FILE_PATH.exists() or \
FILE_PATH.exists() or \
SHORT_FILE_PATH.exists():
return True
else:
return False
def download(submissions):
"""Analyze list of submissions and call the right function
to download each one, catch errors, update the log files
"""
def downloadPost(SUBMISSION):
directory = GLOBAL.directory / SUBMISSION['postSubreddit']
subsLenght = len(submissions)
lastRequestTime = 0
downloadedCount = subsLenght
duplicates = 0
BACKUP = {}
global lastRequestTime
FAILED_FILE = createLogFile("FAILED")
downloaders = {
"imgur":Imgur,"gfycat":Gfycat,"erome":Erome,"direct":Direct,"self":Self
}
for i in range(subsLenght):
print("\n({}/{})".format(i+1,subsLenght))
print(
"https://reddit.com/r/{subreddit}/comments/{id}".format(
subreddit=submissions[i]['postSubreddit'],
id=submissions[i]['postId']
)
)
print()
if SUBMISSION['postType'] in downloaders:
if postExists(submissions[i]):
result = False
print(submissions[i]['postType'].upper())
print("It already exists")
duplicates += 1
downloadedCount -= 1
continue
directory = GLOBAL.directory / submissions[i]['postSubreddit']
if submissions[i]['postType'] == 'imgur':
print("IMGUR",end="")
if SUBMISSION['postType'] == "imgur":
while int(time.time() - lastRequestTime) <= 2:
pass
credit = Imgur.get_credits()
IMGUR_RESET_TIME = credit['UserReset']-time.time()
USER_RESET = ("after " \
+ str(int(IMGUR_RESET_TIME/60)) \
+ " Minutes " \
+ str(int(IMGUR_RESET_TIME%60)) \
+ " Seconds")
+ str(int(IMGUR_RESET_TIME/60)) \
+ " Minutes " \
+ str(int(IMGUR_RESET_TIME%60)) \
+ " Seconds")
if credit['ClientRemaining'] < 25 or credit['UserRemaining'] < 25:
printCredit = {"noPrint":False}
else:
printCredit = {"noPrint":True}
print(
" => Client: {} - User: {} - Reset {}".format(
"==> Client: {} - User: {} - Reset {}\n".format(
credit['ClientRemaining'],
credit['UserRemaining'],
USER_RESET
)
),end="",**printCredit
)
if not (credit['UserRemaining'] == 0 or \
@@ -494,94 +511,105 @@ def download(submissions):
"""
while int(time.time() - lastRequestTime) <= 2:
pass
lastRequestTime = time.time()
try:
Imgur(directory,submissions[i])
except FileAlreadyExistsError:
print("It already exists")
duplicates += 1
downloadedCount -= 1
except ImgurLoginError:
print(
"Imgur login failed. Quitting the program "\
"as unexpected errors might occur."
)
sys.exit()
except Exception as exception:
print(exception)
FAILED_FILE.add({int(i+1):[str(exception),submissions[i]]})
downloadedCount -= 1
else:
if credit['UserRemaining'] == 0:
KEYWORD = "user"
elif credit['ClientRemaining'] == 0:
KEYWORD = "client"
print('{} LIMIT EXCEEDED\n'.format(KEYWORD.upper()))
FAILED_FILE.add(
{int(i+1):['{} LIMIT EXCEEDED\n'.format(KEYWORD.upper()),
submissions[i]]}
)
downloadedCount -= 1
raise ImgurLimitError('{} LIMIT EXCEEDED\n'.format(KEYWORD.upper()))
elif submissions[i]['postType'] == 'gfycat':
print("GFYCAT")
try:
Gfycat(directory,submissions[i])
downloaders[SUBMISSION['postType']] (directory,SUBMISSION)
except FileAlreadyExistsError:
print("It already exists")
duplicates += 1
downloadedCount -= 1
except NotADownloadableLinkError as exception:
print(exception)
FAILED_FILE.add({int(i+1):[str(exception),submissions[i]]})
downloadedCount -= 1
else:
raise NoSuitablePost
except Exception as exception:
print(exception)
FAILED_FILE.add({int(i+1):[str(exception),submissions[i]]})
downloadedCount -= 1
return None
elif submissions[i]['postType'] == 'direct':
print("DIRECT")
try:
Direct(directory,submissions[i])
def download(submissions):
"""Analyze list of submissions and call the right function
to download each one, catch errors, update the log files
"""
except FileAlreadyExistsError:
print("It already exists")
downloadedCount -= 1
duplicates += 1
subsLenght = len(submissions)
global lastRequestTime
lastRequestTime = 0
downloadedCount = subsLenght
duplicates = 0
except Exception as exception:
print(exception)
FAILED_FILE.add({int(i+1):[str(exception),submissions[i]]})
downloadedCount -= 1
FAILED_FILE = createLogFile("FAILED")
for i in range(subsLenght):
print(
f"\n({i+1}/{subsLenght}) {submissions[i]['postType'].upper()} " \
f" r/{submissions[i]['postSubreddit']}",end=""
)
if isPostExists(submissions[i]):
print("\nIt already exists")
duplicates += 1
downloadedCount -= 1
continue
try:
downloadPost(submissions[i])
elif submissions[i]['postType'] == 'self':
print("SELF")
try:
Self(directory,submissions[i])
except FileAlreadyExistsError:
print("It already exists")
duplicates += 1
downloadedCount -= 1
except FileAlreadyExistsError:
print("It already exists")
downloadedCount -= 1
duplicates += 1
except ImgurLoginError:
print(
"Imgur login failed. \nQuitting the program "\
"as unexpected errors might occur."
)
sys.exit()
except Exception as exception:
print(exception)
FAILED_FILE.add({int(i+1):[str(exception),submissions[i]]})
downloadedCount -= 1
except ImgurLimitError as exception:
FAILED_FILE.add({int(i+1):[
"{class_name}: {info}".format(
class_name=exception.__class__.__name__,info=str(exception)
),
submissions[i]
]})
downloadedCount -= 1
else:
except NotADownloadableLinkError as exception:
print(
"{class_name}: {info}".format(
class_name=exception.__class__.__name__,info=str(exception)
)
)
FAILED_FILE.add({int(i+1):[
"{class_name}: {info}".format(
class_name=exception.__class__.__name__,info=str(exception)
),
submissions[i]
]})
downloadedCount -= 1
except NoSuitablePost:
print("No match found, skipping...")
downloadedCount -= 1
except Exception as exception:
# raise exception
print(
"{class_name}: {info}".format(
class_name=exception.__class__.__name__,info=str(exception)
)
)
FAILED_FILE.add({int(i+1):[
"{class_name}: {info}".format(
class_name=exception.__class__.__name__,info=str(exception)
),
submissions[i]
]})
downloadedCount -= 1
if duplicates:
print("\n There was {} duplicates".format(duplicates))
@@ -600,15 +628,12 @@ def main():
else:
GLOBAL.directory = Path(input("download directory: "))
print("\n"," ".join(sys.argv),"\n")
print("\n"," ".join(sys.argv),"\n",noPrint=True)
try:
checkConflicts()
except ProgramModeError as err:
PromptUser()
except Exception as err:
print(err)
sys.exit()
if not Path(GLOBAL.configDirectory).is_dir():
os.makedirs(GLOBAL.configDirectory)
@@ -618,7 +643,7 @@ def main():
logDir = Path(GLOBAL.arguments.log)
download(postFromLog(logDir))
sys.exit()
try:
POSTS = getPosts(prepareAttributes())
except InsufficientPermission:
@@ -663,12 +688,16 @@ if __name__ == "__main__":
print = printToFile
GLOBAL.RUN_TIME = time.time()
main()
except KeyboardInterrupt:
if GLOBAL.directory is None:
GLOBAL.directory = Path(".\\")
print("\nQUITTING...")
except Exception as exception:
logging.error("Runtime error!", exc_info=full_exc_info(sys.exc_info()))
if GLOBAL.directory is None:
GLOBAL.directory = Path(".\\")
logging.error(sys.exc_info()[0].__name__,
exc_info=full_exc_info(sys.exc_info()))
print(log_stream.getvalue())
input("Press enter to quit\n")
if not GLOBAL.arguments.quit: input("\nPress enter to quit\n")

View File

@@ -2,7 +2,9 @@ import io
import os
import sys
import urllib.request
from html.parser import HTMLParser
from pathlib import Path
from urllib.error import HTTPError
import imgurpython
from multiprocessing import Queue
@@ -59,16 +61,150 @@ def getFile(fileDir,tempDir,imageURL,indent=0):
tempDir,
reporthook=dlProgress)
os.rename(tempDir,fileDir)
print(" "*indent+"Downloaded"+" "*10)
break
except ConnectionResetError as exception:
print(" "*indent + str(exception))
print(" "*indent + "Trying again\n")
except FileNotFoundError:
raise FileNameTooLong
else:
print(" "*indent+"Downloaded"+" "*10)
break
else:
raise FileAlreadyExistsError
class Erome:
def __init__(self,directory,post):
try:
IMAGES = self.getLinks(post['postURL'])
except urllib.error.HTTPError:
raise NotADownloadableLinkError("Not a downloadable link")
imagesLenght = len(IMAGES)
howManyDownloaded = imagesLenght
duplicates = 0
if imagesLenght == 1:
extension = getExtension(IMAGES[0])
title = nameCorrector(post['postTitle'])
print(post["postSubmitter"]+"_"+title+"_"+post['postId']+extension)
fileDir = directory / (
post["postSubmitter"]+"_"+title+"_"+post['postId']+extension
)
tempDir = directory / (
post["postSubmitter"]+"_"+title+"_"+post['postId']+".tmp"
)
imageURL = "https:" + IMAGES[0]
try:
getFile(fileDir,tempDir,imageURL)
except FileNameTooLong:
fileDir = directory / (post['postId'] + extension)
tempDir = directory / (post['postId'] + '.tmp')
getFile(fileDir,tempDir,imageURL)
else:
title = nameCorrector(post['postTitle'])
print(post["postSubmitter"]+"_"+title+"_"+post['postId'],end="\n\n")
folderDir = directory / (
post["postSubmitter"] + "_" + title + "_" + post['postId']
)
try:
if not os.path.exists(folderDir):
os.makedirs(folderDir)
except FileNotFoundError:
folderDir = directory / post['postId']
os.makedirs(folderDir)
for i in range(imagesLenght):
extension = getExtension(IMAGES[i])
fileName = str(i+1)
imageURL = "https:" + IMAGES[i]
fileDir = folderDir / (fileName + extension)
tempDir = folderDir / (fileName + ".tmp")
print(" ({}/{})".format(i+1,imagesLenght))
print(" {}".format(fileName+extension))
try:
getFile(fileDir,tempDir,imageURL,indent=2)
print()
except FileAlreadyExistsError:
print(" The file already exists" + " "*10,end="\n\n")
duplicates += 1
howManyDownloaded -= 1
except Exception as exception:
# raise exception
print("\n Could not get the file")
print(
" "
+ "{class_name}: {info}".format(
class_name=exception.__class__.__name__,
info=str(exception)
)
+ "\n"
)
exceptionType = exception
howManyDownloaded -= 1
if duplicates == imagesLenght:
raise FileAlreadyExistsError
elif howManyDownloaded + duplicates < imagesLenght:
raise AlbumNotDownloadedCompletely(
"Album Not Downloaded Completely"
)
def getLinks(self,url,lineNumber=129):
content = []
lineNumber = None
class EromeParser(HTMLParser):
tag = None
def handle_starttag(self, tag, attrs):
self.tag = {tag:{attr[0]: attr[1] for attr in attrs}}
pageSource = (urllib.request.urlopen(url).read().decode().split('\n'))
""" FIND WHERE ALBUM STARTS IN ORDER NOT TO GET WRONG LINKS"""
for i in range(len(pageSource)):
obj = EromeParser()
obj.feed(pageSource[i])
tag = obj.tag
if tag is not None:
if "div" in tag:
if "id" in tag["div"]:
if tag["div"]["id"] == "album":
lineNumber = i
break
for line in pageSource[lineNumber:]:
obj = EromeParser()
obj.feed(line)
tag = obj.tag
if tag is not None:
if "img" in tag:
if "class" in tag["img"]:
if tag["img"]["class"]=="img-front":
content.append(tag["img"]["src"])
elif "source" in tag:
content.append(tag["source"]["src"])
return [
link for link in content \
if link.endswith("_480p.mp4") or not link.endswith(".mp4")
]
class Imgur:
def __init__(self,directory,post):
self.imgurClient = self.initImgur()
@@ -88,13 +224,22 @@ class Imgur:
post['postExt'] = getExtension(post['mediaURL'])
title = nameCorrector(post['postTitle'])
print(title+"_" +post['postId']+post['postExt'])
print(post["postSubmitter"]+"_"+title+"_"+post['postId']+post['postExt'])
fileDir = title + "_" + post['postId'] + post['postExt']
fileDir = directory / fileDir
fileDir = directory / (
post["postSubmitter"]
+ "_" + title
+ "_" + post['postId']
+ post['postExt']
)
tempDir = directory / (
post["postSubmitter"]
+ "_" + title
+ "_" + post['postId']
+ ".tmp"
)
tempDir = title + "_" + post['postId'] + '.tmp'
tempDir = directory / tempDir
try:
getFile(fileDir,tempDir,post['mediaURL'])
except FileNameTooLong:
@@ -110,9 +255,11 @@ class Imgur:
duplicates = 0
title = nameCorrector(post['postTitle'])
print(title+"_"+post['postId'],end="\n\n")
print(post["postSubmitter"]+"_"+title+"_"+post['postId'],end="\n\n")
folderDir = directory / (title+"_"+post['postId'])
folderDir = directory / (
post["postSubmitter"] + "_" + title + "_" + post['postId']
)
try:
if not os.path.exists(folderDir):
@@ -165,13 +312,20 @@ class Imgur:
except Exception as exception:
print("\n Could not get the file")
print(" " + str(exception) + "\n")
print(
" "
+ "{class_name}: {info}".format(
class_name=exception.__class__.__name__,
info=str(exception)
)
+ "\n"
)
exceptionType = exception
howManyDownloaded -= 1
if duplicates == imagesLenght:
raise FileAlreadyExistsError
elif howManyDownloaded < imagesLenght:
elif howManyDownloaded + duplicates < imagesLenght:
raise AlbumNotDownloadedCompletely(
"Album Not Downloaded Completely"
)
@@ -230,10 +384,15 @@ class Gfycat:
if not os.path.exists(directory): os.makedirs(directory)
title = nameCorrector(POST['postTitle'])
print(title+"_"+POST['postId']+POST['postExt'])
print(POST["postSubmitter"]+"_"+title+"_"+POST['postId']+POST['postExt'])
fileDir = directory / (title+"_"+POST['postId']+POST['postExt'])
tempDir = directory / (title+"_"+POST['postId']+".tmp")
fileDir = directory / (
POST["postSubmitter"]+"_"+title+"_"+POST['postId']+POST['postExt']
)
tempDir = directory / (
POST["postSubmitter"]+"_"+title+"_"+POST['postId']+".tmp"
)
try:
getFile(fileDir,tempDir,POST['mediaURL'])
except FileNameTooLong:
@@ -279,13 +438,14 @@ class Direct:
POST['postExt'] = getExtension(POST['postURL'])
if not os.path.exists(directory): os.makedirs(directory)
title = nameCorrector(POST['postTitle'])
print(title+"_"+POST['postId']+POST['postExt'])
print(POST["postSubmitter"]+"_"+title+"_"+POST['postId']+POST['postExt'])
fileDir = title+"_"+POST['postId']+POST['postExt']
fileDir = directory / fileDir
tempDir = title+"_"+POST['postId']+".tmp"
tempDir = directory / tempDir
fileDir = directory / (
POST["postSubmitter"]+"_"+title+"_"+POST['postId']+POST['postExt']
)
tempDir = directory / (
POST["postSubmitter"]+"_"+title+"_"+POST['postId']+".tmp"
)
try:
getFile(fileDir,tempDir,POST['postURL'])
@@ -300,10 +460,11 @@ class Self:
if not os.path.exists(directory): os.makedirs(directory)
title = nameCorrector(post['postTitle'])
print(title+"_"+post['postId']+".md")
print(post["postSubmitter"]+"_"+title+"_"+post['postId']+".md")
fileDir = title+"_"+post['postId']+".md"
fileDir = directory / fileDir
fileDir = directory / (
post["postSubmitter"]+"_"+title+"_"+post['postId']+".md"
)
if Path.is_file(fileDir):
raise FileAlreadyExistsError
@@ -326,7 +487,11 @@ class Self:
+ ")\n"
+ post["postContent"]
+ "\n\n---\n\n"
+ "submitted by [u/"
+ "submitted to [r/"
+ post["postSubreddit"]
+ "](https://www.reddit.com/r/"
+ post["postSubreddit"]
+ ") by [u/"
+ post["postSubmitter"]
+ "](https://www.reddit.com/user/"
+ post["postSubmitter"]

View File

@@ -80,4 +80,10 @@ class InvalidSortingType(Exception):
pass
class FileNotFoundError(Exception):
pass
class NoSuitablePost(Exception):
pass
class ImgurLimitError(Exception):
pass

View File

@@ -1,4 +1,5 @@
import os
import sys
import random
import socket
import webbrowser
@@ -14,60 +15,62 @@ from src.errors import (NoMatchingSubmissionFound, NoPrawSupport,
print = printToFile
class GetAuth:
def __init__(self,redditInstance,port):
self.redditInstance = redditInstance
self.PORT = int(port)
def recieve_connection(self):
"""Wait for and then return a connected socket..
Opens a TCP connection on port 8080, and waits for a single client.
"""
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server.bind(('localhost', self.PORT))
server.listen(1)
client = server.accept()[0]
server.close()
return client
def send_message(self, message):
"""Send message to client and close the connection."""
self.client.send('HTTP/1.1 200 OK\r\n\r\n{}'.format(message).encode('utf-8'))
self.client.close()
def getRefreshToken(self,*scopes):
state = str(random.randint(0, 65000))
url = self.redditInstance.auth.url(scopes, state, 'permanent')
print("Go to this URL and login to reddit:\n\n",url)
webbrowser.open(url,new=2)
self.client = self.recieve_connection()
data = self.client.recv(1024).decode('utf-8')
param_tokens = data.split(' ', 2)[1].split('?', 1)[1].split('&')
params = {
key: value for (key, value) in [token.split('=') \
for token in param_tokens]
}
if state != params['state']:
self.send_message(
client, 'State mismatch. Expected: {} Received: {}'
.format(state, params['state'])
)
raise RedditLoginFailed
elif 'error' in params:
self.send_message(client, params['error'])
raise RedditLoginFailed
refresh_token = self.redditInstance.auth.authorize(params['code'])
self.send_message(
"<script>" \
"alert(\"You can go back to terminal window now.\");" \
"</script>"
)
return (self.redditInstance,refresh_token)
def beginPraw(config,user_agent = str(socket.gethostname())):
class GetAuth:
def __init__(self,redditInstance,port):
self.redditInstance = redditInstance
self.PORT = int(port)
def recieve_connection(self):
"""Wait for and then return a connected socket..
Opens a TCP connection on port 8080, and waits for a single client.
"""
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server.bind(('localhost', self.PORT))
server.listen(1)
client = server.accept()[0]
server.close()
return client
def send_message(self, message):
"""Send message to client and close the connection."""
self.client.send(
'HTTP/1.1 200 OK\r\n\r\n{}'.format(message).encode('utf-8')
)
self.client.close()
def getRefreshToken(self,*scopes):
state = str(random.randint(0, 65000))
url = self.redditInstance.auth.url(scopes, state, 'permanent')
print("Go to this URL and login to reddit:\n\n",url)
webbrowser.open(url,new=2)
self.client = self.recieve_connection()
data = self.client.recv(1024).decode('utf-8')
param_tokens = data.split(' ', 2)[1].split('?', 1)[1].split('&')
params = {
key: value for (key, value) in [token.split('=') \
for token in param_tokens]
}
if state != params['state']:
self.send_message(
client, 'State mismatch. Expected: {} Received: {}'
.format(state, params['state'])
)
raise RedditLoginFailed
elif 'error' in params:
self.send_message(client, params['error'])
raise RedditLoginFailed
refresh_token = self.redditInstance.auth.authorize(params['code'])
self.send_message(
"<script>" \
"alert(\"You can go back to terminal window now.\");" \
"</script>"
)
return (self.redditInstance,refresh_token)
"""Start reddit instance"""
scopes = ['identity','history','read']
@@ -123,8 +126,6 @@ def getPosts(args):
if args["user"] == "me":
args["user"] = str(reddit.user.me())
print("\nGETTING POSTS\n.\n.\n.\n")
if not "search" in args:
if args["sort"] == "top" or args["sort"] == "controversial":
keyword_params = {
@@ -156,7 +157,7 @@ def getPosts(args):
sort=args["sort"],
subreddit=args["subreddit"],
time=args["time"]
).upper()
).upper(),noPrint=True
)
return redditSearcher(
reddit.subreddit(args["subreddit"]).search(
@@ -184,7 +185,7 @@ def getPosts(args):
"saved posts\nuser:{username}\nlimit={limit}\n".format(
username=reddit.user.me(),
limit=args["limit"]
).upper()
).upper(),noPrint=True
)
return redditSearcher(reddit.user.me().saved(limit=args["limit"]))
@@ -199,7 +200,7 @@ def getPosts(args):
sort=args["sort"],
subreddit=args["subreddit"],
time=args["time"]
).upper()
).upper(),noPrint=True
)
return redditSearcher(
getattr(reddit.front,args["sort"]) (**keyword_params)
@@ -213,7 +214,7 @@ def getPosts(args):
sort=args["sort"],
subreddit=args["subreddit"],
time=args["time"]
).upper()
).upper(),noPrint=True
)
return redditSearcher(
getattr(
@@ -231,7 +232,7 @@ def getPosts(args):
sort=args["sort"],
multireddit=args["multireddit"],
time=args["time"]
).upper()
).upper(),noPrint=True
)
try:
return redditSearcher(
@@ -245,8 +246,6 @@ def getPosts(args):
raise MultiredditNotFound
elif "submitted" in args:
# TODO
# USE REDDIT.USER.ME() INSTEAD WHEN "ME" PASSED AS A --USER
print (
"submitted posts of {user}\nsort: {sort}\n" \
"time: {time}\nlimit: {limit}\n".format(
@@ -254,7 +253,7 @@ def getPosts(args):
sort=args["sort"],
user=args["user"],
time=args["time"]
).upper()
).upper(),noPrint=True
)
return redditSearcher(
getattr(
@@ -263,13 +262,11 @@ def getPosts(args):
)
elif "upvoted" in args:
# TODO
# USE REDDIT.USER.ME() INSTEAD WHEN "ME" PASSED AS A --USER
print (
"upvoted posts of {user}\nlimit: {limit}\n".format(
user=args["user"],
limit=args["limit"]
).upper()
).upper(),noPrint=True
)
try:
return redditSearcher(
@@ -279,7 +276,7 @@ def getPosts(args):
raise InsufficientPermission
elif "post" in args:
print("post: {post}\n".format(post=args["post"]).upper())
print("post: {post}\n".format(post=args["post"]).upper(),noPrint=True)
return redditSearcher(
reddit.submission(url=args["post"]),SINGLE_POST=True
)
@@ -299,6 +296,8 @@ def redditSearcher(posts,SINGLE_POST=False):
gfycatCount = 0
global imgurCount
imgurCount = 0
global eromeCount
eromeCount = 0
global directCount
directCount = 0
global selfCount
@@ -306,6 +305,8 @@ def redditSearcher(posts,SINGLE_POST=False):
allPosts = {}
print("\nGETTING POSTS")
if GLOBAL.arguments.verbose: print("\n")
postsFile = createLogFile("POSTS")
if SINGLE_POST:
@@ -326,43 +327,62 @@ def redditSearcher(posts,SINGLE_POST=False):
if result is not None:
details = result
orderCount += 1
printSubmission(submission,subCount,orderCount)
if GLOBAL.arguments.verbose:
printSubmission(submission,subCount,orderCount)
subList.append(details)
postsFile.add({subCount:[details]})
else:
for submission in posts:
subCount += 1
try:
for submission in posts:
subCount += 1
try:
details = {'postId':submission.id,
'postTitle':submission.title,
'postSubmitter':str(submission.author),
'postType':None,
'postURL':submission.url,
'postSubreddit':submission.subreddit.display_name}
except AttributeError:
continue
if subCount % 100 == 0 and not GLOBAL.arguments.verbose:
sys.stdout.write("")
sys.stdout.flush()
result = checkIfMatching(submission)
if subCount % 1000 == 0:
sys.stdout.write("\n"+" "*14)
sys.stdout.flush()
if result is not None:
details = result
orderCount += 1
printSubmission(submission,subCount,orderCount)
subList.append(details)
try:
details = {'postId':submission.id,
'postTitle':submission.title,
'postSubmitter':str(submission.author),
'postType':None,
'postURL':submission.url,
'postSubreddit':submission.subreddit.display_name}
except AttributeError:
continue
allPosts[subCount] = [details]
result = checkIfMatching(submission)
if result is not None:
details = result
orderCount += 1
if GLOBAL.arguments.verbose:
printSubmission(submission,subCount,orderCount)
subList.append(details)
allPosts[subCount] = [details]
except KeyboardInterrupt:
print("\nKeyboardInterrupt",end="")
postsFile.add(allPosts)
if not len(subList) == 0:
print(
"\nTotal of {} submissions found!\n"\
"{} GFYCATs, {} IMGURs, {} DIRECTs and {} SELF POSTS\n"
.format(len(subList),gfycatCount,imgurCount,directCount,selfCount)
)
if not len(subList) == 0:
if GLOBAL.arguments.NoDownload or GLOBAL.arguments.verbose:
print(
f"\n\nTotal of {len(subList)} submissions found!"
)
print(
f"{gfycatCount} GFYCATs, {imgurCount} IMGURs, " \
f"{eromeCount} EROMEs, {directCount} DIRECTs " \
f"and {selfCount} SELF POSTS",noPrint=True
)
else:
print()
return subList
else:
raise NoMatchingSubmissionFound
@@ -370,6 +390,7 @@ def redditSearcher(posts,SINGLE_POST=False):
def checkIfMatching(submission):
global gfycatCount
global imgurCount
global eromeCount
global directCount
global selfCount
@@ -383,19 +404,20 @@ def checkIfMatching(submission):
except AttributeError:
return None
if ('gfycat' in submission.domain) or \
('imgur' in submission.domain):
if 'gfycat' in submission.domain:
details['postType'] = 'gfycat'
gfycatCount += 1
return details
if 'gfycat' in submission.domain:
details['postType'] = 'gfycat'
gfycatCount += 1
return details
elif 'imgur' in submission.domain:
details['postType'] = 'imgur'
imgurCount += 1
return details
elif 'imgur' in submission.domain:
details['postType'] = 'imgur'
imgurCount += 1
return details
elif 'erome' in submission.domain:
details['postType'] = 'erome'
eromeCount += 1
return details
elif isDirectLink(submission.url) is not False:
details['postType'] = 'direct'

View File

@@ -75,8 +75,10 @@ def createLogFile(TITLE):
put given arguments inside \"HEADER\" key
"""
folderDirectory = GLOBAL.directory / str(time.strftime("%d-%m-%Y_%H-%M-%S",
time.localtime(GLOBAL.RUN_TIME)))
folderDirectory = GLOBAL.directory / "LOG_FILES" / \
str(time.strftime(
"%d-%m-%Y_%H-%M-%S",time.localtime(GLOBAL.RUN_TIME)
))
logFilename = TITLE.upper()+'.json'
if not path.exists(folderDirectory):
@@ -88,23 +90,29 @@ def createLogFile(TITLE):
return FILE
def printToFile(*args, **kwargs):
def printToFile(*args, noPrint=False,**kwargs):
"""Print to both CONSOLE and
CONSOLE LOG file in a folder time stampt in the name
"""
TIME = str(time.strftime("%d-%m-%Y_%H-%M-%S",
time.localtime(GLOBAL.RUN_TIME)))
folderDirectory = GLOBAL.directory / TIME
print(*args,**kwargs)
folderDirectory = GLOBAL.directory / "LOG_FILES" / TIME
if not noPrint or \
GLOBAL.arguments.verbose or \
"file" in kwargs:
print(*args,**kwargs)
if not path.exists(folderDirectory):
makedirs(folderDirectory)
with io.open(
folderDirectory / "CONSOLE_LOG.txt","a",encoding="utf-8"
) as FILE:
print(*args, file=FILE, **kwargs)
if not "file" in kwargs:
with io.open(
folderDirectory / "CONSOLE_LOG.txt","a",encoding="utf-8"
) as FILE:
print(*args, file=FILE, **kwargs)
def nameCorrector(string):
"""Swap strange characters from given string
@@ -130,7 +138,7 @@ def nameCorrector(string):
if len(string.split('\n')) > 1:
string = "".join(string.split('\n'))
BAD_CHARS = ['\\','/',':','*','?','"','<','>','|','.',]
BAD_CHARS = ['\\','/',':','*','?','"','<','>','|','.','#']
if any(x in string for x in BAD_CHARS):
for char in string: