mirror of
https://github.com/KevinMidboe/bulk-downloader-for-reddit.git
synced 2026-01-09 18:55:36 +00:00
Compare commits
57 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d685860c22 | ||
|
|
dcf9f35273 | ||
|
|
7fdf03aa24 | ||
|
|
25d61a4c78 | ||
|
|
558eb107f4 | ||
|
|
6e74630050 | ||
|
|
2fd9248715 | ||
|
|
457b8cd21c | ||
|
|
e953456ead | ||
|
|
ed0564fba0 | ||
|
|
5378555f74 | ||
|
|
95ef308915 | ||
|
|
436f867f2e | ||
|
|
91d71565cc | ||
|
|
c7b7361ded | ||
|
|
cd81a6c38b | ||
|
|
1623722138 | ||
|
|
dad5669441 | ||
|
|
35d54d1eb1 | ||
|
|
394b864d86 | ||
|
|
837281c3c6 | ||
|
|
e6b648d8b3 | ||
|
|
cfaf2de7db | ||
|
|
80546d7094 | ||
|
|
139a81a0e7 | ||
|
|
9bb0a5da7f | ||
|
|
6f2273f182 | ||
|
|
b5d6165802 | ||
|
|
b98815376f | ||
|
|
d9586f99b8 | ||
|
|
76711892a2 | ||
|
|
bfea548eab | ||
|
|
2e852db4c3 | ||
|
|
8ac02e7aff | ||
|
|
5eccf4dd3d | ||
|
|
7a68ff3efa | ||
|
|
3ea2e16b62 | ||
|
|
fc6787aa28 | ||
|
|
21533bb78c | ||
|
|
1781ab8ffe | ||
|
|
821383c465 | ||
|
|
9d0fdc7521 | ||
|
|
0387dd5243 | ||
|
|
93732b0367 | ||
|
|
400ce01918 | ||
|
|
ccedac4bdc | ||
|
|
a6997898ce | ||
|
|
61632c7143 | ||
|
|
9bff3399a8 | ||
|
|
b00d185f67 | ||
|
|
7314e17125 | ||
|
|
2d334d56bf | ||
|
|
974517928f | ||
|
|
bcae177b1e | ||
|
|
229def6578 | ||
|
|
59b0376d6e | ||
|
|
cf1dc7d08c |
53
README.md
53
README.md
@@ -6,13 +6,12 @@ This program downloads imgur, gfycat and direct image and video links of saved p
|
||||
## What it can do
|
||||
- Can get posts from: frontpage, subreddits, multireddits, redditor's submissions, upvoted and saved posts; search results or just plain reddit links
|
||||
- Sorts posts by hot, top, new and so on
|
||||
- Downloads imgur albums, gfycat links, [self posts](#how-do-i-open-self-post-files) and any link to a direct image
|
||||
- Downloads **REDDIT** images and videos, **IMGUR** images and albums, **GFYCAT** links, **EROME** images and albums, **SELF POSTS** and any link to a **DIRECT IMAGE**
|
||||
- Skips the existing ones
|
||||
- Puts post titles to file's name
|
||||
- Puts post title and OP's name in file's name
|
||||
- Puts every post to its subreddit's folder
|
||||
- Saves a reusable copy of posts' details that are found so that they can be re-downloaded again
|
||||
- Logs failed ones in a file to so that you can try to download them later
|
||||
- Can run with double-clicking on Windows
|
||||
|
||||
## [Download the latest release](https://github.com/aliparlakci/bulk-downloader-for-reddit/releases/latest)
|
||||
|
||||
@@ -24,7 +23,8 @@ This program downloads imgur, gfycat and direct image and video links of saved p
|
||||
|
||||
- **MacOS** users have to **[compile it from source code](docs/COMPILE_FROM_SOURCE.md)**.
|
||||
|
||||
Script also accepts **command-line arguments**, get further information from **[`--help`](docs/COMMAND_LINE_ARGUMENTS.md)**
|
||||
### Additional options
|
||||
Script also accepts additional options via command-line arguments, get further information from **[`--help`](docs/COMMAND_LINE_ARGUMENTS.md)**
|
||||
|
||||
## Setting up the script
|
||||
Because this is not a commercial app, you need to create an imgur developer app in order API to work.
|
||||
@@ -37,11 +37,24 @@ Because this is not a commercial app, you need to create an imgur developer app
|
||||
* Correct CHAPTCHA
|
||||
* Click **submit** button
|
||||
|
||||
It should redirect to a page which shows your **imgur_client_id** and **imgur_client_secret**
|
||||
It should redirect you to a page which shows your **imgur_client_id** and **imgur_client_secret**
|
||||
|
||||
\* Select **OAuth 2 authorization without a callback URL** first then select **Anonymous usage without user authorization** if it says *Authorization callback URL: required*
|
||||
\* Select **OAuth 2 authorization without a callback URL** first then select **Anonymous usage without user authorization** if it says *Authorization callback URL: required*. If this does not work, it is safe to proceed with **OAuth 2 authorization without a callback URL**.
|
||||
|
||||
## FAQ
|
||||
### What do the dots resemble when getting posts?
|
||||
- Each dot means that 100 posts are scanned.
|
||||
|
||||
### Getting posts is taking too long.
|
||||
- You can press Ctrl+C to interrupt it and start downloading.
|
||||
|
||||
### How downloaded files' names are formatted?
|
||||
- Self posts and images that are not belong to an album are formatted as **`[SUBMITTER NAME]_[POST TITLE]_[REDDIT ID]`**.
|
||||
You can use *reddit id* to go to post's reddit page by going to link **reddit.com/[REDDIT ID]**
|
||||
|
||||
- An image in an imgur album is formatted as **`[ITEM NUMBER]_[IMAGE TITLE]_[IMGUR ID]`**
|
||||
Similarly, you can use *imgur id* to go to image's imgur page by going to link **imgur.com/[IMGUR ID]**.
|
||||
|
||||
### How do I open self post files?
|
||||
- Self posts are held at reddit as styled with markdown. So, the script downloads them as they are in order not to lose their stylings.
|
||||
However, there is a [great Chrome extension](https://chrome.google.com/webstore/detail/markdown-viewer/ckkdlimhmcjmikdlpkmbgfkaikojcbjk) for viewing Markdown files with its styling. Install it and open the files with [Chrome](https://www.google.com/intl/tr/chrome/).
|
||||
@@ -52,15 +65,35 @@ It should redirect to a page which shows your **imgur_client_id** and **imgur_cl
|
||||
- All of the user data is held in **config.json** file which is in a folder named "Bulk Downloader for Reddit" in your **Home** directory. You can edit
|
||||
them, there.
|
||||
|
||||
## Changelog
|
||||
### [22/07/2018](https://github.com/aliparlakci/bulk-downloader-for-reddit/tree/a67da461d2fcd70672effcb20c8179e3224091bb)
|
||||
## Changes on *master*
|
||||
### [26/07/2018](https://github.com/aliparlakci/bulk-downloader-for-reddit/tree/1623722138bad80ae39ffcd5fb38baf80680deac)
|
||||
- Improved verbose mode
|
||||
- Minimalized the console output
|
||||
- Added quit option for auto quitting the program after process finished
|
||||
|
||||
### [25/07/2018](https://github.com/aliparlakci/bulk-downloader-for-reddit/tree/1623722138bad80ae39ffcd5fb38baf80680deac)
|
||||
- Added verbose mode
|
||||
- Stylized the console output
|
||||
|
||||
### [24/07/2018](https://github.com/aliparlakci/bulk-downloader-for-reddit/tree/7a68ff3efac9939f9574c2cef6184b92edb135f4)
|
||||
- Added OP's name to file names (backwards compatible)
|
||||
- Deleted # char from file names (backwards compatible)
|
||||
- Improved exception handling
|
||||
|
||||
### [23/07/2018](https://github.com/aliparlakci/bulk-downloader-for-reddit/tree/7314e17125aa78fd4e6b28e26fda7ec7db7e0147)
|
||||
- Splited download() function
|
||||
- Added erome support
|
||||
- Removed exclude feature
|
||||
- Bug fixes
|
||||
|
||||
### [22/07/2018](https://github.com/aliparlakci/bulk-downloader-for-reddit/tree/6e7463005051026ad64006a8580b0b5dc9536b8c)
|
||||
- Put log files in a folder named "LOG_FILES"
|
||||
- Fixed the bug that makes multireddit mode unusable
|
||||
|
||||
### [21/07/2018](https://github.com/aliparlakci/bulk-downloader-for-reddit/tree/4a8c2377f9fb4d60ed7eeb8d50aaf9a26492462a)
|
||||
- Added exclude mode
|
||||
|
||||
### [20/07/2018](https://github.com/aliparlakci/bulk-downloader-for-reddit/commit/7548a010198fb693841ca03654d2c9bdf5742139)
|
||||
### [20/07/2018](https://github.com/aliparlakci/bulk-downloader-for-reddit/tree/7548a010198fb693841ca03654d2c9bdf5742139)
|
||||
- "0" input for no limit
|
||||
- Fixed the bug that recognizes none image direct links as image links
|
||||
|
||||
@@ -70,7 +103,7 @@ It should redirect to a page which shows your **imgur_client_id** and **imgur_cl
|
||||
- Fixed the bug that prevents downloading some gfycat URLs
|
||||
|
||||
### [13/07/2018](https://github.com/aliparlakci/bulk-downloader-for-reddit/tree/9f831e1b784a770c82252e909462871401a05c11)
|
||||
- Change config.json file's path to home directory
|
||||
- Changed config.json file's path to home directory
|
||||
|
||||
### [12/07/2018](https://github.com/aliparlakci/bulk-downloader-for-reddit/tree/50a77f6ba54c24f5647d5ea4e177400b71ff04a7)
|
||||
- Added binaries for Windows and Linux
|
||||
|
||||
@@ -5,20 +5,24 @@ See **[compiling from source](COMPILE_FROM_SOURCE.md)** page first unless you ar
|
||||
***Use*** `.\bulk-downloader-for-reddit.exe` ***or*** `./bulk-downloader-for-reddit` ***if you are using the executable***.
|
||||
```console
|
||||
$ python script.py --help
|
||||
usage: script.py [-h] [--directory DIRECTORY] [--link link] [--saved]
|
||||
[--submitted] [--upvoted] [--log LOG FILE]
|
||||
[--subreddit SUBREDDIT [SUBREDDIT ...]]
|
||||
usage: script.py [-h] [--directory DIRECTORY] [--NoDownload] [--verbose]
|
||||
[--quit] [--link link] [--saved] [--submitted] [--upvoted]
|
||||
[--log LOG FILE] [--subreddit SUBREDDIT [SUBREDDIT ...]]
|
||||
[--multireddit MULTIREDDIT] [--user redditor]
|
||||
[--search query] [--sort SORT TYPE] [--limit Limit]
|
||||
[--time TIME_LIMIT] [--NoDownload]
|
||||
[--time TIME_LIMIT]
|
||||
|
||||
This program downloads media from reddit posts
|
||||
|
||||
optional arguments:
|
||||
-h, --help show this help message and exit
|
||||
--directory DIRECTORY
|
||||
--directory DIRECTORY, -d DIRECTORY
|
||||
Specifies the directory where posts will be downloaded
|
||||
to
|
||||
--NoDownload Just gets the posts and stores them in a file for
|
||||
downloading later
|
||||
--verbose, -v Verbose Mode
|
||||
--quit, -q Auto quit afer the process finishes
|
||||
--link link, -l link Get posts from link
|
||||
--saved Triggers saved mode
|
||||
--submitted Gets posts of --user
|
||||
@@ -38,10 +42,6 @@ optional arguments:
|
||||
--limit Limit default: unlimited
|
||||
--time TIME_LIMIT Either hour, day, week, month, year or all. default:
|
||||
all
|
||||
--NoDownload Just gets the posts and store them in a file for
|
||||
downloading later
|
||||
--exclude {imgur,gfycat,direct,self} [{imgur,gfycat,direct,self} ...]
|
||||
Do not download specified links
|
||||
```
|
||||
|
||||
# Examples
|
||||
|
||||
@@ -1,16 +1,14 @@
|
||||
# Compiling from source code
|
||||
## Requirements
|
||||
### Python 3 Interpreter
|
||||
Latest* version of **Python 3** is needed. See if it is already installed [here](#finding-the-correct-keyword-for-python). If not, download the matching release for your platform [here](https://www.python.org/downloads/) and install it. If you are a *Windows* user, selecting **Add Python 3 to PATH** option is mandatory.
|
||||
Latest* version of **Python 3** is needed. See if it is already installed [here](#finding-the-correct-keyword-for-python). If not, download the matching release for your platform [here](https://www.python.org/downloads/) and install it. If you are a *Windows* user, selecting **Add Python 3 to PATH** option when installing the software is mandatory.
|
||||
|
||||
\* *Use Python 3.6.5 if you encounter an issue*
|
||||
## Using terminal
|
||||
### To open it...
|
||||
- **On Windows 8/8.1/10**: Press the File tab on **Windows Explorer**, click on **Open Windows PowerShell** or **Open Windows Command Prompt** or look for *Command Prompt* or *PowerShell* in *Start Menu*.
|
||||
- **On Windows**: Press **Shift+Right Click**, select **Open Powershell window here** or **Open Command Prompt window here**
|
||||
|
||||
- **On Windows 7**: Press **WindowsKey+R**, type **cmd** and hit Enter or look for *Command Prompt* or *PowerShell* in *Start Menu*.
|
||||
|
||||
- **On Linux**: Right-click in a folder and select **Open Terminal** or press **Ctrl+Alt+T** or look for **Terminal** in the programs.
|
||||
- **On Linux**: Right-click in a folder and select **Open Terminal** or press **Ctrl+Alt+T**.
|
||||
|
||||
- **On MacOS**: Look for an app called **Terminal**.
|
||||
|
||||
@@ -39,4 +37,4 @@ python -m pip install -r requirements.txt
|
||||
|
||||
---
|
||||
|
||||
Now, you can go to [Using command-line arguments](COMMAND_LINE_ARGUMENTS.md)
|
||||
Now, you can go to [Using command-line arguments](COMMAND_LINE_ARGUMENTS.md)
|
||||
|
||||
322
script.py
322
script.py
@@ -13,7 +13,7 @@ import time
|
||||
from io import StringIO
|
||||
from pathlib import Path, PurePath
|
||||
|
||||
from src.downloader import Direct, Gfycat, Imgur, Self
|
||||
from src.downloader import Direct, Gfycat, Imgur, Self, Erome
|
||||
from src.errors import *
|
||||
from src.parser import LinkDesigner
|
||||
from src.searcher import getPosts
|
||||
@@ -22,7 +22,7 @@ from src.tools import (GLOBAL, createLogFile, jsonFile, nameCorrector,
|
||||
|
||||
__author__ = "Ali Parlakci"
|
||||
__license__ = "GPL"
|
||||
__version__ = "1.3.1"
|
||||
__version__ = "1.6.0"
|
||||
__maintainer__ = "Ali Parlakci"
|
||||
__email__ = "parlakciali@gmail.com"
|
||||
|
||||
@@ -62,10 +62,26 @@ def parseArguments(arguments=[]):
|
||||
description="This program downloads " \
|
||||
"media from reddit " \
|
||||
"posts")
|
||||
parser.add_argument("--directory",
|
||||
parser.add_argument("--directory","-d",
|
||||
help="Specifies the directory where posts will be " \
|
||||
"downloaded to",
|
||||
metavar="DIRECTORY")
|
||||
|
||||
parser.add_argument("--NoDownload",
|
||||
help="Just gets the posts and stores them in a file" \
|
||||
" for downloading later",
|
||||
action="store_true",
|
||||
default=False)
|
||||
|
||||
parser.add_argument("--verbose","-v",
|
||||
help="Verbose Mode",
|
||||
action="store_true",
|
||||
default=False)
|
||||
|
||||
parser.add_argument("--quit","-q",
|
||||
help="Auto quit afer the process finishes",
|
||||
action="store_true",
|
||||
default=False)
|
||||
|
||||
parser.add_argument("--link","-l",
|
||||
help="Get posts from link",
|
||||
@@ -137,18 +153,6 @@ def parseArguments(arguments=[]):
|
||||
choices=["all","hour","day","week","month","year"],
|
||||
metavar="TIME_LIMIT",
|
||||
type=str)
|
||||
|
||||
parser.add_argument("--NoDownload",
|
||||
help="Just gets the posts and store them in a file" \
|
||||
" for downloading later",
|
||||
action="store_true",
|
||||
default=False)
|
||||
|
||||
parser.add_argument("--exclude",
|
||||
nargs="+",
|
||||
help="Do not download specified links",
|
||||
choices=["imgur","gfycat","direct","self"],
|
||||
type=str)
|
||||
|
||||
if arguments == []:
|
||||
return parser.parse_args()
|
||||
@@ -242,10 +246,10 @@ class PromptUser:
|
||||
|
||||
if programMode == "subreddit":
|
||||
|
||||
subredditInput = input("subreddit: ")
|
||||
subredditInput = input("subreddit (enter frontpage for frontpage): ")
|
||||
GLOBAL.arguments.subreddit = subredditInput
|
||||
|
||||
while not subredditInput == "":
|
||||
while not (subredditInput == "" or subredditInput.lower() == "frontpage"):
|
||||
subredditInput = input("subreddit: ")
|
||||
GLOBAL.arguments.subreddit += "+" + subredditInput
|
||||
|
||||
@@ -253,7 +257,8 @@ class PromptUser:
|
||||
GLOBAL.arguments.subreddit = "+".join(GLOBAL.arguments.subreddit.split())
|
||||
|
||||
# DELETE THE PLUS (+) AT THE END
|
||||
GLOBAL.arguments.subreddit = GLOBAL.arguments.subreddit[:-1]
|
||||
if not subredditInput.lower() == "frontpage":
|
||||
GLOBAL.arguments.subreddit = GLOBAL.arguments.subreddit[:-1]
|
||||
|
||||
print("\nselect sort type:")
|
||||
sortTypes = [
|
||||
@@ -326,33 +331,6 @@ class PromptUser:
|
||||
GLOBAL.arguments.log = input("\nlog file directory:")
|
||||
if Path(GLOBAL.arguments.log ).is_file():
|
||||
break
|
||||
|
||||
GLOBAL.arguments.exclude = []
|
||||
|
||||
sites = ["imgur","gfycat","direct","self"]
|
||||
|
||||
excludeInput = input("exclude: ").lower()
|
||||
if excludeInput in sites and excludeInput != "":
|
||||
GLOBAL.arguments.exclude = [excludeInput]
|
||||
|
||||
while not excludeInput == "":
|
||||
while True:
|
||||
excludeInput = input("exclude: ").lower()
|
||||
if not excludeInput in sites or excludeInput in GLOBAL.arguments.exclude:
|
||||
break
|
||||
elif excludeInput == "":
|
||||
break
|
||||
else:
|
||||
GLOBAL.arguments.exclude.append(excludeInput)
|
||||
|
||||
for i in range(len(GLOBAL.arguments.exclude)):
|
||||
if " " in GLOBAL.arguments.exclude[i]:
|
||||
inputWithWhitespace = GLOBAL.arguments.exclude[i]
|
||||
del GLOBAL.arguments.exclude[i]
|
||||
for siteInput in inputWithWhitespace.split():
|
||||
if siteInput in sites and siteInput not in GLOBAL.arguments.exclude:
|
||||
GLOBAL.arguments.exclude.append(siteInput)
|
||||
|
||||
while True:
|
||||
try:
|
||||
GLOBAL.arguments.limit = int(input("\nlimit (0 for none): "))
|
||||
@@ -454,79 +432,76 @@ def postFromLog(fileName):
|
||||
|
||||
return posts
|
||||
|
||||
def postExists(POST):
|
||||
def isPostExists(POST):
|
||||
"""Figure out a file's name and checks if the file already exists"""
|
||||
|
||||
title = nameCorrector(POST['postTitle'])
|
||||
FILENAME = title + "_" + POST['postId']
|
||||
PATH = GLOBAL.directory / POST["postSubreddit"]
|
||||
|
||||
possibleExtensions = [".jpg",".png",".mp4",".gif",".webm",".md"]
|
||||
|
||||
for i in range(2):
|
||||
for extension in possibleExtensions:
|
||||
FILE_PATH = PATH / (FILENAME+extension)
|
||||
if FILE_PATH.exists():
|
||||
return True
|
||||
else:
|
||||
FILENAME = POST['postId']
|
||||
for extension in possibleExtensions:
|
||||
|
||||
OLD_FILE_PATH = PATH / (
|
||||
title
|
||||
+ "_" + POST['postId']
|
||||
+ extension
|
||||
)
|
||||
FILE_PATH = PATH / (
|
||||
POST["postSubmitter"]
|
||||
+ "_" + title
|
||||
+ "_" + POST['postId']
|
||||
+ extension
|
||||
)
|
||||
|
||||
SHORT_FILE_PATH = PATH / (POST['postId']+extension)
|
||||
|
||||
if OLD_FILE_PATH.exists() or \
|
||||
FILE_PATH.exists() or \
|
||||
SHORT_FILE_PATH.exists():
|
||||
|
||||
return True
|
||||
|
||||
else:
|
||||
return False
|
||||
|
||||
def download(submissions):
|
||||
"""Analyze list of submissions and call the right function
|
||||
to download each one, catch errors, update the log files
|
||||
"""
|
||||
def downloadPost(SUBMISSION):
|
||||
directory = GLOBAL.directory / SUBMISSION['postSubreddit']
|
||||
|
||||
subsLenght = len(submissions)
|
||||
lastRequestTime = 0
|
||||
downloadedCount = subsLenght
|
||||
duplicates = 0
|
||||
BACKUP = {}
|
||||
if GLOBAL.arguments.exclude is not None:
|
||||
ToBeDownloaded = GLOBAL.arguments.exclude
|
||||
else:
|
||||
ToBeDownloaded = []
|
||||
global lastRequestTime
|
||||
|
||||
FAILED_FILE = createLogFile("FAILED")
|
||||
downloaders = {
|
||||
"imgur":Imgur,"gfycat":Gfycat,"erome":Erome,"direct":Direct,"self":Self
|
||||
}
|
||||
|
||||
for i in range(subsLenght):
|
||||
print("\n({}/{})".format(i+1,subsLenght))
|
||||
print(
|
||||
"https://reddit.com/r/{subreddit}/comments/{id}".format(
|
||||
subreddit=submissions[i]['postSubreddit'],
|
||||
id=submissions[i]['postId']
|
||||
)
|
||||
)
|
||||
print()
|
||||
if SUBMISSION['postType'] in downloaders:
|
||||
|
||||
if postExists(submissions[i]):
|
||||
result = False
|
||||
print(submissions[i]['postType'].upper())
|
||||
print("It already exists")
|
||||
duplicates += 1
|
||||
downloadedCount -= 1
|
||||
continue
|
||||
|
||||
directory = GLOBAL.directory / submissions[i]['postSubreddit']
|
||||
|
||||
if submissions[i]['postType'] == 'imgur' and not 'imgur' in ToBeDownloaded:
|
||||
print("IMGUR",end="")
|
||||
if SUBMISSION['postType'] == "imgur":
|
||||
|
||||
while int(time.time() - lastRequestTime) <= 2:
|
||||
pass
|
||||
|
||||
credit = Imgur.get_credits()
|
||||
|
||||
IMGUR_RESET_TIME = credit['UserReset']-time.time()
|
||||
USER_RESET = ("after " \
|
||||
+ str(int(IMGUR_RESET_TIME/60)) \
|
||||
+ " Minutes " \
|
||||
+ str(int(IMGUR_RESET_TIME%60)) \
|
||||
+ " Seconds")
|
||||
+ str(int(IMGUR_RESET_TIME/60)) \
|
||||
+ " Minutes " \
|
||||
+ str(int(IMGUR_RESET_TIME%60)) \
|
||||
+ " Seconds")
|
||||
|
||||
if credit['ClientRemaining'] < 25 or credit['UserRemaining'] < 25:
|
||||
printCredit = {"noPrint":False}
|
||||
else:
|
||||
printCredit = {"noPrint":True}
|
||||
|
||||
print(
|
||||
" => Client: {} - User: {} - Reset {}".format(
|
||||
"==> Client: {} - User: {} - Reset {}\n".format(
|
||||
credit['ClientRemaining'],
|
||||
credit['UserRemaining'],
|
||||
USER_RESET
|
||||
)
|
||||
),end="",**printCredit
|
||||
)
|
||||
|
||||
if not (credit['UserRemaining'] == 0 or \
|
||||
@@ -536,94 +511,105 @@ def download(submissions):
|
||||
"""
|
||||
while int(time.time() - lastRequestTime) <= 2:
|
||||
pass
|
||||
|
||||
lastRequestTime = time.time()
|
||||
|
||||
try:
|
||||
Imgur(directory,submissions[i])
|
||||
|
||||
except FileAlreadyExistsError:
|
||||
print("It already exists")
|
||||
duplicates += 1
|
||||
downloadedCount -= 1
|
||||
|
||||
except ImgurLoginError:
|
||||
print(
|
||||
"Imgur login failed. Quitting the program "\
|
||||
"as unexpected errors might occur."
|
||||
)
|
||||
sys.exit()
|
||||
|
||||
except Exception as exception:
|
||||
print(exception)
|
||||
FAILED_FILE.add({int(i+1):[str(exception),submissions[i]]})
|
||||
downloadedCount -= 1
|
||||
|
||||
else:
|
||||
if credit['UserRemaining'] == 0:
|
||||
KEYWORD = "user"
|
||||
elif credit['ClientRemaining'] == 0:
|
||||
KEYWORD = "client"
|
||||
|
||||
print('{} LIMIT EXCEEDED\n'.format(KEYWORD.upper()))
|
||||
FAILED_FILE.add(
|
||||
{int(i+1):['{} LIMIT EXCEEDED\n'.format(KEYWORD.upper()),
|
||||
submissions[i]]}
|
||||
)
|
||||
downloadedCount -= 1
|
||||
raise ImgurLimitError('{} LIMIT EXCEEDED\n'.format(KEYWORD.upper()))
|
||||
|
||||
elif submissions[i]['postType'] == 'gfycat' and not 'gfycat' in ToBeDownloaded:
|
||||
print("GFYCAT")
|
||||
try:
|
||||
Gfycat(directory,submissions[i])
|
||||
downloaders[SUBMISSION['postType']] (directory,SUBMISSION)
|
||||
|
||||
except FileAlreadyExistsError:
|
||||
print("It already exists")
|
||||
duplicates += 1
|
||||
downloadedCount -= 1
|
||||
|
||||
except NotADownloadableLinkError as exception:
|
||||
print(exception)
|
||||
FAILED_FILE.add({int(i+1):[str(exception),submissions[i]]})
|
||||
downloadedCount -= 1
|
||||
else:
|
||||
raise NoSuitablePost
|
||||
|
||||
except Exception as exception:
|
||||
print(exception)
|
||||
FAILED_FILE.add({int(i+1):[str(exception),submissions[i]]})
|
||||
downloadedCount -= 1
|
||||
return None
|
||||
|
||||
elif submissions[i]['postType'] == 'direct' and not 'direct' in ToBeDownloaded:
|
||||
print("DIRECT")
|
||||
try:
|
||||
Direct(directory,submissions[i])
|
||||
def download(submissions):
|
||||
"""Analyze list of submissions and call the right function
|
||||
to download each one, catch errors, update the log files
|
||||
"""
|
||||
|
||||
except FileAlreadyExistsError:
|
||||
print("It already exists")
|
||||
downloadedCount -= 1
|
||||
duplicates += 1
|
||||
subsLenght = len(submissions)
|
||||
global lastRequestTime
|
||||
lastRequestTime = 0
|
||||
downloadedCount = subsLenght
|
||||
duplicates = 0
|
||||
|
||||
except Exception as exception:
|
||||
print(exception)
|
||||
FAILED_FILE.add({int(i+1):[str(exception),submissions[i]]})
|
||||
downloadedCount -= 1
|
||||
FAILED_FILE = createLogFile("FAILED")
|
||||
|
||||
for i in range(subsLenght):
|
||||
print(
|
||||
f"\n({i+1}/{subsLenght}) – {submissions[i]['postType'].upper()} " \
|
||||
f"– r/{submissions[i]['postSubreddit']}",end=""
|
||||
)
|
||||
|
||||
if isPostExists(submissions[i]):
|
||||
print("\nIt already exists")
|
||||
duplicates += 1
|
||||
downloadedCount -= 1
|
||||
continue
|
||||
|
||||
try:
|
||||
downloadPost(submissions[i])
|
||||
|
||||
elif submissions[i]['postType'] == 'self' and not 'self' in ToBeDownloaded:
|
||||
print("SELF")
|
||||
try:
|
||||
Self(directory,submissions[i])
|
||||
except FileAlreadyExistsError:
|
||||
print("It already exists")
|
||||
duplicates += 1
|
||||
downloadedCount -= 1
|
||||
|
||||
except FileAlreadyExistsError:
|
||||
print("It already exists")
|
||||
downloadedCount -= 1
|
||||
duplicates += 1
|
||||
except ImgurLoginError:
|
||||
print(
|
||||
"Imgur login failed. \nQuitting the program "\
|
||||
"as unexpected errors might occur."
|
||||
)
|
||||
sys.exit()
|
||||
|
||||
except Exception as exception:
|
||||
print(exception)
|
||||
FAILED_FILE.add({int(i+1):[str(exception),submissions[i]]})
|
||||
downloadedCount -= 1
|
||||
except ImgurLimitError as exception:
|
||||
FAILED_FILE.add({int(i+1):[
|
||||
"{class_name}: {info}".format(
|
||||
class_name=exception.__class__.__name__,info=str(exception)
|
||||
),
|
||||
submissions[i]
|
||||
]})
|
||||
downloadedCount -= 1
|
||||
|
||||
else:
|
||||
except NotADownloadableLinkError as exception:
|
||||
print(
|
||||
"{class_name}: {info}".format(
|
||||
class_name=exception.__class__.__name__,info=str(exception)
|
||||
)
|
||||
)
|
||||
FAILED_FILE.add({int(i+1):[
|
||||
"{class_name}: {info}".format(
|
||||
class_name=exception.__class__.__name__,info=str(exception)
|
||||
),
|
||||
submissions[i]
|
||||
]})
|
||||
downloadedCount -= 1
|
||||
|
||||
except NoSuitablePost:
|
||||
print("No match found, skipping...")
|
||||
downloadedCount -= 1
|
||||
|
||||
except Exception as exception:
|
||||
# raise exception
|
||||
print(
|
||||
"{class_name}: {info}".format(
|
||||
class_name=exception.__class__.__name__,info=str(exception)
|
||||
)
|
||||
)
|
||||
FAILED_FILE.add({int(i+1):[
|
||||
"{class_name}: {info}".format(
|
||||
class_name=exception.__class__.__name__,info=str(exception)
|
||||
),
|
||||
submissions[i]
|
||||
]})
|
||||
downloadedCount -= 1
|
||||
|
||||
if duplicates:
|
||||
print("\n There was {} duplicates".format(duplicates))
|
||||
@@ -642,15 +628,12 @@ def main():
|
||||
else:
|
||||
GLOBAL.directory = Path(input("download directory: "))
|
||||
|
||||
print("\n"," ".join(sys.argv),"\n")
|
||||
print("\n"," ".join(sys.argv),"\n",noPrint=True)
|
||||
|
||||
try:
|
||||
checkConflicts()
|
||||
except ProgramModeError as err:
|
||||
PromptUser()
|
||||
except Exception as err:
|
||||
print(err)
|
||||
sys.exit()
|
||||
|
||||
if not Path(GLOBAL.configDirectory).is_dir():
|
||||
os.makedirs(GLOBAL.configDirectory)
|
||||
@@ -660,7 +643,7 @@ def main():
|
||||
logDir = Path(GLOBAL.arguments.log)
|
||||
download(postFromLog(logDir))
|
||||
sys.exit()
|
||||
|
||||
|
||||
try:
|
||||
POSTS = getPosts(prepareAttributes())
|
||||
except InsufficientPermission:
|
||||
@@ -705,10 +688,11 @@ if __name__ == "__main__":
|
||||
print = printToFile
|
||||
GLOBAL.RUN_TIME = time.time()
|
||||
main()
|
||||
|
||||
except KeyboardInterrupt:
|
||||
if GLOBAL.directory is None:
|
||||
GLOBAL.directory = Path(".\\")
|
||||
print("\nQUITTING...")
|
||||
|
||||
except Exception as exception:
|
||||
if GLOBAL.directory is None:
|
||||
GLOBAL.directory = Path(".\\")
|
||||
@@ -716,4 +700,4 @@ if __name__ == "__main__":
|
||||
exc_info=full_exc_info(sys.exc_info()))
|
||||
print(log_stream.getvalue())
|
||||
|
||||
input("Press enter to quit\n")
|
||||
if not GLOBAL.arguments.quit: input("\nPress enter to quit\n")
|
||||
|
||||
@@ -2,7 +2,9 @@ import io
|
||||
import os
|
||||
import sys
|
||||
import urllib.request
|
||||
from html.parser import HTMLParser
|
||||
from pathlib import Path
|
||||
from urllib.error import HTTPError
|
||||
|
||||
import imgurpython
|
||||
from multiprocessing import Queue
|
||||
@@ -59,16 +61,150 @@ def getFile(fileDir,tempDir,imageURL,indent=0):
|
||||
tempDir,
|
||||
reporthook=dlProgress)
|
||||
os.rename(tempDir,fileDir)
|
||||
print(" "*indent+"Downloaded"+" "*10)
|
||||
break
|
||||
except ConnectionResetError as exception:
|
||||
print(" "*indent + str(exception))
|
||||
print(" "*indent + "Trying again\n")
|
||||
except FileNotFoundError:
|
||||
raise FileNameTooLong
|
||||
else:
|
||||
print(" "*indent+"Downloaded"+" "*10)
|
||||
break
|
||||
else:
|
||||
raise FileAlreadyExistsError
|
||||
|
||||
class Erome:
|
||||
def __init__(self,directory,post):
|
||||
try:
|
||||
IMAGES = self.getLinks(post['postURL'])
|
||||
except urllib.error.HTTPError:
|
||||
raise NotADownloadableLinkError("Not a downloadable link")
|
||||
|
||||
imagesLenght = len(IMAGES)
|
||||
howManyDownloaded = imagesLenght
|
||||
duplicates = 0
|
||||
|
||||
if imagesLenght == 1:
|
||||
|
||||
extension = getExtension(IMAGES[0])
|
||||
|
||||
title = nameCorrector(post['postTitle'])
|
||||
print(post["postSubmitter"]+"_"+title+"_"+post['postId']+extension)
|
||||
|
||||
fileDir = directory / (
|
||||
post["postSubmitter"]+"_"+title+"_"+post['postId']+extension
|
||||
)
|
||||
tempDir = directory / (
|
||||
post["postSubmitter"]+"_"+title+"_"+post['postId']+".tmp"
|
||||
)
|
||||
|
||||
imageURL = "https:" + IMAGES[0]
|
||||
|
||||
try:
|
||||
getFile(fileDir,tempDir,imageURL)
|
||||
except FileNameTooLong:
|
||||
fileDir = directory / (post['postId'] + extension)
|
||||
tempDir = directory / (post['postId'] + '.tmp')
|
||||
getFile(fileDir,tempDir,imageURL)
|
||||
|
||||
else:
|
||||
title = nameCorrector(post['postTitle'])
|
||||
print(post["postSubmitter"]+"_"+title+"_"+post['postId'],end="\n\n")
|
||||
|
||||
folderDir = directory / (
|
||||
post["postSubmitter"] + "_" + title + "_" + post['postId']
|
||||
)
|
||||
|
||||
try:
|
||||
if not os.path.exists(folderDir):
|
||||
os.makedirs(folderDir)
|
||||
except FileNotFoundError:
|
||||
folderDir = directory / post['postId']
|
||||
os.makedirs(folderDir)
|
||||
|
||||
for i in range(imagesLenght):
|
||||
|
||||
extension = getExtension(IMAGES[i])
|
||||
|
||||
fileName = str(i+1)
|
||||
imageURL = "https:" + IMAGES[i]
|
||||
|
||||
fileDir = folderDir / (fileName + extension)
|
||||
tempDir = folderDir / (fileName + ".tmp")
|
||||
|
||||
print(" ({}/{})".format(i+1,imagesLenght))
|
||||
print(" {}".format(fileName+extension))
|
||||
|
||||
try:
|
||||
getFile(fileDir,tempDir,imageURL,indent=2)
|
||||
print()
|
||||
except FileAlreadyExistsError:
|
||||
print(" The file already exists" + " "*10,end="\n\n")
|
||||
duplicates += 1
|
||||
howManyDownloaded -= 1
|
||||
|
||||
except Exception as exception:
|
||||
# raise exception
|
||||
print("\n Could not get the file")
|
||||
print(
|
||||
" "
|
||||
+ "{class_name}: {info}".format(
|
||||
class_name=exception.__class__.__name__,
|
||||
info=str(exception)
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
exceptionType = exception
|
||||
howManyDownloaded -= 1
|
||||
|
||||
if duplicates == imagesLenght:
|
||||
raise FileAlreadyExistsError
|
||||
elif howManyDownloaded + duplicates < imagesLenght:
|
||||
raise AlbumNotDownloadedCompletely(
|
||||
"Album Not Downloaded Completely"
|
||||
)
|
||||
|
||||
def getLinks(self,url,lineNumber=129):
|
||||
|
||||
content = []
|
||||
lineNumber = None
|
||||
|
||||
class EromeParser(HTMLParser):
|
||||
tag = None
|
||||
def handle_starttag(self, tag, attrs):
|
||||
self.tag = {tag:{attr[0]: attr[1] for attr in attrs}}
|
||||
|
||||
pageSource = (urllib.request.urlopen(url).read().decode().split('\n'))
|
||||
|
||||
""" FIND WHERE ALBUM STARTS IN ORDER NOT TO GET WRONG LINKS"""
|
||||
for i in range(len(pageSource)):
|
||||
obj = EromeParser()
|
||||
obj.feed(pageSource[i])
|
||||
tag = obj.tag
|
||||
|
||||
if tag is not None:
|
||||
if "div" in tag:
|
||||
if "id" in tag["div"]:
|
||||
if tag["div"]["id"] == "album":
|
||||
lineNumber = i
|
||||
break
|
||||
|
||||
for line in pageSource[lineNumber:]:
|
||||
obj = EromeParser()
|
||||
obj.feed(line)
|
||||
tag = obj.tag
|
||||
if tag is not None:
|
||||
if "img" in tag:
|
||||
if "class" in tag["img"]:
|
||||
if tag["img"]["class"]=="img-front":
|
||||
content.append(tag["img"]["src"])
|
||||
elif "source" in tag:
|
||||
content.append(tag["source"]["src"])
|
||||
|
||||
return [
|
||||
link for link in content \
|
||||
if link.endswith("_480p.mp4") or not link.endswith(".mp4")
|
||||
]
|
||||
|
||||
class Imgur:
|
||||
def __init__(self,directory,post):
|
||||
self.imgurClient = self.initImgur()
|
||||
@@ -88,13 +224,22 @@ class Imgur:
|
||||
post['postExt'] = getExtension(post['mediaURL'])
|
||||
|
||||
title = nameCorrector(post['postTitle'])
|
||||
print(title+"_" +post['postId']+post['postExt'])
|
||||
print(post["postSubmitter"]+"_"+title+"_"+post['postId']+post['postExt'])
|
||||
|
||||
fileDir = title + "_" + post['postId'] + post['postExt']
|
||||
fileDir = directory / fileDir
|
||||
fileDir = directory / (
|
||||
post["postSubmitter"]
|
||||
+ "_" + title
|
||||
+ "_" + post['postId']
|
||||
+ post['postExt']
|
||||
)
|
||||
|
||||
tempDir = directory / (
|
||||
post["postSubmitter"]
|
||||
+ "_" + title
|
||||
+ "_" + post['postId']
|
||||
+ ".tmp"
|
||||
)
|
||||
|
||||
tempDir = title + "_" + post['postId'] + '.tmp'
|
||||
tempDir = directory / tempDir
|
||||
try:
|
||||
getFile(fileDir,tempDir,post['mediaURL'])
|
||||
except FileNameTooLong:
|
||||
@@ -110,9 +255,11 @@ class Imgur:
|
||||
duplicates = 0
|
||||
|
||||
title = nameCorrector(post['postTitle'])
|
||||
print(title+"_"+post['postId'],end="\n\n")
|
||||
print(post["postSubmitter"]+"_"+title+"_"+post['postId'],end="\n\n")
|
||||
|
||||
folderDir = directory / (title+"_"+post['postId'])
|
||||
folderDir = directory / (
|
||||
post["postSubmitter"] + "_" + title + "_" + post['postId']
|
||||
)
|
||||
|
||||
try:
|
||||
if not os.path.exists(folderDir):
|
||||
@@ -165,13 +312,20 @@ class Imgur:
|
||||
|
||||
except Exception as exception:
|
||||
print("\n Could not get the file")
|
||||
print(" " + str(exception) + "\n")
|
||||
print(
|
||||
" "
|
||||
+ "{class_name}: {info}".format(
|
||||
class_name=exception.__class__.__name__,
|
||||
info=str(exception)
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
exceptionType = exception
|
||||
howManyDownloaded -= 1
|
||||
|
||||
if duplicates == imagesLenght:
|
||||
raise FileAlreadyExistsError
|
||||
elif howManyDownloaded < imagesLenght:
|
||||
elif howManyDownloaded + duplicates < imagesLenght:
|
||||
raise AlbumNotDownloadedCompletely(
|
||||
"Album Not Downloaded Completely"
|
||||
)
|
||||
@@ -230,10 +384,15 @@ class Gfycat:
|
||||
|
||||
if not os.path.exists(directory): os.makedirs(directory)
|
||||
title = nameCorrector(POST['postTitle'])
|
||||
print(title+"_"+POST['postId']+POST['postExt'])
|
||||
print(POST["postSubmitter"]+"_"+title+"_"+POST['postId']+POST['postExt'])
|
||||
|
||||
fileDir = directory / (title+"_"+POST['postId']+POST['postExt'])
|
||||
tempDir = directory / (title+"_"+POST['postId']+".tmp")
|
||||
fileDir = directory / (
|
||||
POST["postSubmitter"]+"_"+title+"_"+POST['postId']+POST['postExt']
|
||||
)
|
||||
tempDir = directory / (
|
||||
POST["postSubmitter"]+"_"+title+"_"+POST['postId']+".tmp"
|
||||
)
|
||||
|
||||
try:
|
||||
getFile(fileDir,tempDir,POST['mediaURL'])
|
||||
except FileNameTooLong:
|
||||
@@ -279,13 +438,14 @@ class Direct:
|
||||
POST['postExt'] = getExtension(POST['postURL'])
|
||||
if not os.path.exists(directory): os.makedirs(directory)
|
||||
title = nameCorrector(POST['postTitle'])
|
||||
print(title+"_"+POST['postId']+POST['postExt'])
|
||||
print(POST["postSubmitter"]+"_"+title+"_"+POST['postId']+POST['postExt'])
|
||||
|
||||
fileDir = title+"_"+POST['postId']+POST['postExt']
|
||||
fileDir = directory / fileDir
|
||||
|
||||
tempDir = title+"_"+POST['postId']+".tmp"
|
||||
tempDir = directory / tempDir
|
||||
fileDir = directory / (
|
||||
POST["postSubmitter"]+"_"+title+"_"+POST['postId']+POST['postExt']
|
||||
)
|
||||
tempDir = directory / (
|
||||
POST["postSubmitter"]+"_"+title+"_"+POST['postId']+".tmp"
|
||||
)
|
||||
|
||||
try:
|
||||
getFile(fileDir,tempDir,POST['postURL'])
|
||||
@@ -300,10 +460,11 @@ class Self:
|
||||
if not os.path.exists(directory): os.makedirs(directory)
|
||||
|
||||
title = nameCorrector(post['postTitle'])
|
||||
print(title+"_"+post['postId']+".md")
|
||||
print(post["postSubmitter"]+"_"+title+"_"+post['postId']+".md")
|
||||
|
||||
fileDir = title+"_"+post['postId']+".md"
|
||||
fileDir = directory / fileDir
|
||||
fileDir = directory / (
|
||||
post["postSubmitter"]+"_"+title+"_"+post['postId']+".md"
|
||||
)
|
||||
|
||||
if Path.is_file(fileDir):
|
||||
raise FileAlreadyExistsError
|
||||
@@ -326,7 +487,11 @@ class Self:
|
||||
+ ")\n"
|
||||
+ post["postContent"]
|
||||
+ "\n\n---\n\n"
|
||||
+ "submitted by [u/"
|
||||
+ "submitted to [r/"
|
||||
+ post["postSubreddit"]
|
||||
+ "](https://www.reddit.com/r/"
|
||||
+ post["postSubreddit"]
|
||||
+ ") by [u/"
|
||||
+ post["postSubmitter"]
|
||||
+ "](https://www.reddit.com/user/"
|
||||
+ post["postSubmitter"]
|
||||
|
||||
@@ -80,4 +80,10 @@ class InvalidSortingType(Exception):
|
||||
pass
|
||||
|
||||
class FileNotFoundError(Exception):
|
||||
pass
|
||||
|
||||
class NoSuitablePost(Exception):
|
||||
pass
|
||||
|
||||
class ImgurLimitError(Exception):
|
||||
pass
|
||||
228
src/searcher.py
228
src/searcher.py
@@ -1,4 +1,5 @@
|
||||
import os
|
||||
import sys
|
||||
import random
|
||||
import socket
|
||||
import webbrowser
|
||||
@@ -14,60 +15,62 @@ from src.errors import (NoMatchingSubmissionFound, NoPrawSupport,
|
||||
|
||||
print = printToFile
|
||||
|
||||
class GetAuth:
|
||||
def __init__(self,redditInstance,port):
|
||||
self.redditInstance = redditInstance
|
||||
self.PORT = int(port)
|
||||
|
||||
def recieve_connection(self):
|
||||
"""Wait for and then return a connected socket..
|
||||
Opens a TCP connection on port 8080, and waits for a single client.
|
||||
"""
|
||||
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
server.bind(('localhost', self.PORT))
|
||||
server.listen(1)
|
||||
client = server.accept()[0]
|
||||
server.close()
|
||||
return client
|
||||
|
||||
def send_message(self, message):
|
||||
"""Send message to client and close the connection."""
|
||||
self.client.send('HTTP/1.1 200 OK\r\n\r\n{}'.format(message).encode('utf-8'))
|
||||
self.client.close()
|
||||
|
||||
def getRefreshToken(self,*scopes):
|
||||
state = str(random.randint(0, 65000))
|
||||
url = self.redditInstance.auth.url(scopes, state, 'permanent')
|
||||
print("Go to this URL and login to reddit:\n\n",url)
|
||||
webbrowser.open(url,new=2)
|
||||
|
||||
self.client = self.recieve_connection()
|
||||
data = self.client.recv(1024).decode('utf-8')
|
||||
param_tokens = data.split(' ', 2)[1].split('?', 1)[1].split('&')
|
||||
params = {
|
||||
key: value for (key, value) in [token.split('=') \
|
||||
for token in param_tokens]
|
||||
}
|
||||
if state != params['state']:
|
||||
self.send_message(
|
||||
client, 'State mismatch. Expected: {} Received: {}'
|
||||
.format(state, params['state'])
|
||||
)
|
||||
raise RedditLoginFailed
|
||||
elif 'error' in params:
|
||||
self.send_message(client, params['error'])
|
||||
raise RedditLoginFailed
|
||||
|
||||
refresh_token = self.redditInstance.auth.authorize(params['code'])
|
||||
self.send_message(
|
||||
"<script>" \
|
||||
"alert(\"You can go back to terminal window now.\");" \
|
||||
"</script>"
|
||||
)
|
||||
return (self.redditInstance,refresh_token)
|
||||
|
||||
def beginPraw(config,user_agent = str(socket.gethostname())):
|
||||
class GetAuth:
|
||||
def __init__(self,redditInstance,port):
|
||||
self.redditInstance = redditInstance
|
||||
self.PORT = int(port)
|
||||
|
||||
def recieve_connection(self):
|
||||
"""Wait for and then return a connected socket..
|
||||
Opens a TCP connection on port 8080, and waits for a single client.
|
||||
"""
|
||||
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
server.bind(('localhost', self.PORT))
|
||||
server.listen(1)
|
||||
client = server.accept()[0]
|
||||
server.close()
|
||||
return client
|
||||
|
||||
def send_message(self, message):
|
||||
"""Send message to client and close the connection."""
|
||||
self.client.send(
|
||||
'HTTP/1.1 200 OK\r\n\r\n{}'.format(message).encode('utf-8')
|
||||
)
|
||||
self.client.close()
|
||||
|
||||
def getRefreshToken(self,*scopes):
|
||||
state = str(random.randint(0, 65000))
|
||||
url = self.redditInstance.auth.url(scopes, state, 'permanent')
|
||||
print("Go to this URL and login to reddit:\n\n",url)
|
||||
webbrowser.open(url,new=2)
|
||||
|
||||
self.client = self.recieve_connection()
|
||||
data = self.client.recv(1024).decode('utf-8')
|
||||
param_tokens = data.split(' ', 2)[1].split('?', 1)[1].split('&')
|
||||
params = {
|
||||
key: value for (key, value) in [token.split('=') \
|
||||
for token in param_tokens]
|
||||
}
|
||||
if state != params['state']:
|
||||
self.send_message(
|
||||
client, 'State mismatch. Expected: {} Received: {}'
|
||||
.format(state, params['state'])
|
||||
)
|
||||
raise RedditLoginFailed
|
||||
elif 'error' in params:
|
||||
self.send_message(client, params['error'])
|
||||
raise RedditLoginFailed
|
||||
|
||||
refresh_token = self.redditInstance.auth.authorize(params['code'])
|
||||
self.send_message(
|
||||
"<script>" \
|
||||
"alert(\"You can go back to terminal window now.\");" \
|
||||
"</script>"
|
||||
)
|
||||
return (self.redditInstance,refresh_token)
|
||||
|
||||
"""Start reddit instance"""
|
||||
|
||||
scopes = ['identity','history','read']
|
||||
@@ -123,8 +126,6 @@ def getPosts(args):
|
||||
if args["user"] == "me":
|
||||
args["user"] = str(reddit.user.me())
|
||||
|
||||
print("\nGETTING POSTS\n.\n.\n.\n")
|
||||
|
||||
if not "search" in args:
|
||||
if args["sort"] == "top" or args["sort"] == "controversial":
|
||||
keyword_params = {
|
||||
@@ -156,7 +157,7 @@ def getPosts(args):
|
||||
sort=args["sort"],
|
||||
subreddit=args["subreddit"],
|
||||
time=args["time"]
|
||||
).upper()
|
||||
).upper(),noPrint=True
|
||||
)
|
||||
return redditSearcher(
|
||||
reddit.subreddit(args["subreddit"]).search(
|
||||
@@ -184,7 +185,7 @@ def getPosts(args):
|
||||
"saved posts\nuser:{username}\nlimit={limit}\n".format(
|
||||
username=reddit.user.me(),
|
||||
limit=args["limit"]
|
||||
).upper()
|
||||
).upper(),noPrint=True
|
||||
)
|
||||
return redditSearcher(reddit.user.me().saved(limit=args["limit"]))
|
||||
|
||||
@@ -199,7 +200,7 @@ def getPosts(args):
|
||||
sort=args["sort"],
|
||||
subreddit=args["subreddit"],
|
||||
time=args["time"]
|
||||
).upper()
|
||||
).upper(),noPrint=True
|
||||
)
|
||||
return redditSearcher(
|
||||
getattr(reddit.front,args["sort"]) (**keyword_params)
|
||||
@@ -213,7 +214,7 @@ def getPosts(args):
|
||||
sort=args["sort"],
|
||||
subreddit=args["subreddit"],
|
||||
time=args["time"]
|
||||
).upper()
|
||||
).upper(),noPrint=True
|
||||
)
|
||||
return redditSearcher(
|
||||
getattr(
|
||||
@@ -231,7 +232,7 @@ def getPosts(args):
|
||||
sort=args["sort"],
|
||||
multireddit=args["multireddit"],
|
||||
time=args["time"]
|
||||
).upper()
|
||||
).upper(),noPrint=True
|
||||
)
|
||||
try:
|
||||
return redditSearcher(
|
||||
@@ -245,8 +246,6 @@ def getPosts(args):
|
||||
raise MultiredditNotFound
|
||||
|
||||
elif "submitted" in args:
|
||||
# TODO
|
||||
# USE REDDIT.USER.ME() INSTEAD WHEN "ME" PASSED AS A --USER
|
||||
print (
|
||||
"submitted posts of {user}\nsort: {sort}\n" \
|
||||
"time: {time}\nlimit: {limit}\n".format(
|
||||
@@ -254,7 +253,7 @@ def getPosts(args):
|
||||
sort=args["sort"],
|
||||
user=args["user"],
|
||||
time=args["time"]
|
||||
).upper()
|
||||
).upper(),noPrint=True
|
||||
)
|
||||
return redditSearcher(
|
||||
getattr(
|
||||
@@ -263,13 +262,11 @@ def getPosts(args):
|
||||
)
|
||||
|
||||
elif "upvoted" in args:
|
||||
# TODO
|
||||
# USE REDDIT.USER.ME() INSTEAD WHEN "ME" PASSED AS A --USER
|
||||
print (
|
||||
"upvoted posts of {user}\nlimit: {limit}\n".format(
|
||||
user=args["user"],
|
||||
limit=args["limit"]
|
||||
).upper()
|
||||
).upper(),noPrint=True
|
||||
)
|
||||
try:
|
||||
return redditSearcher(
|
||||
@@ -279,7 +276,7 @@ def getPosts(args):
|
||||
raise InsufficientPermission
|
||||
|
||||
elif "post" in args:
|
||||
print("post: {post}\n".format(post=args["post"]).upper())
|
||||
print("post: {post}\n".format(post=args["post"]).upper(),noPrint=True)
|
||||
return redditSearcher(
|
||||
reddit.submission(url=args["post"]),SINGLE_POST=True
|
||||
)
|
||||
@@ -299,6 +296,8 @@ def redditSearcher(posts,SINGLE_POST=False):
|
||||
gfycatCount = 0
|
||||
global imgurCount
|
||||
imgurCount = 0
|
||||
global eromeCount
|
||||
eromeCount = 0
|
||||
global directCount
|
||||
directCount = 0
|
||||
global selfCount
|
||||
@@ -306,6 +305,8 @@ def redditSearcher(posts,SINGLE_POST=False):
|
||||
|
||||
allPosts = {}
|
||||
|
||||
print("\nGETTING POSTS")
|
||||
if GLOBAL.arguments.verbose: print("\n")
|
||||
postsFile = createLogFile("POSTS")
|
||||
|
||||
if SINGLE_POST:
|
||||
@@ -326,43 +327,62 @@ def redditSearcher(posts,SINGLE_POST=False):
|
||||
if result is not None:
|
||||
details = result
|
||||
orderCount += 1
|
||||
printSubmission(submission,subCount,orderCount)
|
||||
if GLOBAL.arguments.verbose:
|
||||
printSubmission(submission,subCount,orderCount)
|
||||
subList.append(details)
|
||||
|
||||
postsFile.add({subCount:[details]})
|
||||
|
||||
else:
|
||||
for submission in posts:
|
||||
subCount += 1
|
||||
try:
|
||||
for submission in posts:
|
||||
subCount += 1
|
||||
|
||||
try:
|
||||
details = {'postId':submission.id,
|
||||
'postTitle':submission.title,
|
||||
'postSubmitter':str(submission.author),
|
||||
'postType':None,
|
||||
'postURL':submission.url,
|
||||
'postSubreddit':submission.subreddit.display_name}
|
||||
except AttributeError:
|
||||
continue
|
||||
if subCount % 100 == 0 and not GLOBAL.arguments.verbose:
|
||||
sys.stdout.write("• ")
|
||||
sys.stdout.flush()
|
||||
|
||||
result = checkIfMatching(submission)
|
||||
if subCount % 1000 == 0:
|
||||
sys.stdout.write("\n"+" "*14)
|
||||
sys.stdout.flush()
|
||||
|
||||
if result is not None:
|
||||
details = result
|
||||
orderCount += 1
|
||||
printSubmission(submission,subCount,orderCount)
|
||||
subList.append(details)
|
||||
try:
|
||||
details = {'postId':submission.id,
|
||||
'postTitle':submission.title,
|
||||
'postSubmitter':str(submission.author),
|
||||
'postType':None,
|
||||
'postURL':submission.url,
|
||||
'postSubreddit':submission.subreddit.display_name}
|
||||
except AttributeError:
|
||||
continue
|
||||
|
||||
allPosts[subCount] = [details]
|
||||
result = checkIfMatching(submission)
|
||||
|
||||
if result is not None:
|
||||
details = result
|
||||
orderCount += 1
|
||||
if GLOBAL.arguments.verbose:
|
||||
printSubmission(submission,subCount,orderCount)
|
||||
subList.append(details)
|
||||
|
||||
allPosts[subCount] = [details]
|
||||
except KeyboardInterrupt:
|
||||
print("\nKeyboardInterrupt",end="")
|
||||
|
||||
postsFile.add(allPosts)
|
||||
|
||||
if not len(subList) == 0:
|
||||
print(
|
||||
"\nTotal of {} submissions found!\n"\
|
||||
"{} GFYCATs, {} IMGURs, {} DIRECTs and {} SELF POSTS\n"
|
||||
.format(len(subList),gfycatCount,imgurCount,directCount,selfCount)
|
||||
)
|
||||
if not len(subList) == 0:
|
||||
if GLOBAL.arguments.NoDownload or GLOBAL.arguments.verbose:
|
||||
print(
|
||||
f"\n\nTotal of {len(subList)} submissions found!"
|
||||
)
|
||||
print(
|
||||
f"{gfycatCount} GFYCATs, {imgurCount} IMGURs, " \
|
||||
f"{eromeCount} EROMEs, {directCount} DIRECTs " \
|
||||
f"and {selfCount} SELF POSTS",noPrint=True
|
||||
)
|
||||
else:
|
||||
print()
|
||||
return subList
|
||||
else:
|
||||
raise NoMatchingSubmissionFound
|
||||
@@ -370,6 +390,7 @@ def redditSearcher(posts,SINGLE_POST=False):
|
||||
def checkIfMatching(submission):
|
||||
global gfycatCount
|
||||
global imgurCount
|
||||
global eromeCount
|
||||
global directCount
|
||||
global selfCount
|
||||
|
||||
@@ -383,19 +404,20 @@ def checkIfMatching(submission):
|
||||
except AttributeError:
|
||||
return None
|
||||
|
||||
if ('gfycat' in submission.domain) or \
|
||||
('imgur' in submission.domain):
|
||||
if 'gfycat' in submission.domain:
|
||||
details['postType'] = 'gfycat'
|
||||
gfycatCount += 1
|
||||
return details
|
||||
|
||||
if 'gfycat' in submission.domain:
|
||||
details['postType'] = 'gfycat'
|
||||
gfycatCount += 1
|
||||
return details
|
||||
elif 'imgur' in submission.domain:
|
||||
details['postType'] = 'imgur'
|
||||
imgurCount += 1
|
||||
return details
|
||||
|
||||
elif 'imgur' in submission.domain:
|
||||
details['postType'] = 'imgur'
|
||||
|
||||
imgurCount += 1
|
||||
return details
|
||||
elif 'erome' in submission.domain:
|
||||
details['postType'] = 'erome'
|
||||
eromeCount += 1
|
||||
return details
|
||||
|
||||
elif isDirectLink(submission.url) is not False:
|
||||
details['postType'] = 'direct'
|
||||
|
||||
22
src/tools.py
22
src/tools.py
@@ -90,7 +90,7 @@ def createLogFile(TITLE):
|
||||
|
||||
return FILE
|
||||
|
||||
def printToFile(*args, **kwargs):
|
||||
def printToFile(*args, noPrint=False,**kwargs):
|
||||
"""Print to both CONSOLE and
|
||||
CONSOLE LOG file in a folder time stampt in the name
|
||||
"""
|
||||
@@ -98,15 +98,21 @@ def printToFile(*args, **kwargs):
|
||||
TIME = str(time.strftime("%d-%m-%Y_%H-%M-%S",
|
||||
time.localtime(GLOBAL.RUN_TIME)))
|
||||
folderDirectory = GLOBAL.directory / "LOG_FILES" / TIME
|
||||
print(*args,**kwargs)
|
||||
|
||||
if not noPrint or \
|
||||
GLOBAL.arguments.verbose or \
|
||||
"file" in kwargs:
|
||||
|
||||
print(*args,**kwargs)
|
||||
|
||||
if not path.exists(folderDirectory):
|
||||
makedirs(folderDirectory)
|
||||
|
||||
with io.open(
|
||||
folderDirectory / "CONSOLE_LOG.txt","a",encoding="utf-8"
|
||||
) as FILE:
|
||||
print(*args, file=FILE, **kwargs)
|
||||
|
||||
if not "file" in kwargs:
|
||||
with io.open(
|
||||
folderDirectory / "CONSOLE_LOG.txt","a",encoding="utf-8"
|
||||
) as FILE:
|
||||
print(*args, file=FILE, **kwargs)
|
||||
|
||||
def nameCorrector(string):
|
||||
"""Swap strange characters from given string
|
||||
@@ -132,7 +138,7 @@ def nameCorrector(string):
|
||||
if len(string.split('\n')) > 1:
|
||||
string = "".join(string.split('\n'))
|
||||
|
||||
BAD_CHARS = ['\\','/',':','*','?','"','<','>','|','.',]
|
||||
BAD_CHARS = ['\\','/',':','*','?','"','<','>','|','.','#']
|
||||
|
||||
if any(x in string for x in BAD_CHARS):
|
||||
for char in string:
|
||||
|
||||
Reference in New Issue
Block a user