From 8c7ee1fdd919f0e9c995d4c1a53c9eb2198412ef Mon Sep 17 00:00:00 2001 From: sigvartmh Date: Fri, 12 May 2017 15:27:51 +0200 Subject: [PATCH] try to prep for dist --- app.py | 1 + itl_scrape.py | 19 ++++++++++--------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/app.py b/app.py index 04b5cff..64c706c 100644 --- a/app.py +++ b/app.py @@ -1,4 +1,5 @@ import appJar +import itl_scrape from appJar import gui from itl_scrape import itslearning_scraper diff --git a/itl_scrape.py b/itl_scrape.py index bc61be5..7a1a25f 100644 --- a/itl_scrape.py +++ b/itl_scrape.py @@ -4,7 +4,9 @@ import re import requests as rq import mechanicalsoup as ms import html2text - +import multiprocessing +import bs4 +import getpass from multiprocessing import Process from bs4 import BeautifulSoup as bs from getpass import getpass @@ -250,7 +252,7 @@ class itslearning_scraper(): if not title: title = "Failed to name"+str(self.failure) self.failure +=1 - make_folder(itl_path, title) + make_folder(itl_path, str(title)) r = rq.get(base_url+link.get("href"), cookies=self.cookies) table = self.find_folder_table(r.text) #print(table) @@ -262,7 +264,7 @@ class itslearning_scraper(): print("read_essay:",link.get("href")) itl_path = os.path.join(os.path.abspath(os.path.curdir)) title = link.contents[0] - make_folder(itl_path, title) + make_folder(itl_path, str(title)) r = rq.get(base_url+link.get("href"), cookies=self.cookies) self.find_essay_files(r.text) os.chdir('..') @@ -321,9 +323,8 @@ class itslearning_scraper(): if __name__ == '__main__': scraper = itslearning_scraper() scraper.enter() - scraper.find_all_courses() - #url = input("Enter course url or press enter to download all active courses:") - #if url: - # scraper.download_one(url) - #else: - # scraper.download_all() + url = input("Enter course url or press enter to download all active courses:") + if url: + scraper.download_one(url) + else: + scraper.download_all()