wattpad-ebook-scraper
wattpad-ebook-scraper copied to clipboard
de3sw2aq1
be nice and not thrash wattpad with fast downloads.
*** scrape.py.ORIG 2015-11-24 11:06:25.000000000 +0100
--- scrape.py 2015-11-24 11:08:04.000000000 +0100
***************
*** 7,12 ****
--- 7,13 ----
import requests
import dateutil.parser
from genshi.input import HTML
+ import time
import random
import smartypants
**************_
**_ 86,91 ****
--- 87,95 ----
chapter_html = session.get(API_STORYTEXT, params={'id': chapter_id, 'output': 'json'}).json()['text']
chapter_html = smartypants.smartypants(chapter_html)
\+ # be nice and not thrash wattpad with fast downloads
\+ time.sleep(random.uniform(0.5, 1.5))
\+
section = ez_epub.Section()
section.html = HTML(chapter_html, encoding='utf-8')
section.title = chapter_title
~
