You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
34 lines
835 B
34 lines
835 B
import time
|
|
import requests
|
|
import html.parser
|
|
|
|
def safe_request(session, url, method="GET", *args, **kwargs):
|
|
tries = 0
|
|
while True:
|
|
# try to connect to remote source
|
|
try:
|
|
r = session.request(method, url, *args, **kwargs)
|
|
except requests.exceptions.ConnectionError:
|
|
tries += 1
|
|
time.sleep(1)
|
|
if tries == 5:
|
|
raise
|
|
continue
|
|
|
|
# reject error-status-codes
|
|
if r.status_code != requests.codes.ok:
|
|
tries += 1
|
|
time.sleep(1)
|
|
if tries == 5:
|
|
r.raise_for_status()
|
|
continue
|
|
|
|
# everything ok -- proceed to download
|
|
return r
|
|
|
|
def filename_from_url(url):
|
|
pos = url.rfind("/")
|
|
return url[pos+1:]
|
|
|
|
unescape = html.parser.HTMLParser().unescape
|