You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
103 lines
3.3 KiB
103 lines
3.3 KiB
# -*- coding: utf-8 -*-
|
|
|
|
# Copyright 2014-2017 Mike Fährmann
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License version 2 as
|
|
# published by the Free Software Foundation.
|
|
|
|
"""Downloader module for http:// and https:// urls"""
|
|
|
|
import time
|
|
import requests
|
|
import requests.exceptions as rexcepts
|
|
import mimetypes
|
|
from .common import BasicDownloader
|
|
from .. import config
|
|
|
|
|
|
class Downloader(BasicDownloader):
|
|
|
|
def __init__(self, output):
|
|
BasicDownloader.__init__(self)
|
|
self.session = requests.session()
|
|
self.out = output
|
|
self.max_tries = config.get(("retries",), 5)
|
|
self.timeout = config.get(("timeout",), None)
|
|
|
|
def download_impl(self, url, pathfmt):
|
|
tries = 0
|
|
msg = ""
|
|
while True:
|
|
tries += 1
|
|
if tries > 1:
|
|
self.out.error(pathfmt.path, msg, tries-1, self.max_tries)
|
|
if tries > self.max_tries:
|
|
return
|
|
time.sleep(1)
|
|
|
|
# try to connect to remote source
|
|
try:
|
|
response = self.session.get(
|
|
url, stream=True, timeout=self.timeout
|
|
)
|
|
except (rexcepts.ConnectionError, rexcepts.Timeout) as exception:
|
|
msg = exception
|
|
continue
|
|
except (rexcepts.RequestException, UnicodeError) as exception:
|
|
msg = exception
|
|
break
|
|
|
|
# reject error-status-codes
|
|
if response.status_code != 200:
|
|
msg = 'HTTP status "{} {}"'.format(
|
|
response.status_code, response.reason
|
|
)
|
|
response.close()
|
|
if response.status_code == 404:
|
|
break
|
|
continue
|
|
|
|
if not pathfmt.has_extension:
|
|
# set 'extension' keyword from Content-Type header
|
|
mtype = response.headers.get("Content-Type", "image/jpeg")
|
|
exts = mimetypes.guess_all_extensions(mtype, strict=False)
|
|
exts.sort()
|
|
pathfmt.set_extension(exts[-1][1:])
|
|
if pathfmt.exists():
|
|
self.out.skip(pathfmt.path)
|
|
response.close()
|
|
return
|
|
|
|
# everything ok -- proceed to download
|
|
self.out.start(pathfmt.path)
|
|
self.downloading = True
|
|
try:
|
|
with pathfmt.open() as file:
|
|
for data in response.iter_content(16384):
|
|
file.write(data)
|
|
except rexcepts.RequestException as exception:
|
|
msg = exception
|
|
response.close()
|
|
continue
|
|
self.downloading = False
|
|
self.out.success(pathfmt.path, tries)
|
|
return
|
|
|
|
# output for unrecoverable errors
|
|
self.out.error(pathfmt.path, msg, tries, 0)
|
|
|
|
def set_headers(self, headers):
|
|
"""Set headers for http requests"""
|
|
self.set_dict(self.session.headers, headers)
|
|
|
|
def set_cookies(self, cookies):
|
|
"""Set cookies for http requests"""
|
|
self.set_dict(self.session.cookies, cookies)
|
|
|
|
@staticmethod
|
|
def set_dict(dest, src):
|
|
"""Copy the contents of dictionary 'src' to 'dest'"""
|
|
dest.clear()
|
|
dest.update(src)
|