You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
252 lines
7.9 KiB
252 lines
7.9 KiB
# -*- coding: utf-8 -*-
|
|
|
|
# Copyright 2014-2019 Mike Fährmann
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License version 2 as
|
|
# published by the Free Software Foundation.
|
|
|
|
"""Downloader module for http:// and https:// URLs"""
|
|
|
|
import os
|
|
import time
|
|
import mimetypes
|
|
from requests.exceptions import RequestException, ConnectionError, Timeout
|
|
from .common import DownloaderBase
|
|
from .. import text
|
|
|
|
try:
|
|
from OpenSSL.SSL import Error as SSLError
|
|
except ImportError:
|
|
from ssl import SSLError
|
|
|
|
|
|
class HttpDownloader(DownloaderBase):
|
|
scheme = "http"
|
|
|
|
def __init__(self, extractor, output):
|
|
DownloaderBase.__init__(self, extractor, output)
|
|
self.adjust_extension = self.config("adjust-extensions", True)
|
|
self.retries = self.config("retries", extractor._retries)
|
|
self.timeout = self.config("timeout", extractor._timeout)
|
|
self.verify = self.config("verify", extractor._verify)
|
|
self.mtime = self.config("mtime", True)
|
|
self.rate = self.config("rate")
|
|
self.downloading = False
|
|
self.chunk_size = 16384
|
|
|
|
if self.retries < 0:
|
|
self.retries = float("inf")
|
|
if self.rate:
|
|
rate = text.parse_bytes(self.rate)
|
|
if not rate:
|
|
self.log.warning("Invalid rate limit (%r)", self.rate)
|
|
elif rate < self.chunk_size:
|
|
self.chunk_size = rate
|
|
self.rate = rate
|
|
|
|
def download(self, url, pathfmt):
|
|
try:
|
|
return self._download_impl(url, pathfmt)
|
|
except Exception:
|
|
print()
|
|
raise
|
|
finally:
|
|
# remove file from incomplete downloads
|
|
if self.downloading and not self.part:
|
|
try:
|
|
os.unlink(pathfmt.temppath)
|
|
except (OSError, AttributeError):
|
|
pass
|
|
|
|
def _download_impl(self, url, pathfmt):
|
|
response = None
|
|
tries = 0
|
|
msg = ""
|
|
|
|
if self.part:
|
|
pathfmt.part_enable(self.partdir)
|
|
|
|
while True:
|
|
if tries:
|
|
if response:
|
|
response.close()
|
|
self.log.warning("%s (%s/%s)", msg, tries, self.retries+1)
|
|
if tries > self.retries:
|
|
return False
|
|
time.sleep(min(2 ** (tries-1), 1800))
|
|
tries += 1
|
|
|
|
# check for .part file
|
|
filesize = pathfmt.part_size()
|
|
if filesize:
|
|
headers = {"Range": "bytes={}-".format(filesize)}
|
|
else:
|
|
headers = None
|
|
|
|
# connect to (remote) source
|
|
try:
|
|
response = self.session.request(
|
|
"GET", url, stream=True, headers=headers,
|
|
timeout=self.timeout, verify=self.verify)
|
|
except (ConnectionError, Timeout) as exc:
|
|
msg = str(exc)
|
|
continue
|
|
except Exception as exc:
|
|
self.log.warning("%s", exc)
|
|
return False
|
|
|
|
# check response
|
|
code = response.status_code
|
|
if code == 200: # OK
|
|
offset = 0
|
|
size = response.headers.get("Content-Length")
|
|
elif code == 206: # Partial Content
|
|
offset = filesize
|
|
size = response.headers["Content-Range"].rpartition("/")[2]
|
|
elif code == 416 and filesize: # Requested Range Not Satisfiable
|
|
break
|
|
else:
|
|
msg = "{}: {} for url: {}".format(code, response.reason, url)
|
|
if code == 429 or 500 <= code < 600: # Server Error
|
|
continue
|
|
self.log.warning("%s", msg)
|
|
return False
|
|
size = text.parse_int(size)
|
|
|
|
# set missing filename extension
|
|
if not pathfmt.extension:
|
|
pathfmt.set_extension(self.get_extension(response))
|
|
if pathfmt.exists():
|
|
pathfmt.temppath = ""
|
|
return True
|
|
|
|
# set open mode
|
|
if not offset:
|
|
mode = "w+b"
|
|
if filesize:
|
|
self.log.debug("Unable to resume partial download")
|
|
else:
|
|
mode = "r+b"
|
|
self.log.debug("Resuming download at byte %d", offset)
|
|
|
|
# start downloading
|
|
self.out.start(pathfmt.path)
|
|
self.downloading = True
|
|
with pathfmt.open(mode) as file:
|
|
if offset:
|
|
file.seek(offset)
|
|
|
|
# download content
|
|
try:
|
|
self.receive(response, file)
|
|
except (RequestException, SSLError) as exc:
|
|
msg = str(exc)
|
|
print()
|
|
continue
|
|
|
|
# check filesize
|
|
if size and file.tell() < size:
|
|
msg = "filesize mismatch ({} < {})".format(
|
|
file.tell(), size)
|
|
print()
|
|
continue
|
|
|
|
# check filename extension
|
|
if self.adjust_extension:
|
|
adj_ext = self.check_extension(file, pathfmt.extension)
|
|
if adj_ext:
|
|
pathfmt.set_extension(adj_ext)
|
|
|
|
break
|
|
|
|
self.downloading = False
|
|
if self.mtime:
|
|
pathfmt.kwdict["_mtime"] = response.headers.get("Last-Modified")
|
|
return True
|
|
|
|
def receive(self, response, file):
|
|
if self.rate:
|
|
total = 0 # total amount of bytes received
|
|
start = time.time() # start time
|
|
|
|
for data in response.iter_content(self.chunk_size):
|
|
file.write(data)
|
|
|
|
if self.rate:
|
|
total += len(data)
|
|
expected = total / self.rate # expected elapsed time
|
|
delta = time.time() - start # actual elapsed time since start
|
|
if delta < expected:
|
|
# sleep if less time passed than expected
|
|
time.sleep(expected - delta)
|
|
|
|
def get_extension(self, response):
|
|
mtype = response.headers.get("Content-Type", "image/jpeg")
|
|
mtype = mtype.partition(";")[0]
|
|
|
|
if mtype in MIMETYPE_MAP:
|
|
return MIMETYPE_MAP[mtype]
|
|
|
|
exts = mimetypes.guess_all_extensions(mtype, strict=False)
|
|
if exts:
|
|
exts.sort()
|
|
return exts[-1][1:]
|
|
|
|
self.log.warning(
|
|
"No filename extension found for MIME type '%s'", mtype)
|
|
return "txt"
|
|
|
|
@staticmethod
|
|
def check_extension(file, extension):
|
|
"""Check filename extension against fileheader"""
|
|
if extension in FILETYPE_CHECK:
|
|
file.seek(0)
|
|
header = file.read(8)
|
|
if len(header) >= 8 and not FILETYPE_CHECK[extension](header):
|
|
for ext, check in FILETYPE_CHECK.items():
|
|
if ext != extension and check(header):
|
|
return ext
|
|
return None
|
|
|
|
|
|
FILETYPE_CHECK = {
|
|
"jpg": lambda h: h[0:2] == b"\xff\xd8",
|
|
"png": lambda h: h[0:8] == b"\x89\x50\x4e\x47\x0d\x0a\x1a\x0a",
|
|
"gif": lambda h: h[0:4] == b"GIF8" and h[5] == 97,
|
|
}
|
|
|
|
|
|
MIMETYPE_MAP = {
|
|
"image/jpeg": "jpg",
|
|
"image/jpg": "jpg",
|
|
"image/png": "png",
|
|
"image/gif": "gif",
|
|
"image/bmp": "bmp",
|
|
"image/webp": "webp",
|
|
"image/svg+xml": "svg",
|
|
|
|
"video/webm": "webm",
|
|
"video/ogg": "ogg",
|
|
"video/mp4": "mp4",
|
|
|
|
"audio/wav": "wav",
|
|
"audio/x-wav": "wav",
|
|
"audio/webm": "webm",
|
|
"audio/ogg": "ogg",
|
|
"audio/mpeg": "mp3",
|
|
|
|
"application/zip": "zip",
|
|
"application/x-zip": "zip",
|
|
"application/x-zip-compressed": "zip",
|
|
"application/x-rar": "rar",
|
|
"application/x-rar-compressed": "rar",
|
|
"application/x-7z-compressed": "7z",
|
|
|
|
"application/ogg": "ogg",
|
|
"application/octet-stream": "bin",
|
|
}
|
|
|
|
|
|
__downloader__ = HttpDownloader
|