|
|
|
@ -31,6 +31,7 @@ class HttpDownloader(DownloaderBase):
|
|
|
|
|
self.downloading = False
|
|
|
|
|
|
|
|
|
|
self.adjust_extension = self.config("adjust-extensions", True)
|
|
|
|
|
self.progress = self.config("progress", 3.0)
|
|
|
|
|
self.headers = self.config("headers")
|
|
|
|
|
self.minsize = self.config("filesize-min")
|
|
|
|
|
self.maxsize = self.config("filesize-max")
|
|
|
|
@ -63,6 +64,8 @@ class HttpDownloader(DownloaderBase):
|
|
|
|
|
self.receive = self._receive_rate
|
|
|
|
|
else:
|
|
|
|
|
self.log.warning("Invalid rate limit (%r)", self.rate)
|
|
|
|
|
if self.progress is not None:
|
|
|
|
|
self.receive = self._receive_rate
|
|
|
|
|
|
|
|
|
|
def download(self, url, pathfmt):
|
|
|
|
|
try:
|
|
|
|
@ -202,6 +205,7 @@ class HttpDownloader(DownloaderBase):
|
|
|
|
|
with pathfmt.open(mode) as fp:
|
|
|
|
|
if file_header:
|
|
|
|
|
fp.write(file_header)
|
|
|
|
|
offset += len(file_header)
|
|
|
|
|
elif offset:
|
|
|
|
|
if adjust_extension and \
|
|
|
|
|
pathfmt.extension in FILE_SIGNATURES:
|
|
|
|
@ -210,7 +214,7 @@ class HttpDownloader(DownloaderBase):
|
|
|
|
|
|
|
|
|
|
self.out.start(pathfmt.path)
|
|
|
|
|
try:
|
|
|
|
|
self.receive(fp, content)
|
|
|
|
|
self.receive(fp, content, size, offset)
|
|
|
|
|
except (RequestException, SSLError, OpenSSLError) as exc:
|
|
|
|
|
msg = str(exc)
|
|
|
|
|
print()
|
|
|
|
@ -234,28 +238,42 @@ class HttpDownloader(DownloaderBase):
|
|
|
|
|
return True
|
|
|
|
|
|
|
|
|
|
@staticmethod
|
|
|
|
|
def receive(fp, content):
|
|
|
|
|
def receive(fp, content, bytes_total, bytes_downloaded):
|
|
|
|
|
write = fp.write
|
|
|
|
|
for data in content:
|
|
|
|
|
write(data)
|
|
|
|
|
|
|
|
|
|
def _receive_rate(self, fp, content):
|
|
|
|
|
rt = self.rate
|
|
|
|
|
t1 = time.time()
|
|
|
|
|
def _receive_rate(self, fp, content, bytes_total, bytes_downloaded):
|
|
|
|
|
rate = self.rate
|
|
|
|
|
progress = self.progress
|
|
|
|
|
bytes_start = bytes_downloaded
|
|
|
|
|
write = fp.write
|
|
|
|
|
t1 = tstart = time.time()
|
|
|
|
|
|
|
|
|
|
for data in content:
|
|
|
|
|
fp.write(data)
|
|
|
|
|
write(data)
|
|
|
|
|
|
|
|
|
|
t2 = time.time() # current time
|
|
|
|
|
actual = t2 - t1 # actual elapsed time
|
|
|
|
|
expected = len(data) / rt # expected elapsed time
|
|
|
|
|
elapsed = t2 - t1 # elapsed time
|
|
|
|
|
num_bytes = len(data)
|
|
|
|
|
|
|
|
|
|
if progress is not None:
|
|
|
|
|
bytes_downloaded += num_bytes
|
|
|
|
|
tdiff = t2 - tstart
|
|
|
|
|
if tdiff >= progress:
|
|
|
|
|
self.out.progress(
|
|
|
|
|
bytes_total, bytes_downloaded,
|
|
|
|
|
int((bytes_downloaded - bytes_start) / tdiff),
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
if actual < expected:
|
|
|
|
|
# sleep if less time elapsed than expected
|
|
|
|
|
time.sleep(expected - actual)
|
|
|
|
|
t1 = time.time()
|
|
|
|
|
else:
|
|
|
|
|
t1 = t2
|
|
|
|
|
if rate:
|
|
|
|
|
expected = num_bytes / rate # expected elapsed time
|
|
|
|
|
if elapsed < expected:
|
|
|
|
|
# sleep if less time elapsed than expected
|
|
|
|
|
time.sleep(expected - elapsed)
|
|
|
|
|
t2 = time.time()
|
|
|
|
|
|
|
|
|
|
t1 = t2
|
|
|
|
|
|
|
|
|
|
def _find_extension(self, response):
|
|
|
|
|
"""Get filename extension from MIME type"""
|
|
|
|
|