diff --git a/CHANGELOG.md b/CHANGELOG.md index 27e7079d..8a0eed4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## Unreleased +- Improved retry behavior for failed HTTP downloads + ## 1.0.1 - 2017-11-10 - Added support for: - `xvideos` - https://www.xvideos.com/ ([#45](https://github.com/mikf/gallery-dl/issues/45)) diff --git a/gallery_dl/downloader/common.py b/gallery_dl/downloader/common.py index f96885f3..a4732dff 100644 --- a/gallery_dl/downloader/common.py +++ b/gallery_dl/downloader/common.py @@ -11,7 +11,7 @@ import os import time import logging -from .. import config, util +from .. import config, util, exception from requests.exceptions import RequestException @@ -65,7 +65,7 @@ class DownloaderBase(): self.out.error(pathfmt.path, msg, tries, self.retries) if tries >= self.retries: return False - time.sleep(1) + time.sleep(tries) tries += 1 # check for .part file @@ -74,6 +74,11 @@ class DownloaderBase(): # connect to (remote) source try: offset, size = self.connect(url, filesize) + except exception.DownloadError as exc: + self.out.error(pathfmt.path, exc, 0, 0) + return False + except exception.DownloadComplete: + break except Exception as exc: msg = exc continue @@ -83,8 +88,6 @@ class DownloaderBase(): mode = "wb" if filesize: self.log.info("Unable to resume partial download") - elif offset == -1: - break # early finish else: mode = "ab" self.log.info("Resuming download at byte %d", offset) @@ -124,8 +127,7 @@ class DownloaderBase(): Returns a 2-tuple containing the actual offset and expected filesize. If the returned offset-value is greater than zero, all received data - will be appended to the existing .part file. If it is '-1', the - download will finish early and be considered successfull. + will be appended to the existing .part file. Return '0' as second tuple-field to indicate an unknown filesize. """ diff --git a/gallery_dl/downloader/http.py b/gallery_dl/downloader/http.py index a303e220..00247b57 100644 --- a/gallery_dl/downloader/http.py +++ b/gallery_dl/downloader/http.py @@ -10,7 +10,7 @@ import mimetypes from .common import DownloaderBase -from .. import util +from .. import util, exception class Downloader(DownloaderBase): @@ -33,14 +33,17 @@ class Downloader(DownloaderBase): timeout=self.timeout, verify=self.verify) code = self.response.status_code - if code == 200: + if code == 200: # OK offset = 0 size = self.response.headers.get("Content-Length") - elif code == 206: + elif code == 206: # Partial Content size = self.response.headers["Content-Range"].rpartition("/")[2] - elif code == 416: - # file is already complete - return -1, 0 + elif code == 416: # Requested Range Not Satisfiable + raise exception.DownloadComplete() + elif 400 <= code < 500 and code != 429: # Client Error + raise exception.DownloadError( + "{} Client Error: {} for url: {}".format( + code, self.response.reason, url)) else: self.response.raise_for_status() diff --git a/gallery_dl/exception.py b/gallery_dl/exception.py index b9b89e45..f35f5089 100644 --- a/gallery_dl/exception.py +++ b/gallery_dl/exception.py @@ -17,6 +17,8 @@ Exception | +-- AuthorizationError | +-- NotFoundError | +-- HttpError + +-- DownloadError + +-- DownloadComplete +-- NoExtractorError +-- FormatError +-- FilterError @@ -48,6 +50,14 @@ class HttpError(ExtractionError): """HTTP request during extraction failed""" +class DownloadError(GalleryDLException): + """Error during file download""" + + +class DownloadComplete(GalleryDLException): + """Output file of attempted download is already complete""" + + class NoExtractorError(GalleryDLException): """No extractor can handle the given URL""" diff --git a/gallery_dl/output.py b/gallery_dl/output.py index 1536bae4..7756f078 100644 --- a/gallery_dl/output.py +++ b/gallery_dl/output.py @@ -89,9 +89,10 @@ class TerminalOutput(NullOutput): if tries <= 1 and path: print("\r", end="") safeprint(self.shorten(CHAR_ERROR + path)) + if max_tries > 1: + error = "{} ({}/{})".format(error, tries, max_tries) print("\r[Error] ", end="") - safeprint(error, end="") - print(" (", tries, "/", max_tries, ")", sep="") + safeprint(error) def shorten(self, txt): """Reduce the length of 'txt' to the width of the terminal""" @@ -119,8 +120,9 @@ class ColorOutput(TerminalOutput): def error(self, path, error, tries, max_tries): if tries <= 1 and path: print("\r\033[1;31m", self.shorten(path), sep="") - print("\r\033[0;31m[Error]\033[0m ", error, - " (", tries, "/", max_tries, ")", sep="") + if max_tries > 1: + error = "{} ({}/{})".format(error, tries, max_tries) + print("\r\033[0;31m[Error]\033[0m", error) if os.name == "nt": diff --git a/gallery_dl/version.py b/gallery_dl/version.py index aade2b2a..23c96958 100644 --- a/gallery_dl/version.py +++ b/gallery_dl/version.py @@ -6,4 +6,4 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -__version__ = "1.0.1" +__version__ = "1.0.2-dev"