diff --git a/docs/configuration.rst b/docs/configuration.rst index b3e4c2da..00a91f98 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -120,6 +120,20 @@ Description Alternate location for ``.part`` files. =========== ===== +downloader.http.rate +-------------------- +=========== ===== +Type ``string`` +Default ``null`` +Examples ``"32000"``, ``"500k"``, ``"2.5M"`` +Description Maximum download rate in bytes per second. + + Possible values are valid integer or floating-point numbers + optionally followed by one of ``k``, ``m``. ``g``, ``t`` or ``p``. + These suffixes are case-insensitive. +=========== ===== + + downloader.http.retries ----------------------- =========== ===== diff --git a/docs/gallery-dl.conf b/docs/gallery-dl.conf index bf67d670..048fddf9 100644 --- a/docs/gallery-dl.conf +++ b/docs/gallery-dl.conf @@ -7,6 +7,7 @@ "part-directory": null, "http": { + "rate": null, "retries": 5, "timeout": 30, "verify": true diff --git a/gallery_dl/downloader/http.py b/gallery_dl/downloader/http.py index d1249c59..2477a3a6 100644 --- a/gallery_dl/downloader/http.py +++ b/gallery_dl/downloader/http.py @@ -8,6 +8,7 @@ """Downloader module for http:// and https:// URLs""" +import time import mimetypes from .common import DownloaderBase from .. import util, exception @@ -22,6 +23,15 @@ class Downloader(DownloaderBase): self.retries = self.config("retries", 5) self.timeout = self.config("timeout", 30) self.verify = self.config("verify", True) + self.rate = self.config("rate") + self.chunk_size = 16384 + + if self.rate: + self.rate = util.parse_bytes(self.rate) + if not self.rate: + self.log.warning("Invalid rate limit specified") + elif self.rate < self.chunk_size: + self.chunk_size = self.rate def connect(self, url, offset): headers = {} @@ -50,9 +60,21 @@ class Downloader(DownloaderBase): return offset, util.safe_int(size) def receive(self, file): - for data in self.response.iter_content(16384): + if self.rate: + total = 0 # total amount of bytes received + start = time.time() # start time + + for data in self.response.iter_content(self.chunk_size): file.write(data) + if self.rate: + total += len(data) + expected = total / self.rate # expected elapsed time + delta = time.time() - start # actual elapsed time since start + if delta < expected: + # sleep if less time passed than expected + time.sleep(expected - delta) + def reset(self): if self.response: self.response.close() diff --git a/gallery_dl/option.py b/gallery_dl/option.py index 526c2667..cff21ad6 100644 --- a/gallery_dl/option.py +++ b/gallery_dl/option.py @@ -135,6 +135,11 @@ def build_parser(): ) downloader = parser.add_argument_group("Downloader Options") + downloader.add_argument( + "-r", "--limit-rate", + metavar="RATE", action=ConfigAction, dest="rate", + help="Maximum download rate (e.g. 500k or 2.5M)", + ) downloader.add_argument( "-R", "--retries", metavar="RETRIES", action=ConfigAction, dest="retries", type=int,