diff --git a/docs/configuration.rst b/docs/configuration.rst index 17a1c912..53c99aad 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -2344,6 +2344,19 @@ Description alongside the actual output files. +downloader.*.progress +--------------------- +Type + ``float`` +Default + ``3.0`` +Description + Number of seconds until a download progress indicator + for the current download is displayed. + + Set this option to ``null`` to disable this indicator. + + downloader.*.rate ----------------- Type diff --git a/docs/gallery-dl.conf b/docs/gallery-dl.conf index f8b6c363..b998597c 100644 --- a/docs/gallery-dl.conf +++ b/docs/gallery-dl.conf @@ -319,6 +319,7 @@ "mtime": true, "part": true, "part-directory": null, + "progress": 3.0, "rate": null, "retries": 4, "timeout": 30.0, diff --git a/gallery_dl/downloader/http.py b/gallery_dl/downloader/http.py index 78d8d347..d2efd3fc 100644 --- a/gallery_dl/downloader/http.py +++ b/gallery_dl/downloader/http.py @@ -31,6 +31,7 @@ class HttpDownloader(DownloaderBase): self.downloading = False self.adjust_extension = self.config("adjust-extensions", True) + self.progress = self.config("progress", 3.0) self.headers = self.config("headers") self.minsize = self.config("filesize-min") self.maxsize = self.config("filesize-max") @@ -63,6 +64,8 @@ class HttpDownloader(DownloaderBase): self.receive = self._receive_rate else: self.log.warning("Invalid rate limit (%r)", self.rate) + if self.progress is not None: + self.receive = self._receive_rate def download(self, url, pathfmt): try: @@ -202,6 +205,7 @@ class HttpDownloader(DownloaderBase): with pathfmt.open(mode) as fp: if file_header: fp.write(file_header) + offset += len(file_header) elif offset: if adjust_extension and \ pathfmt.extension in FILE_SIGNATURES: @@ -210,7 +214,7 @@ class HttpDownloader(DownloaderBase): self.out.start(pathfmt.path) try: - self.receive(fp, content) + self.receive(fp, content, size, offset) except (RequestException, SSLError, OpenSSLError) as exc: msg = str(exc) print() @@ -234,28 +238,42 @@ class HttpDownloader(DownloaderBase): return True @staticmethod - def receive(fp, content): + def receive(fp, content, bytes_total, bytes_downloaded): write = fp.write for data in content: write(data) - def _receive_rate(self, fp, content): - rt = self.rate - t1 = time.time() + def _receive_rate(self, fp, content, bytes_total, bytes_downloaded): + rate = self.rate + progress = self.progress + bytes_start = bytes_downloaded + write = fp.write + t1 = tstart = time.time() for data in content: - fp.write(data) + write(data) t2 = time.time() # current time - actual = t2 - t1 # actual elapsed time - expected = len(data) / rt # expected elapsed time + elapsed = t2 - t1 # elapsed time + num_bytes = len(data) + + if progress is not None: + bytes_downloaded += num_bytes + tdiff = t2 - tstart + if tdiff >= progress: + self.out.progress( + bytes_total, bytes_downloaded, + int((bytes_downloaded - bytes_start) / tdiff), + ) - if actual < expected: - # sleep if less time elapsed than expected - time.sleep(expected - actual) - t1 = time.time() - else: - t1 = t2 + if rate: + expected = num_bytes / rate # expected elapsed time + if elapsed < expected: + # sleep if less time elapsed than expected + time.sleep(expected - elapsed) + t2 = time.time() + + t1 = t2 def _find_extension(self, response): """Get filename extension from MIME type""" diff --git a/gallery_dl/downloader/ytdl.py b/gallery_dl/downloader/ytdl.py index b1e1d588..86e247b7 100644 --- a/gallery_dl/downloader/ytdl.py +++ b/gallery_dl/downloader/ytdl.py @@ -42,6 +42,10 @@ class YoutubeDLDownloader(DownloaderBase): if raw_options: options.update(raw_options) + self.progress = self.config("progress", 3.0) + if self.progress is not None: + options["progress_hooks"] = (self._progress_hook,) + if self.config("logging", True): options["logger"] = self.log self.forward_cookies = self.config("forward-cookies", False) @@ -56,7 +60,10 @@ class YoutubeDLDownloader(DownloaderBase): kwdict = pathfmt.kwdict ytdl = kwdict.pop("_ytdl_instance", None) - if not ytdl: + if ytdl: + if self.progress is not None and not ytdl._progress_hooks: + ytdl.add_progress_hook(self._progress_hook) + else: ytdl = self.ytdl if self.forward_cookies: set_cookie = ytdl.cookiejar.set_cookie @@ -126,6 +133,15 @@ class YoutubeDLDownloader(DownloaderBase): ytdl.process_info(entry) return True + def _progress_hook(self, info): + if info["status"] == "downloading" and \ + info["elapsed"] >= self.progress: + self.out.progress( + info["total_bytes"], + info["downloaded_bytes"], + int(info["speed"]), + ) + @staticmethod def _set_outtmpl(ytdl, outtmpl): try: diff --git a/gallery_dl/output.py b/gallery_dl/output.py index 37a4de58..5d0c3ed8 100644 --- a/gallery_dl/output.py +++ b/gallery_dl/output.py @@ -258,6 +258,9 @@ class NullOutput(): def success(self, path, tries): """Print a message indicating the completion of a download""" + def progress(self, bytes_total, bytes_downloaded, bytes_per_second): + """Display download progress""" + class PipeOutput(NullOutput): @@ -289,6 +292,19 @@ class TerminalOutput(NullOutput): def success(self, path, tries): print("\r", self.shorten(CHAR_SUCCESS + path), sep="") + def progress(self, bytes_total, bytes_downloaded, bytes_per_second): + if bytes_total is None: + print("\r {:>8} {:>10} \r".format( + util.format_value(bytes_downloaded, "B"), + util.format_value(bytes_per_second, "B/s"), + ), end="") + else: + print("\r{:>3}% {:>8} {:>10} \r".format( + bytes_downloaded * 100 // bytes_total, + util.format_value(bytes_downloaded, "B"), + util.format_value(bytes_per_second, "B/s"), + ), end="") + class ColorOutput(TerminalOutput):