[downloader:http] add 'chunk-size' option (#3143)

and double the previous default from 16384 (2**14) to 32768 (2**15)

docs/configuration.rst

@@ -3009,7 +3009,7 @@ Description
     Any file smaller/larger than this limit will not be downloaded.
 
     Possible values are valid integer or floating-point numbers
-    optionally followed by one of ``k``, ``m``. ``g``, ``t`` or ``p``.
+    optionally followed by one of ``k``, ``m``. ``g``, ``t``, or ``p``.
     These suffixes are case-insensitive.
 
 
@@ -3079,7 +3079,7 @@ Description
     Maximum download rate in bytes per second.
 
     Possible values are valid integer or floating-point numbers
-    optionally followed by one of ``k``, ``m``. ``g``, ``t`` or ``p``.
+    optionally followed by one of ``k``, ``m``. ``g``, ``t``, or ``p``.
     These suffixes are case-insensitive.
 
 
@@ -3140,6 +3140,22 @@ Description
     contains JPEG/JFIF data.
 
 
+downloader.http.chunk-size
+--------------------------
+Type
+    ``integer`` or ``string``
+Default
+    ``32768``
+Example
+    ``"50k"``, ``"0.8M"``
+Description
+    Number of bytes per downloaded chunk.
+
+    Possible values are integer numbers
+    optionally followed by one of ``k``, ``m``. ``g``, ``t``, or ``p``.
+    These suffixes are case-insensitive.
+
+
 downloader.http.headers
 -----------------------
 Type

docs/gallery-dl.conf

@@ -381,6 +381,7 @@
         "http":
         {
             "adjust-extensions": true,
+            "chunk-size": 32768,
             "headers": null
         },
 

gallery_dl/downloader/http.py

@@ -27,10 +27,10 @@ class HttpDownloader(DownloaderBase):
     def __init__(self, job):
         DownloaderBase.__init__(self, job)
         extractor = job.extractor
-        self.chunk_size = 16384
         self.downloading = False
 
         self.adjust_extension = self.config("adjust-extensions", True)
+        self.chunk_size = self.config("chunk-size", 32768)
         self.progress = self.config("progress", 3.0)
         self.headers = self.config("headers")
         self.minsize = self.config("filesize-min")
@@ -55,6 +55,13 @@ class HttpDownloader(DownloaderBase):
                 self.log.warning(
                     "Invalid maximum file size (%r)", self.maxsize)
             self.maxsize = maxsize
+        if isinstance(self.chunk_size, str):
+            chunk_size = text.parse_bytes(self.chunk_size)
+            if not chunk_size:
+                self.log.warning(
+                    "Invalid chunk size (%r)", self.chunk_size)
+                chunk_size = 32768
+            self.chunk_size = chunk_size
         if self.rate:
             rate = text.parse_bytes(self.rate)
             if rate: