add 'brotli' as optional dependency (#2716)

only send 'Accept-Encoding: br' if supported

README.rst

@@ -25,6 +25,7 @@ Optional
 - FFmpeg_: Pixiv Ugoira to WebM conversion
 - yt-dlp_ or youtube-dl_: Video downloads
 - PySocks_: SOCKS proxy support
+- brotli_ or brotlicffi_: Brotli compression support
 
 
 Installation
@@ -332,6 +333,8 @@ To authenticate with a ``mastodon`` instance, run *gallery-dl* with
 .. _yt-dlp:     https://github.com/yt-dlp/yt-dlp
 .. _youtube-dl: https://ytdl-org.github.io/youtube-dl/
 .. _PySocks:    https://pypi.org/project/PySocks/
+.. _brotli:     https://github.com/google/brotli
+.. _brotlicffi: https://github.com/python-hyper/brotlicffi
 .. _pyOpenSSL:  https://pyopenssl.org/
 .. _Snapd:      https://docs.snapcraft.io/installing-snapd
 .. _OAuth:      https://en.wikipedia.org/wiki/OAuth

gallery_dl/extractor/common.py

@@ -259,6 +259,10 @@ class Extractor():
                 "rv:102.0) Gecko/20100101 Firefox/102.0"))
             headers["Accept"] = "*/*"
             headers["Accept-Language"] = "en-US,en;q=0.5"
+
+        if BROTLI:
+            headers["Accept-Encoding"] = "gzip, deflate, br"
+        else:
             headers["Accept-Encoding"] = "gzip, deflate"
 
         custom_headers = self.config("headers")
@@ -718,7 +722,7 @@ HTTP_HEADERS = {
         ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,"
                    "image/avif,image/webp,*/*;q=0.8"),
         ("Accept-Language", "en-US,en;q=0.5"),
-        ("Accept-Encoding", "gzip, deflate, br"),
+        ("Accept-Encoding", None),
         ("Referer", None),
         ("DNT", "1"),
         ("Connection", "keep-alive"),
@@ -736,7 +740,7 @@ HTTP_HEADERS = {
         ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,"
                    "image/webp,image/apng,*/*;q=0.8"),
         ("Referer", None),
-        ("Accept-Encoding", "gzip, deflate"),
+        ("Accept-Encoding", None),
         ("Accept-Language", "en-US,en;q=0.9"),
         ("Cookie", None),
     ),
@@ -783,6 +787,13 @@ SSL_CIPHERS = {
 }
 
 
+# detect brotli support
+try:
+    BROTLI = requests.packages.urllib3.response.brotli is not None
+except AttributeError:
+    BROTLI = False
+
+
 # Undo automatic pyOpenSSL injection by requests
 pyopenssl = config.get((), "pyopenssl", False)
 if not pyopenssl: