[newgrounds] extend 'format' option (#5709)

- check more extensions for original formats (mp4, webm, m4v, mov, mkv)
- allow specifying which extensions and recoded formats to check

docs/configuration.rst

@@ -2758,17 +2758,23 @@ Description
 extractor.newgrounds.format
 ---------------------------
 Type
-    ``string``
+    * ``string``
+    * ``list`` of ``string``
 Default
     ``"original"``
 Example
-    ``"720p"``
+    * ``"720p"``
+    * ``["mp4", "mov", "1080p", "720p"]``
 Description
     Selects the preferred format for video downloads.
 
     If the selected format is not available,
     the next smaller one gets chosen.
 
+    If this is a ``list``, try each given
+    filename extension in original resolution or recoded format
+    until an available format is found.
+
 
 extractor.newgrounds.include
 ----------------------------

gallery_dl/extractor/newgrounds.py

@@ -12,6 +12,7 @@ from .common import Extractor, Message
 from .. import text, util, exception
 from ..cache import cache
 import itertools
+import re
 
 
 class NewgroundsExtractor(Extractor):
@@ -33,9 +34,15 @@ class NewgroundsExtractor(Extractor):
     def _init(self):
         self.flash = self.config("flash", True)
 
-        fmt = self.config("format", "original")
+        fmt = self.config("format")
-        self.format = (True if not fmt or fmt == "original" else
+        if not fmt or fmt == "original":
-                       fmt if isinstance(fmt, int) else
+            self.format = ("mp4", "webm", "m4v", "mov", "mkv",
+                           1080, 720, 360)
+        elif isinstance(fmt, (list, tuple)):
+            self.format = fmt
+        else:
+            self._video_formats = self._video_formats_limit
+            self.format = (fmt if isinstance(fmt, int) else
                            text.parse_int(fmt.rstrip("p")))
 
     def items(self):
@@ -266,7 +273,7 @@ class NewgroundsExtractor(Extractor):
 
         if src:
             src = src.replace("\\/", "/")
-            fallback = ()
+            formats = ()
             date = text.parse_datetime(extr(
                 'itemprop="datePublished" content="', '"'))
         else:
@@ -276,23 +283,8 @@ class NewgroundsExtractor(Extractor):
                 "X-Requested-With": "XMLHttpRequest",
             }
             sources = self.request(url, headers=headers).json()["sources"]
-
+            formats = self._video_formats(sources)
-            if self.format is True:
+            src = next(formats, "")
-                src = sources["360p"][0]["src"].replace(".360p.", ".")
-                formats = sources
-            else:
-                formats = []
-                for fmt, src in sources.items():
-                    width = text.parse_int(fmt.rstrip("p"))
-                    if width <= self.format:
-                        formats.append((width, src))
-                if formats:
-                    formats.sort(reverse=True)
-                    src, formats = formats[0][1][0]["src"], formats[1:]
-                else:
-                    src = ""
-
-            fallback = self._video_fallback(formats)
             date = text.parse_timestamp(src.rpartition("?")[2])
 
         return {
@@ -306,15 +298,33 @@ class NewgroundsExtractor(Extractor):
             "rating"     : extr('class="rated-', '"'),
             "index"      : text.parse_int(index),
             "_index"     : index,
-            "_fallback"  : fallback,
+            "_fallback"  : formats,
         }
 
-    @staticmethod
+    def _video_formats(self, sources):
-    def _video_fallback(formats):
+        src = sources["360p"][0]["src"]
-        if isinstance(formats, dict):
+        sub = re.compile(r"\.360p\.\w+").sub
-            formats = list(formats.items())
+
-            formats.sort(key=lambda fmt: text.parse_int(fmt[0].rstrip("p")),
+        for fmt in self.format:
-                         reverse=True)
+            try:
+                if isinstance(fmt, int):
+                    yield sources[str(fmt) + "p"][0]["src"]
+                elif fmt in sources:
+                    yield sources[fmt][0]["src"]
+                else:
+                    yield sub("." + fmt, src, 1)
+            except Exception as exc:
+                self.log.debug("Video format '%s' not available (%s: %s)",
+                               fmt, exc.__class__.__name__, exc)
+
+    def _video_formats_limit(self, sources):
+        formats = []
+        for fmt, src in sources.items():
+            width = text.parse_int(fmt.rstrip("p"))
+            if width <= self.format:
+                formats.append((width, src))
+
+        formats.sort(reverse=True)
         for fmt in formats:
             yield fmt[1][0]["src"]
 

test/results/newgrounds.py

@@ -107,7 +107,7 @@ __tests__ = (
     "#url"     : "https://www.newgrounds.com/portal/view/595355",
     "#category": ("", "newgrounds", "media"),
     "#class"   : newgrounds.NewgroundsMediaExtractor,
-    "#pattern" : r"https://uploads\.ungrounded\.net/alternate/564000/564957_alternate_31\.mp4\?1359712249",
+    "#urls"    : "https://uploads.ungrounded.net/alternate/564000/564957_alternate_31.mp4?1359712249",
 
     "artist"     : [
         "kickinthehead",
@@ -131,6 +131,22 @@ __tests__ = (
     "user"       : "kickinthehead",
 },
 
+{
+    "#url"     : "https://www.newgrounds.com/portal/view/595355",
+    "#category": ("", "newgrounds", "media"),
+    "#class"   : newgrounds.NewgroundsMediaExtractor,
+    "#options" : {"format": ["mkv", "mov", 1080]},
+    "#urls"    : "https://uploads.ungrounded.net/alternate/564000/564957_alternate_31.mkv?1359712249",
+},
+
+{
+    "#url"     : "https://www.newgrounds.com/portal/view/595355",
+    "#category": ("", "newgrounds", "media"),
+    "#class"   : newgrounds.NewgroundsMediaExtractor,
+    "#options" : {"format": "720p"},
+    "#urls"    : "https://uploads.ungrounded.net/alternate/564000/564957_alternate_31.720p.mp4?1359712249",
+},
+
 {
     "#url"     : "https://www.newgrounds.com/audio/listen/609768",
     "#category": ("", "newgrounds", "media"),