diff --git a/docs/configuration.rst b/docs/configuration.rst index e6ccd411..c16d1be8 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -2758,17 +2758,23 @@ Description extractor.newgrounds.format --------------------------- Type - ``string`` + * ``string`` + * ``list`` of ``string`` Default ``"original"`` Example - ``"720p"`` + * ``"720p"`` + * ``["mp4", "mov", "1080p", "720p"]`` Description Selects the preferred format for video downloads. If the selected format is not available, the next smaller one gets chosen. + If this is a ``list``, try each given + filename extension in original resolution or recoded format + until an available format is found. + extractor.newgrounds.include ---------------------------- diff --git a/gallery_dl/extractor/newgrounds.py b/gallery_dl/extractor/newgrounds.py index 7ac3a3a9..ecd6619d 100644 --- a/gallery_dl/extractor/newgrounds.py +++ b/gallery_dl/extractor/newgrounds.py @@ -12,6 +12,7 @@ from .common import Extractor, Message from .. import text, util, exception from ..cache import cache import itertools +import re class NewgroundsExtractor(Extractor): @@ -33,10 +34,16 @@ class NewgroundsExtractor(Extractor): def _init(self): self.flash = self.config("flash", True) - fmt = self.config("format", "original") - self.format = (True if not fmt or fmt == "original" else - fmt if isinstance(fmt, int) else - text.parse_int(fmt.rstrip("p"))) + fmt = self.config("format") + if not fmt or fmt == "original": + self.format = ("mp4", "webm", "m4v", "mov", "mkv", + 1080, 720, 360) + elif isinstance(fmt, (list, tuple)): + self.format = fmt + else: + self._video_formats = self._video_formats_limit + self.format = (fmt if isinstance(fmt, int) else + text.parse_int(fmt.rstrip("p"))) def items(self): self.login() @@ -266,7 +273,7 @@ class NewgroundsExtractor(Extractor): if src: src = src.replace("\\/", "/") - fallback = () + formats = () date = text.parse_datetime(extr( 'itemprop="datePublished" content="', '"')) else: @@ -276,23 +283,8 @@ class NewgroundsExtractor(Extractor): "X-Requested-With": "XMLHttpRequest", } sources = self.request(url, headers=headers).json()["sources"] - - if self.format is True: - src = sources["360p"][0]["src"].replace(".360p.", ".") - formats = sources - else: - formats = [] - for fmt, src in sources.items(): - width = text.parse_int(fmt.rstrip("p")) - if width <= self.format: - formats.append((width, src)) - if formats: - formats.sort(reverse=True) - src, formats = formats[0][1][0]["src"], formats[1:] - else: - src = "" - - fallback = self._video_fallback(formats) + formats = self._video_formats(sources) + src = next(formats, "") date = text.parse_timestamp(src.rpartition("?")[2]) return { @@ -306,15 +298,33 @@ class NewgroundsExtractor(Extractor): "rating" : extr('class="rated-', '"'), "index" : text.parse_int(index), "_index" : index, - "_fallback" : fallback, + "_fallback" : formats, } - @staticmethod - def _video_fallback(formats): - if isinstance(formats, dict): - formats = list(formats.items()) - formats.sort(key=lambda fmt: text.parse_int(fmt[0].rstrip("p")), - reverse=True) + def _video_formats(self, sources): + src = sources["360p"][0]["src"] + sub = re.compile(r"\.360p\.\w+").sub + + for fmt in self.format: + try: + if isinstance(fmt, int): + yield sources[str(fmt) + "p"][0]["src"] + elif fmt in sources: + yield sources[fmt][0]["src"] + else: + yield sub("." + fmt, src, 1) + except Exception as exc: + self.log.debug("Video format '%s' not available (%s: %s)", + fmt, exc.__class__.__name__, exc) + + def _video_formats_limit(self, sources): + formats = [] + for fmt, src in sources.items(): + width = text.parse_int(fmt.rstrip("p")) + if width <= self.format: + formats.append((width, src)) + + formats.sort(reverse=True) for fmt in formats: yield fmt[1][0]["src"] diff --git a/test/results/newgrounds.py b/test/results/newgrounds.py index e3740309..3bf15eae 100644 --- a/test/results/newgrounds.py +++ b/test/results/newgrounds.py @@ -107,7 +107,7 @@ __tests__ = ( "#url" : "https://www.newgrounds.com/portal/view/595355", "#category": ("", "newgrounds", "media"), "#class" : newgrounds.NewgroundsMediaExtractor, - "#pattern" : r"https://uploads\.ungrounded\.net/alternate/564000/564957_alternate_31\.mp4\?1359712249", + "#urls" : "https://uploads.ungrounded.net/alternate/564000/564957_alternate_31.mp4?1359712249", "artist" : [ "kickinthehead", @@ -131,6 +131,22 @@ __tests__ = ( "user" : "kickinthehead", }, +{ + "#url" : "https://www.newgrounds.com/portal/view/595355", + "#category": ("", "newgrounds", "media"), + "#class" : newgrounds.NewgroundsMediaExtractor, + "#options" : {"format": ["mkv", "mov", 1080]}, + "#urls" : "https://uploads.ungrounded.net/alternate/564000/564957_alternate_31.mkv?1359712249", +}, + +{ + "#url" : "https://www.newgrounds.com/portal/view/595355", + "#category": ("", "newgrounds", "media"), + "#class" : newgrounds.NewgroundsMediaExtractor, + "#options" : {"format": "720p"}, + "#urls" : "https://uploads.ungrounded.net/alternate/564000/564957_alternate_31.720p.mp4?1359712249", +}, + { "#url" : "https://www.newgrounds.com/audio/listen/609768", "#category": ("", "newgrounds", "media"),