diff --git a/gallery_dl/extractor/8chan.py b/gallery_dl/extractor/8chan.py index d56e5e6f..559951fa 100644 --- a/gallery_dl/extractor/8chan.py +++ b/gallery_dl/extractor/8chan.py @@ -8,65 +8,25 @@ """Extract image- and video-urls from threads on https://8ch.net/""" -from .common import SequentialExtractor, Message -from urllib.parse import unquote -import re +from .chan import ChanExtractor info = { "category": "8chan", "extractor": "InfinityChanExtractor", - "directory": ["{category}", "{board}-{thread-id}"], - "filename": "{timestamp}-{name}", + "directory": ["{category}", "{board}-{thread}"], + "filename": "{tim}-{filename}{ext}", "pattern": [ - r"(?:https?://)?(?:www\.)?(?:8chan\.co|8ch\.net)/([^/]+/res/\d+).*", + r"(?:https?://)?(?:www\.)?8ch\.net/([^/]+)/res/(\d+).*", ], } -class InfinityChanExtractor(SequentialExtractor): +class InfinityChanExtractor(ChanExtractor): - url_base = "https://8ch.net" - url_fmt = url_base + "/{board}/res/{thread-id}.html" - regex = ( - r'>File: ([^<]+)\.[^<]+<.*?' - r'([^<]+)<' - ) + api_url = "https://8ch.net/{board}/res/{thread}.json" + file_url = "https://media.8ch.net/{board}/src/{tim}{ext}" def __init__(self, match, config): - SequentialExtractor.__init__(self, config) - self.match = match - - def items(self): - yield Message.Version, 1 - - metadata = self.get_job_metadata() - yield Message.Directory, metadata - - url = self.url_fmt.format(**metadata) - text = self.request(url).text - for match in re.finditer(self.regex, text): - yield Message.Url, self.get_file_url(match), self.get_file_metadata(match) - - def get_job_metadata(self): - """Collect metadata for extractor-job""" - board, _, thread_id = self.match.group(1).split("/") - return { - "category": info["category"], - "board": board, - "thread-id": thread_id, - } - - @staticmethod - def get_file_metadata(match): - """Collect metadata for a downloadable file""" - return { - "timestamp": match.group(2), - "name": unquote(match.group(4) or match.group(5)), - } - - def get_file_url(self, match): - """Extract download-url from 'match'""" - url = match.group(1) - if url.startswith("/"): - url = self.url_base + url - return url - + ChanExtractor.__init__( + self, config, info["category"], + match.group(1), match.group(2) + ) diff --git a/gallery_dl/extractor/chan.py b/gallery_dl/extractor/chan.py index cb336774..2f943068 100644 --- a/gallery_dl/extractor/chan.py +++ b/gallery_dl/extractor/chan.py @@ -1,4 +1,3 @@ - # -*- coding: utf-8 -*- # Copyright 2015 Mike Fährmann @@ -10,6 +9,7 @@ """Base classes for extractors for different Futaba Channel boards""" from .common import SequentialExtractor, Message +import re class ChanExtractor(SequentialExtractor): @@ -34,14 +34,15 @@ class ChanExtractor(SequentialExtractor): continue post.update(self.metadata) yield Message.Url, self.file_url.format(**post), post + if "extra_files" in post: + for file in post["extra_files"]: + post.update(file) + yield Message.Url, self.file_url.format(**post), post @staticmethod def get_thread_title(post): """Return thread title from first post""" if "sub" in post: return post["sub"] - com = post["com"] - pos = com.find("
") - if pos == -1: - return com - return com[:min(pos, 50)] + com = re.sub("<[^>]+?>", "", post["com"]) + return " ".join(com.split())[:50]