[8chan] use api

gallery_dl/extractor/8chan.py

@@ -8,65 +8,25 @@
 
 """Extract image- and video-urls from threads on https://8ch.net/"""
 
-from .common import SequentialExtractor, Message
-from urllib.parse import unquote
-import re
+from .chan import ChanExtractor
 
 info = {
     "category": "8chan",
     "extractor": "InfinityChanExtractor",
-    "directory": ["{category}", "{board}-{thread-id}"],
-    "filename": "{timestamp}-{name}",
+    "directory": ["{category}", "{board}-{thread}"],
+    "filename": "{tim}-{filename}{ext}",
     "pattern": [
-        r"(?:https?://)?(?:www\.)?(?:8chan\.co|8ch\.net)/([^/]+/res/\d+).*",
+        r"(?:https?://)?(?:www\.)?8ch\.net/([^/]+)/res/(\d+).*",
     ],
 }
 
-class InfinityChanExtractor(SequentialExtractor):
+class InfinityChanExtractor(ChanExtractor):
 
-    url_base = "https://8ch.net"
-    url_fmt = url_base + "/{board}/res/{thread-id}.html"
-    regex = (
-        r'>File: <a href="([^"]+)">([^<]+)\.[^<]+<.*?'
-        r'<span class="postfilename"( title="([^"]+)")?>([^<]+)<'
-    )
+    api_url = "https://8ch.net/{board}/res/{thread}.json"
+    file_url = "https://media.8ch.net/{board}/src/{tim}{ext}"
 
     def __init__(self, match, config):
-        SequentialExtractor.__init__(self, config)
-        self.match = match
-
-    def items(self):
-        yield Message.Version, 1
-
-        metadata = self.get_job_metadata()
-        yield Message.Directory, metadata
-
-        url = self.url_fmt.format(**metadata)
-        text = self.request(url).text
-        for match in re.finditer(self.regex, text):
-            yield Message.Url, self.get_file_url(match), self.get_file_metadata(match)
-
-    def get_job_metadata(self):
-        """Collect metadata for extractor-job"""
-        board, _, thread_id = self.match.group(1).split("/")
-        return {
-            "category": info["category"],
-            "board": board,
-            "thread-id": thread_id,
-        }
-
-    @staticmethod
-    def get_file_metadata(match):
-        """Collect metadata for a downloadable file"""
-        return {
-            "timestamp": match.group(2),
-            "name": unquote(match.group(4) or match.group(5)),
-        }
-
-    def get_file_url(self, match):
-        """Extract download-url from 'match'"""
-        url = match.group(1)
-        if url.startswith("/"):
-            url = self.url_base + url
-        return url
-
+        ChanExtractor.__init__(
+            self, config, info["category"],
+            match.group(1), match.group(2)
+        )

gallery_dl/extractor/chan.py

@@ -1,4 +1,3 @@
-
 # -*- coding: utf-8 -*-
 
 # Copyright 2015 Mike Fährmann
@@ -10,6 +9,7 @@
 """Base classes for extractors for different Futaba Channel boards"""
 
 from .common import SequentialExtractor, Message
+import re
 
 class ChanExtractor(SequentialExtractor):
 
@@ -34,14 +34,15 @@ class ChanExtractor(SequentialExtractor):
                 continue
             post.update(self.metadata)
             yield Message.Url, self.file_url.format(**post), post
+            if "extra_files" in post:
+                for file in post["extra_files"]:
+                    post.update(file)
+                    yield Message.Url, self.file_url.format(**post), post
 
     @staticmethod
     def get_thread_title(post):
         """Return thread title from first post"""
         if "sub" in post:
             return post["sub"]
-        com = post["com"]
-        pos = com.find("<br>")
-        if pos == -1:
-            return com
-        return com[:min(pos, 50)]
+        com = re.sub("<[^>]+?>", "", post["com"])
+        return " ".join(com.split())[:50]