parent
0abbee3710
commit
f31c05f522
@ -1,24 +1,73 @@
|
||||
from .common import BasicExtractor
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2014, 2015 Mike Fährmann
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License version 2 as
|
||||
# published by the Free Software Foundation.
|
||||
|
||||
"""Extract image- and video-urls from threads on https://8ch.net/"""
|
||||
|
||||
from .common import SequentialExtractor, Message
|
||||
from urllib.parse import unquote
|
||||
import re
|
||||
|
||||
class Extractor(BasicExtractor):
|
||||
info = {
|
||||
"category": "8chan",
|
||||
"extractor": "InfinityChanExtractor",
|
||||
"directory": ["{category}", "{board}-{thread-id}"],
|
||||
"filename": "{timestamp}-{name}",
|
||||
"pattern": [
|
||||
r"(?:https?://)?(?:www\.)?8chan\.co/([^/]+/res/\d+).*",
|
||||
r"(?:https?://)?(?:www\.)?8ch\.net/([^/]+/res/\d+).*",
|
||||
],
|
||||
}
|
||||
|
||||
class InfinityChanExtractor(SequentialExtractor):
|
||||
|
||||
url_base = "https://8ch.net"
|
||||
thread_url_fmt = url_base + "/{0}/res/{1}.html"
|
||||
regex = r'>File: <a href="([^"]+)">([^<]+)\.[^<]+<.*?<span class="postfilename"( title="([^"]+)")?>([^<]+)<'
|
||||
url_fmt = url_base + "/{board}/res/{thread-id}.html"
|
||||
regex = (
|
||||
r'>File: <a href="([^"]+)">([^<]+)\.[^<]+<.*?'
|
||||
r'<span class="postfilename"( title="([^"]+)")?>([^<]+)<'
|
||||
)
|
||||
|
||||
def __init__(self, match, config):
|
||||
BasicExtractor.__init__(self, config)
|
||||
self.board, _, self.thread_id = match.group(1).split("/")
|
||||
self.category = "8chan"
|
||||
self.directory = self.board + "-" + self.thread_id
|
||||
SequentialExtractor.__init__(self, config)
|
||||
self.match = match
|
||||
|
||||
def items(self):
|
||||
yield Message.Version, 1
|
||||
|
||||
def images(self):
|
||||
url = self.thread_url_fmt.format(self.board, self.thread_id)
|
||||
metadata = self.get_job_metadata()
|
||||
yield Message.Directory, metadata
|
||||
|
||||
url = self.url_fmt.format(**metadata)
|
||||
text = self.request(url).text
|
||||
for match in re.finditer(self.regex, text):
|
||||
url, prefix, fullname, name = match.group(1, 2, 4, 5)
|
||||
yield Message.Url, self.get_file_url(match), self.get_file_metadata(match)
|
||||
|
||||
def get_job_metadata(self):
|
||||
"""Collect metadata for extractor-job"""
|
||||
board, _, thread_id = self.match.group(1).split("/")
|
||||
return {
|
||||
"category": info["category"],
|
||||
"board": board,
|
||||
"thread-id": thread_id,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def get_file_metadata(match):
|
||||
"""Collect metadata for a downloadable file"""
|
||||
return {
|
||||
"timestamp": match.group(2),
|
||||
"name": unquote(match.group(4) or match.group(5)),
|
||||
}
|
||||
|
||||
def get_file_url(self, match):
|
||||
"""Extract download-url from 'match'"""
|
||||
url = match.group(1)
|
||||
if url.startswith("/"):
|
||||
url = self.url_base + url
|
||||
yield (url, prefix + "-" + unquote(fullname or name))
|
||||
return url
|
||||
|
||||
|
Loading…
Reference in new issue