diff --git a/CHANGELOG.md b/CHANGELOG.md index f4257691..19b05bb9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## Unreleased +### Additions +- Support for + - `plurk` - https://www.plurk.com/ (#212) + ## 1.8.2 - 2019-04-12 ### Additions - Support for diff --git a/docs/supportedsites.rst b/docs/supportedsites.rst index 08142555..df8f5790 100644 --- a/docs/supportedsites.rst +++ b/docs/supportedsites.rst @@ -74,6 +74,7 @@ Piczel https://piczel.tv/ Images from Users, Fold Pinterest https://www.pinterest.com/ Boards, Pins, pin.it Links, related Pins Pixiv https://www.pixiv.net/ |pixiv-C| Required Pixnet https://www.pixnet.net/ |pixnet-C| +Plurk https://www.plurk.com/ Posts, Timelines Pornreactor http://pornreactor.cc/ |pornreactor-C| PowerManga https://read.powermanga.org/ Chapters, Manga Pururin https://pururin.io/ Galleries diff --git a/gallery_dl/extractor/__init__.py b/gallery_dl/extractor/__init__.py index f1673df4..cf1cbcd9 100644 --- a/gallery_dl/extractor/__init__.py +++ b/gallery_dl/extractor/__init__.py @@ -68,6 +68,7 @@ modules = [ "pinterest", "pixiv", "pixnet", + "plurk", "pururin", "reactor", "readcomiconline", diff --git a/gallery_dl/extractor/plurk.py b/gallery_dl/extractor/plurk.py new file mode 100644 index 00000000..d9077eb7 --- /dev/null +++ b/gallery_dl/extractor/plurk.py @@ -0,0 +1,98 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 Mike Fährmann +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +"""Extractors for https://www.plurk.com/""" + +from .common import Extractor, Message +from .. import text, exception +import datetime +import json +import re + + +class PlurkExtractor(Extractor): + """Base class for plurk extractors""" + category = "plurk" + root = "https://www.plurk.com" + + def items(self): + yield Message.Version, 1 + for plurk in self.plurks(): + for url in text.extract_iter(plurk["content"], ' href="', '"'): + yield Message.Queue, url, plurk + + def plurks(self): + """Return an iterable with all relevant 'plurk' objects""" + + @staticmethod + def _load(data): + if not data: + raise exception.NotFoundError("user") + return json.loads(re.sub(r"new Date\(([^)]+)\)", r"\1", data)) + + +class PlurkTimelineExtractor(PlurkExtractor): + """Extractor for URLs from all posts in a Plurk timeline""" + subcategory = "timeline" + pattern = r"(?:https?://)?(?:www\.)?plurk\.com/(?!p/)(\w+)/?(?:$|[?&#])" + test = ("https://www.plurk.com/plurkapi", { + "pattern": r"https?://.+", + "count": ">= 23" + }) + + def __init__(self, match): + PlurkExtractor.__init__(self, match) + self.user = match.group(1) + + def plurks(self): + url = "{}/{}".format(self.root, self.user) + page = self.request(url).text + user_id, pos = text.extract(page, '"user_id":', ',') + plurks = self._load(text.extract(page, "_PLURKS = ", ";\n", pos)[0]) + + url = "https://www.plurk.com/TimeLine/getPlurks" + headers = { + "Referer": self.root + "/", + "X-Requested-With": "XMLHttpRequest", + } + data = { + "user_id": user_id.strip(), + } + + while plurks: + yield from plurks + + offset = datetime.datetime.strptime( + plurks[-1]["posted"], "%a, %d %b %Y %H:%M:%S %Z") + data["offset"] = offset.strftime("%Y-%m-%dT%H:%M:%S.000Z") + response = self.request( + url, method="POST", headers=headers, data=data) + plurks = response.json()["plurks"] + + +class PlurkPostExtractor(PlurkExtractor): + """Extractor for URLs from a Plurk post""" + subcategory = "post" + pattern = r"(?:https?://)?(?:www\.)?plurk\.com/p/(\w+)" + test = ("https://www.plurk.com/p/i701j1", { + "url": "2115f208564591b8748525c2807a84596aaaaa5f", + }) + + def __init__(self, match): + PlurkExtractor.__init__(self, match) + self.plurk_id = match.group(1) + + def plurks(self): + url = "{}/p/{}".format(self.root, self.plurk_id) + page = self.request(url).text + user, pos = text.extract(page, " GLOBAL = ", "\n") + data, pos = text.extract(page, "plurk = ", ";\n", pos) + + data = self._load(data) + data["user"] = self._load(user)["page_user"] + return (data,) diff --git a/gallery_dl/version.py b/gallery_dl/version.py index 94d4001e..ceec12a1 100644 --- a/gallery_dl/version.py +++ b/gallery_dl/version.py @@ -6,4 +6,4 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -__version__ = "1.8.2" +__version__ = "1.8.3-dev"