From 279db2c5b2053d88353cadcd97cfe440740a1dd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Thu, 25 Jul 2019 17:51:35 +0200 Subject: [PATCH] [vsco] add collection & image extractor + video support (#331) --- docs/supportedsites.rst | 2 +- gallery_dl/extractor/vsco.py | 146 +++++++++++++++++++++++++---------- 2 files changed, 106 insertions(+), 42 deletions(-) diff --git a/docs/supportedsites.rst b/docs/supportedsites.rst index 80631ee7..18db7dde 100644 --- a/docs/supportedsites.rst +++ b/docs/supportedsites.rst @@ -107,7 +107,7 @@ The /b/ Archive https://thebarchive.com/ Threads Tsumino https://www.tsumino.com/ Galleries, Search Results Optional Tumblr https://www.tumblr.com/ Images from Users, Likes, Posts, Tag-Searches Optional (OAuth) Twitter https://twitter.com/ Media Timelines, Timelines, Tweets Optional -VSCO https://vsco.co/ Images from Users +VSCO https://vsco.co/ Images from Users, Collections, individual Images Wallhaven https://wallhaven.cc/ individual Images, Search Results Warosu https://warosu.org/ Threads Weibo https://www.weibo.com/ Images from Users, Images from Statuses diff --git a/gallery_dl/extractor/vsco.py b/gallery_dl/extractor/vsco.py index 38569186..099b5b92 100644 --- a/gallery_dl/extractor/vsco.py +++ b/gallery_dl/extractor/vsco.py @@ -17,8 +17,10 @@ BASE_PATTERN = r"(?:https?://)?(?:www\.)?vsco\.co/([^/]+)" class VscoExtractor(Extractor): + """Base class for vsco extractors""" category = "vsco" root = "https://vsco.co" + directory_fmt = ("{category}", "{user}") filename_fmt = "{id}.{extension}" archive_fmt = "{id}" @@ -30,7 +32,7 @@ class VscoExtractor(Extractor): yield Message.Version, 1 yield Message.Directory, {"user": self.user} for img in self.images(): - url = "https://" + img["responsive_url"] + url = "https://" + (img.get("video_url") or img["responsive_url"]) data = text.nameext_from_url(url, { "id" : img["_id"], "user" : self.user, @@ -38,15 +40,21 @@ class VscoExtractor(Extractor): "meta" : img.get("image_meta") or {}, "tags" : [tag["text"] for tag in img.get("tags") or ()], "date" : text.parse_timestamp(img["upload_date"] // 1000), + "video" : img["is_video"], "width" : img["width"], "height": img["height"], "description": img["description"], }) yield Message.Url, url, data - def _pagination(self, url, params, token, extra): - yield from extra + def images(self): + """Return an iterable with all relevant image objects""" + + def _extract_preload_state(self, url): + page = self.request(url, notfound=self.subcategory).text + return json.loads(text.extract(page, "__PRELOADED_STATE__ = ", "<")[0]) + def _pagination(self, url, params, token, key, extra): headers = { "Referer" : "{}/{}".format(self.root, self.user), "Authorization" : "Bearer " + token, @@ -54,58 +62,114 @@ class VscoExtractor(Extractor): "X-Client-Build" : "1", } + yield from map(self._transform_media, extra) + while True: - data = self.request(url, headers=headers, params=params).json() - if not data.get("media"): + data = self.request(url, params=params, headers=headers).json() + if not data.get(key): return - yield from data["media"] + yield from data[key] params["page"] += 1 + @staticmethod + def _transform_media(media): + media["_id"] = media["id"] + media["is_video"] = media["isVideo"] + media["grid_name"] = media["gridName"] + media["upload_date"] = media["uploadDate"] + media["responsive_url"] = media["responsiveUrl"] + media["video_url"] = media.get("videoUrl") + media["image_meta"] = media.get("imageMeta") + return media + class VscoUserExtractor(VscoExtractor): + """Extractor for images from a user on vsco.co""" subcategory = "user" - directory_fmt = ("{category}", "{user}") pattern = BASE_PATTERN + r"/images/" test = ("https://vsco.co/missuri/images/1", { "range": "1-80", "count": 80, "pattern": r"https://im\.vsco\.co/[^/]+/[0-9a-f/]+/vsco\w+\.\w+", - "keyword": { - "id" : str, - "user" : "missuri", - "grid" : "anybodyseenmylife", - "meta" : dict, - "tags" : list, - "date" : "type:datetime", - "width" : int, - "height": int, - "description": str, - }, }) def images(self): url = "{}/{}/images/1".format(self.root, self.user) - page = self.request(url, notfound="user").text - data = json.loads(text.extract(page, "__PRELOADED_STATE__ = ", "<")[0]) - site_id = str(data["sites"]["siteByUsername"][self.user]["site"]["id"]) - token = data["users"]["currentUser"]["tkn"] - - url = "https://vsco.co/api/2.0/medias" - params = { - "site_id": site_id, - "page" : 2, - "size" : "30", - } + data = self._extract_preload_state(url) + + tkn = data["users"]["currentUser"]["tkn"] + sid = str(data["sites"]["siteByUsername"][self.user]["site"]["id"]) + + url = "{}/api/2.0/medias".format(self.root) + params = {"page": 2, "size": "30", "site_id": sid} + return self._pagination(url, params, tkn, "media", ( + data["medias"]["byId"][mid]["media"] + for mid in data["medias"]["bySiteId"][sid]["medias"]["1"] + )) + - extra = [] - medias = data["medias"]["byId"] - for mid in data["medias"]["bySiteId"][site_id]["medias"]["1"]: - media = medias[mid]["media"] - media["_id"] = media["id"] - media["grid_name"] = media["gridName"] - media["image_meta"] = media["imageMeta"] - media["upload_date"] = media["uploadDate"] - media["responsive_url"] = media["responsiveUrl"] - extra.append(media) - - return self._pagination(url, params, token, extra) +class VscoCollectionExtractor(VscoExtractor): + """Extractor for images from a collection on vsco.co""" + subcategory = "collection" + directory_fmt = ("{category}", "{user}", "collection") + pattern = BASE_PATTERN + r"/collection/" + test = ("https://vsco.co/vsco/collection/1", { + "range": "1-80", + "count": 80, + "pattern": r"https://im\.vsco\.co/[^/]+/[0-9a-f/]+/vsco\w+\.\w+", + }) + + def images(self): + url = "{}/{}/collection/1".format(self.root, self.user) + data = self._extract_preload_state(url) + + tkn = data["users"]["currentUser"]["tkn"] + cid = (data["sites"]["siteByUsername"][self.user] + ["site"]["siteCollectionId"]) + + url = "{}/api/2.0/collections/{}/medias".format(self.root, cid) + params = {"page": 2, "size": "20"} + return self._pagination(url, params, tkn, "medias", ( + data["medias"]["byId"][mid]["media"] + for mid in data + ["collections"]["byCollectionId"][cid]["collection"]["1"] + )) + + +class VscoImageExtractor(VscoExtractor): + """Extractor for individual images on vsco.co""" + subcategory = "image" + pattern = BASE_PATTERN + r"/media/([0-9a-fA-F]+)" + test = ( + ("https://vsco.co/erenyildiz/media/5d34b93ef632433030707ce2", { + "url": "faa214d10f859f374ad91da3f7547d2439f5af08", + "content": "1394d070828d82078035f19a92f404557b56b83f", + "keyword": { + "id" : "5d34b93ef632433030707ce2", + "user" : "erenyildiz", + "grid" : "erenyildiz", + "meta" : dict, + "tags" : list, + "date" : "type:datetime", + "video" : False, + "width" : 1537, + "height": 1537, + "description": "re:Ni seviyorum. #vsco #vscox #vscochallenges", + }, + }), + ("https://vsco.co/jimenalazof/media/5b4feec558f6c45c18c040fd", { + "url": "08e7eef3301756ce81206c0b47c1e9373756a74a", + "content": "e739f058d726ee42c51c180a505747972a7dfa47", + "keyword": {"video" : True}, + }), + ) + + def __init__(self, match): + VscoExtractor.__init__(self, match) + self.media_id = match.group(2) + + def images(self): + url = "{}/{}/media/{}".format(self.root, self.user, self.media_id) + data = self._extract_preload_state(url) + media = data["medias"]["byId"].popitem()[1]["media"] + return (self._transform_media(media),)