mirror_maker
/
gallery-dl
mirror of https://github.com/mikf/gallery-dl.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[imgur] update

Browse Source 
- fix image/album detection for galleries
- use new API endpoints for image/album data
pull/997/head
Mike Fährmann 4 years ago
parent b5243297ff
commit 799ca07fc8
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88
1 changed files with 124 additions and 118 deletions
Show Stats Download Patch File Download Diff File

242
gallery_dl/extractor/imgur.py
Unescape View File

@ -27,19 +27,17 @@ class ImgurExtractor(Extractor):
self.mp4 = self.config("mp4", True)
def _prepare(self, image):
try:
del image["ad_url"]
del image["ad_type"]
del image["ad_config"]
except KeyError:
pass
image.update(image["metadata"])
del image["metadata"]
if image["animated"] and self.mp4 and "mp4" in image:
url = image["mp4"]
else:
url = image["link"]
if image["ext"] == "jpeg":
image["ext"] = "jpg"
elif image["is_animated"] and self.mp4 and image["ext"] == "gif":
image["ext"] = "mp4"
image["date"] = text.parse_timestamp(image["datetime"])
image["url"] = url = "https://i.imgur.com/{}.{}".format(
image["id"], image["ext"])
image["date"] = text.parse_datetime(image["created_at"])
text.nameext_from_url(url, image)
return url
@ -65,33 +63,38 @@ class ImgurImageExtractor(ImgurExtractor):
"url": "6f2dcfb86815bdd72808c313e5f715610bc7b9b2",
"content": "0c8768055e4e20e7c7259608b67799171b691140",
"keyword": {
"account_id" : None,
"account_url" : None,
"animated" : False,
"bandwidth" : int,
"date" : "dt:2016-11-10 14:24:35",
"datetime" : 1478787875,
"description" : None,
"edited" : "0",
"extension" : "png",
"favorite" : False,
"filename" : "21yMxCS",
"has_sound" : False,
"height" : 32,
"id" : "21yMxCS",
"in_gallery" : False,
"in_most_viral": False,
"is_ad" : False,
"link" : "https://i.imgur.com/21yMxCS.png",
"nsfw" : False,
"section" : None,
"size" : 182,
"tags" : [],
"title" : "Test",
"type" : "image/png",
"views" : int,
"vote" : None,
"width" : 64,
"account_id" : 0,
"comment_count" : int,
"cover_id" : "21yMxCS",
"date" : "dt:2016-11-10 14:24:35",
"description" : "",
"downvote_count": int,
"duration" : 0,
"ext" : "png",
"favorite" : False,
"favorite_count": 0,
"has_sound" : False,
"height" : 32,
"id" : "21yMxCS",
"image_count" : 1,
"in_most_viral" : False,
"is_ad" : False,
"is_album" : False,
"is_animated" : False,
"is_looping" : False,
"is_mature" : False,
"is_pending" : False,
"mime_type" : "image/png",
"name" : "test-テスト",
"point_count" : int,
"privacy" : "",
"score" : int,
"size" : 182,
"title" : "Test",
"upvote_count" : int,
"url" : "https://i.imgur.com/21yMxCS.png",
"view_count" : int,
"width" : 64,
},
}),
("http://imgur.com/0gybAXR", { # gifv/mp4 video
@ -101,30 +104,32 @@ class ImgurImageExtractor(ImgurExtractor):
("https://imgur.com/XFfsmuC", { # missing title in API response (#467)
"keyword": {"title": "Tears are a natural response to irritants"},
}),
("https://imgur.com/HjoXJAd", { # url ends with '.jpg?1'
"url": "ec2cf11a2bfb4939feff374781a6e6f3e9af8e8e",
}),
("https://imgur.com/1Nily2P", { # animated png
"pattern": "https://i.imgur.com/1Nily2P.png",
}),
("https://imgur.com/zzzzzzz", { # not found
"exception": exception.HttpError,
}),
("https://www.imgur.com/21yMxCS"), # www
("https://m.imgur.com/21yMxCS"), # mobile
("https://imgur.com/zxaY6"), # 5 character key
("https://i.imgur.com/21yMxCS.png"), # direct link
("https://www.imgur.com/21yMxCS"), # www
("https://m.imgur.com/21yMxCS"), # mobile
("https://imgur.com/zxaY6"), # 5 character key
("https://i.imgur.com/21yMxCS.png"), # direct link
("https://i.imgur.com/21yMxCSh.png"), # direct link thumbnail
("https://i.imgur.com/zxaY6.gif"), # direct link (short)
("https://i.imgur.com/zxaY6s.gif"), # direct link (short; thumb)
("https://i.imgur.com/zxaY6.gif"), # direct link (short)
("https://i.imgur.com/zxaY6s.gif"), # direct link (short; thumb)
)
def items(self):
image = self.api.image(self.key)
if not image["title"]:
page = self.request(self.root + "/" + self.key, fatal=False).text
title = text.extract(page, "<title>", "<")[0] or ""
image["title"] = text.unescape(title.rpartition(" - ")[0].strip())
try:
del image["ad_url"]
del image["ad_type"]
except KeyError:
pass
image.update(image["media"][0])
del image["media"]
url = self._prepare(image)
yield Message.Version, 1
yield Message.Directory, image
@ -143,53 +148,49 @@ class ImgurAlbumExtractor(ImgurExtractor):
"url": "ce3552f550a5b5316bd9c7ae02e21e39f30c0563",
"keyword": {
"album": {
"account_id" : None,
"account_url" : None,
"cover" : "693j2Kr",
"cover_edited": None,
"cover_height": 1400,
"cover_width" : 951,
"date" : "dt:2015-10-09 10:37:50",
"datetime" : 1444387070,
"description" : None,
"favorite" : False,
"id" : "TcBmP",
"images_count": 19,
"in_gallery" : False,
"is_ad" : False,
"is_album" : True,
"layout" : "blog",
"link" : "https://imgur.com/a/TcBmP",
"nsfw" : bool,
"privacy" : "hidden",
"section" : None,
"title" : "138",
"views" : int,
"account_id" : 0,
"comment_count" : int,
"cover_id" : "693j2Kr",
"date" : "dt:2015-10-09 10:37:50",
"description" : "",
"downvote_count": 0,
"favorite" : False,
"favorite_count": 0,
"id" : "TcBmP",
"image_count" : 19,
"in_most_viral" : False,
"is_ad" : False,
"is_album" : True,
"is_mature" : False,
"is_pending" : False,
"privacy" : "private",
"score" : int,
"title" : "138",
"topic" : "",
"topic_id" : 0,
"upvote_count" : int,
"url" : "https://imgur.com/a/TcBmP",
"view_count" : int,
"virality" : int,
},
"account_id" : None,
"account_url": None,
"animated" : bool,
"bandwidth" : int,
"account_id" : 0,
"count" : 19,
"date" : "type:datetime",
"datetime" : int,
"description": None,
"edited" : "0",
"favorite" : False,
"description": "",
"ext" : "jpg",
"has_sound" : False,
"height" : int,
"id" : str,
"in_gallery" : False,
"is_ad" : False,
"link" : r"re:https://i\.imgur\.com/\w+\.jpg",
"nsfw" : None,
"is_animated": False,
"is_looping" : False,
"mime_type" : "image/jpeg",
"name" : str,
"num" : int,
"section" : None,
"size" : int,
"tags" : list,
"title" : None,
"type" : "image/jpeg",
"views" : int,
"vote" : None,
"title" : str,
"type" : "image",
"updated_at" : None,
"url" : str,
"width" : int,
},
}),
@ -208,13 +209,15 @@ class ImgurAlbumExtractor(ImgurExtractor):
def items(self):
album = self.api.album(self.key)
album["date"] = text.parse_timestamp(album["datetime"])
images = album["images"]
album["date"] = text.parse_datetime(album["created_at"])
images = album["media"]
del album["media"]
count = len(images)
try:
del album["images"]
del album["ad_config"]
del album["ad_url"]
del album["ad_type"]
except KeyError:
pass
@ -239,22 +242,17 @@ class ImgurGalleryExtractor(ImgurExtractor):
("https://imgur.com/gallery/eD9CT", {
"pattern": "https://imgur.com/a/eD9CT",
}),
("https://imgur.com/t/unmuted/26sEhNr", { # unmuted URL
"pattern": "https://imgur.com/26sEhNr",
}),
("https://imgur.com/t/unmuted/26sEhNr"),
("https://imgur.com/t/cat/qSB8NbN"),
)
def items(self):
url = self.root + "/a/" + self.key
with self.request(url, method="HEAD", fatal=False) as response:
if response.status_code < 400:
extr = ImgurAlbumExtractor
else:
extr = ImgurImageExtractor
url = self.root + "/" + self.key
yield Message.Version, 1
if self.api.gallery(self.key)["is_album"]:
url = "{}/a/{}".format(self.root, self.key)
extr = ImgurAlbumExtractor
else:
url = "{}/{}".format(self.root, self.key)
extr = ImgurImageExtractor
yield Message.Queue, url, {"_extractor": extr}
@ -346,38 +344,46 @@ class ImgurAPI():
}
def account_favorites(self, account):
endpoint = "account/{}/gallery_favorites".format(account)
endpoint = "/3/account/{}/gallery_favorites".format(account)
return self._pagination(endpoint)
def gallery_search(self, query):
endpoint = "gallery/search"
endpoint = "/3/gallery/search"
params = {"q": query}
return self._pagination(endpoint, params)
def account_submissions(self, account):
endpoint = "account/{}/submissions".format(account)
endpoint = "/3/account/{}/submissions".format(account)
return self._pagination(endpoint)
def gallery_subreddit(self, subreddit):
endpoint = "gallery/r/{}".format(subreddit)
endpoint = "/3/gallery/r/{}".format(subreddit)
return self._pagination(endpoint)
def gallery_tag(self, tag):
endpoint = "gallery/t/{}".format(tag)
endpoint = "/3/gallery/t/{}".format(tag)
return self._pagination(endpoint, key="items")
def image(self, image_hash):
endpoint = "/post/v1/media/" + image_hash
params = {"include": "media,tags,account"}
return self._call(endpoint, params)
def album(self, album_hash):
return self._call("album/" + album_hash)
endpoint = "/post/v1/albums/" + album_hash
params = {"include": "media,tags,account"}
return self._call(endpoint, params)
def image(self, image_hash):
return self._call("image/" + image_hash)
def gallery(self, gallery_hash):
endpoint = "/post/v1/posts/" + gallery_hash
return self._call(endpoint)
def _call(self, endpoint, params=None):
try:
return self.extractor.request(
"https://api.imgur.com/3/" + endpoint,
"https://api.imgur.com" + endpoint,
params=params, headers=self.headers,
).json()["data"]
).json()
except exception.HttpError as exc:
if exc.status != 403 or b"capacity" not in exc.response.content:
raise
@ -388,7 +394,7 @@ class ImgurAPI():
num = 0
while True:
data = self._call("{}/{}".format(endpoint, num), params)
data = self._call("{}/{}".format(endpoint, num), params)["data"]
if key:
data = data[key]
if not data:

Write Preview
Loading…
Cancel
Save