|
|
|
@ -10,7 +10,6 @@
|
|
|
|
|
|
|
|
|
|
from .common import Extractor, Message
|
|
|
|
|
from .. import text
|
|
|
|
|
import re
|
|
|
|
|
|
|
|
|
|
BASE_PATTERN = r"(?:https://)?(?:www\.|m\.)?vk\.com"
|
|
|
|
|
|
|
|
|
@ -25,47 +24,63 @@ class VkExtractor(Extractor):
|
|
|
|
|
request_interval = 1.0
|
|
|
|
|
|
|
|
|
|
def items(self):
|
|
|
|
|
sizes = "wzyxrqpo"
|
|
|
|
|
|
|
|
|
|
data = self.metadata()
|
|
|
|
|
yield Message.Directory, data
|
|
|
|
|
|
|
|
|
|
for photo in self.photos():
|
|
|
|
|
|
|
|
|
|
for size in sizes:
|
|
|
|
|
size += "_"
|
|
|
|
|
if size in photo:
|
|
|
|
|
break
|
|
|
|
|
else:
|
|
|
|
|
self.log.warning("no photo URL found (%s)", photo.get("id"))
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
|
|
photo.update(data)
|
|
|
|
|
yield Message.Url, photo["url"], photo
|
|
|
|
|
photo["url"], photo["width"], photo["height"] = photo[size]
|
|
|
|
|
photo["id"] = photo["id"].rpartition("_")[2]
|
|
|
|
|
|
|
|
|
|
def _pagination(self, photos_url, user_id):
|
|
|
|
|
sub = re.compile(r"/imp[fg]/").sub
|
|
|
|
|
needle = 'data-id="{}_'.format(user_id)
|
|
|
|
|
text.nameext_from_url(photo["url"], photo)
|
|
|
|
|
yield Message.Url, photo["url"], photo
|
|
|
|
|
|
|
|
|
|
def _pagination(self, photos_id):
|
|
|
|
|
url = self.root + "/al_photos.php"
|
|
|
|
|
headers = {
|
|
|
|
|
"X-Requested-With": "XMLHttpRequest",
|
|
|
|
|
"Origin" : self.root,
|
|
|
|
|
"Referer" : photos_url,
|
|
|
|
|
"Referer" : self.root + "/" + photos_id,
|
|
|
|
|
}
|
|
|
|
|
params = {
|
|
|
|
|
"al" : "1",
|
|
|
|
|
"al_ad" : "0",
|
|
|
|
|
"offset": 0,
|
|
|
|
|
"part" : "1",
|
|
|
|
|
data = {
|
|
|
|
|
"act" : "show",
|
|
|
|
|
"al" : "1",
|
|
|
|
|
"direction": "1",
|
|
|
|
|
"list" : photos_id,
|
|
|
|
|
"offset" : 0,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
while True:
|
|
|
|
|
payload = self.request(
|
|
|
|
|
photos_url, method="POST", headers=headers, data=params
|
|
|
|
|
url, method="POST", headers=headers, data=data,
|
|
|
|
|
).json()["payload"][1]
|
|
|
|
|
|
|
|
|
|
offset = payload[0]
|
|
|
|
|
html = payload[1]
|
|
|
|
|
total = payload[1]
|
|
|
|
|
photos = payload[3]
|
|
|
|
|
|
|
|
|
|
cnt = 0
|
|
|
|
|
for photo in text.extract_iter(html, needle, ')'):
|
|
|
|
|
cnt += 1
|
|
|
|
|
pid = photo[:photo.find('"')]
|
|
|
|
|
url = photo[photo.rindex("(")+1:]
|
|
|
|
|
url = sub("/", url.partition("?")[0])
|
|
|
|
|
yield text.nameext_from_url(url, {"url": url, "id": pid})
|
|
|
|
|
data["offset"] += len(photos)
|
|
|
|
|
if data["offset"] >= total:
|
|
|
|
|
# the last chunk of photos also contains the first few photos
|
|
|
|
|
# again if 'total' is not a multiple of 10
|
|
|
|
|
extra = total - data["offset"]
|
|
|
|
|
if extra:
|
|
|
|
|
del photos[extra:]
|
|
|
|
|
|
|
|
|
|
if cnt <= 20 or offset == params["offset"]:
|
|
|
|
|
yield from photos
|
|
|
|
|
return
|
|
|
|
|
params["offset"] = offset
|
|
|
|
|
|
|
|
|
|
yield from photos
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class VkPhotosExtractor(VkExtractor):
|
|
|
|
@ -76,8 +91,8 @@ class VkPhotosExtractor(VkExtractor):
|
|
|
|
|
r"|(?!album-?\d+_)([^/?#]+))")
|
|
|
|
|
test = (
|
|
|
|
|
("https://vk.com/id398982326", {
|
|
|
|
|
"pattern": r"https://sun\d+-\d+\.userapi\.com"
|
|
|
|
|
r"/\w+/v\d+/[0-9a-f]+/[\w-]+\.jpg",
|
|
|
|
|
"pattern": r"https://sun\d+-\d+\.userapi\.com/s/v1/if1"
|
|
|
|
|
r"/[\w-]+\.jpg\?size=\d+x\d+&quality=96&type=album",
|
|
|
|
|
"count": ">= 35",
|
|
|
|
|
"keywords": {
|
|
|
|
|
"id": r"re:\d+",
|
|
|
|
@ -90,7 +105,7 @@ class VkPhotosExtractor(VkExtractor):
|
|
|
|
|
},
|
|
|
|
|
}),
|
|
|
|
|
("https://vk.com/cosplayinrussia", {
|
|
|
|
|
"range": "75-100",
|
|
|
|
|
"range": "25-35",
|
|
|
|
|
"keywords": {
|
|
|
|
|
"id": r"re:\d+",
|
|
|
|
|
"user": {
|
|
|
|
@ -112,8 +127,7 @@ class VkPhotosExtractor(VkExtractor):
|
|
|
|
|
self.user_id, self.user_name = match.groups()
|
|
|
|
|
|
|
|
|
|
def photos(self):
|
|
|
|
|
url = "{}/photos{}".format(self.root, self.user_id)
|
|
|
|
|
return self._pagination(url, self.user_id)
|
|
|
|
|
return self._pagination("photos" + self.user_id)
|
|
|
|
|
|
|
|
|
|
def metadata(self):
|
|
|
|
|
if self.user_id:
|
|
|
|
@ -159,8 +173,8 @@ class VkAlbumExtractor(VkExtractor):
|
|
|
|
|
self.user_id, self.album_id = match.groups()
|
|
|
|
|
|
|
|
|
|
def photos(self):
|
|
|
|
|
url = "{}/album{}_{}".format(self.root, self.user_id, self.album_id)
|
|
|
|
|
return self._pagination(url, self.user_id)
|
|
|
|
|
return self._pagination("album{}_{}".format(
|
|
|
|
|
self.user_id, self.album_id))
|
|
|
|
|
|
|
|
|
|
def metadata(self):
|
|
|
|
|
return {
|
|
|
|
|