|
|
@ -236,16 +236,20 @@ class NewgroundsExtractor(Extractor):
|
|
|
|
yield fmt[1][0]["src"]
|
|
|
|
yield fmt[1][0]["src"]
|
|
|
|
|
|
|
|
|
|
|
|
def _pagination(self, kind):
|
|
|
|
def _pagination(self, kind):
|
|
|
|
root = self.user_root
|
|
|
|
url = "{}/{}".format(self.user_root, kind)
|
|
|
|
|
|
|
|
params = {
|
|
|
|
|
|
|
|
"page": 1,
|
|
|
|
|
|
|
|
"isAjaxRequest": "1",
|
|
|
|
|
|
|
|
}
|
|
|
|
headers = {
|
|
|
|
headers = {
|
|
|
|
"Accept": "application/json, text/javascript, */*; q=0.01",
|
|
|
|
"Referer": url,
|
|
|
|
"X-Requested-With": "XMLHttpRequest",
|
|
|
|
"X-Requested-With": "XMLHttpRequest",
|
|
|
|
"Referer": root,
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
url = "{}/{}/page/1".format(root, kind)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
while True:
|
|
|
|
while True:
|
|
|
|
with self.request(url, headers=headers, fatal=False) as response:
|
|
|
|
with self.request(
|
|
|
|
|
|
|
|
url, params=params, headers=headers,
|
|
|
|
|
|
|
|
fatal=False) as response:
|
|
|
|
try:
|
|
|
|
try:
|
|
|
|
data = response.json()
|
|
|
|
data = response.json()
|
|
|
|
except ValueError:
|
|
|
|
except ValueError:
|
|
|
@ -256,14 +260,17 @@ class NewgroundsExtractor(Extractor):
|
|
|
|
msg = ", ".join(text.unescape(e) for e in data["errors"])
|
|
|
|
msg = ", ".join(text.unescape(e) for e in data["errors"])
|
|
|
|
raise exception.StopExtraction(msg)
|
|
|
|
raise exception.StopExtraction(msg)
|
|
|
|
|
|
|
|
|
|
|
|
for year in data["sequence"]:
|
|
|
|
for year, items in data["items"].items():
|
|
|
|
for item in data["years"][str(year)]["items"]:
|
|
|
|
for item in items:
|
|
|
|
page_url = text.extract(item, 'href="', '"')[0]
|
|
|
|
page_url = text.extract(item, 'href="', '"')[0]
|
|
|
|
yield text.urljoin(root, page_url)
|
|
|
|
if page_url[0] == "/":
|
|
|
|
|
|
|
|
page_url = self.root + page_url
|
|
|
|
|
|
|
|
yield page_url
|
|
|
|
|
|
|
|
|
|
|
|
if not data["more"]:
|
|
|
|
more = data.get("load_more")
|
|
|
|
|
|
|
|
if not more or len(more) < 8:
|
|
|
|
return
|
|
|
|
return
|
|
|
|
url = text.urljoin(root, data["more"])
|
|
|
|
params["page"] += 1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class NewgroundsImageExtractor(NewgroundsExtractor):
|
|
|
|
class NewgroundsImageExtractor(NewgroundsExtractor):
|
|
|
@ -301,9 +308,9 @@ class NewgroundsImageExtractor(NewgroundsExtractor):
|
|
|
|
"count": 2,
|
|
|
|
"count": 2,
|
|
|
|
}),
|
|
|
|
}),
|
|
|
|
# "adult" rated (#2456)
|
|
|
|
# "adult" rated (#2456)
|
|
|
|
("https://www.newgrounds.com/art/view/bdoneart/vampire-booty", {
|
|
|
|
("https://www.newgrounds.com/art/view/kekiiro/red", {
|
|
|
|
"options": (("username", None),),
|
|
|
|
"options": (("username", None),),
|
|
|
|
"count": 0,
|
|
|
|
"count": 1,
|
|
|
|
}),
|
|
|
|
}),
|
|
|
|
)
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
@ -374,7 +381,7 @@ class NewgroundsMediaExtractor(NewgroundsExtractor):
|
|
|
|
# "adult" rated (#2456)
|
|
|
|
# "adult" rated (#2456)
|
|
|
|
("https://www.newgrounds.com/portal/view/717744", {
|
|
|
|
("https://www.newgrounds.com/portal/view/717744", {
|
|
|
|
"options": (("username", None),),
|
|
|
|
"options": (("username", None),),
|
|
|
|
"count": 0,
|
|
|
|
"count": 1,
|
|
|
|
}),
|
|
|
|
}),
|
|
|
|
)
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
@ -470,25 +477,28 @@ class NewgroundsFavoriteExtractor(NewgroundsExtractor):
|
|
|
|
)
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
def _pagination(self, kind):
|
|
|
|
def _pagination(self, kind):
|
|
|
|
num = 1
|
|
|
|
url = "{}/favorites/{}".format(self.user_root, kind)
|
|
|
|
|
|
|
|
params = {
|
|
|
|
|
|
|
|
"page": 1,
|
|
|
|
|
|
|
|
"isAjaxRequest": "1",
|
|
|
|
|
|
|
|
}
|
|
|
|
headers = {
|
|
|
|
headers = {
|
|
|
|
"Accept": "application/json, text/javascript, */*; q=0.01",
|
|
|
|
"Referer": url,
|
|
|
|
"X-Requested-With": "XMLHttpRequest",
|
|
|
|
"X-Requested-With": "XMLHttpRequest",
|
|
|
|
"Referer": self.user_root,
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
while True:
|
|
|
|
while True:
|
|
|
|
url = "{}/favorites/{}/{}".format(self.user_root, kind, num)
|
|
|
|
response = self.request(url, params=params, headers=headers)
|
|
|
|
response = self.request(url, headers=headers)
|
|
|
|
|
|
|
|
if response.history:
|
|
|
|
if response.history:
|
|
|
|
return
|
|
|
|
return
|
|
|
|
|
|
|
|
|
|
|
|
favs = self._extract_favorites(response.text)
|
|
|
|
data = response.json()
|
|
|
|
|
|
|
|
favs = self._extract_favorites(data.get("component") or "")
|
|
|
|
yield from favs
|
|
|
|
yield from favs
|
|
|
|
|
|
|
|
|
|
|
|
if len(favs) < 24:
|
|
|
|
if len(favs) < 24:
|
|
|
|
return
|
|
|
|
return
|
|
|
|
num += 1
|
|
|
|
params["page"] += 1
|
|
|
|
|
|
|
|
|
|
|
|
def _extract_favorites(self, page):
|
|
|
|
def _extract_favorites(self, page):
|
|
|
|
return [
|
|
|
|
return [
|
|
|
|