|
|
@ -17,8 +17,8 @@ BASE_PATTERN = r"(?:https?://)?(?:www\.)?poringa\.net"
|
|
|
|
class PoringaExtractor(Extractor):
|
|
|
|
class PoringaExtractor(Extractor):
|
|
|
|
category = "poringa"
|
|
|
|
category = "poringa"
|
|
|
|
directory_fmt = ("{category}", "{user}", "{post_id}")
|
|
|
|
directory_fmt = ("{category}", "{user}", "{post_id}")
|
|
|
|
filename_fmt = "{post_id}_{title}_{filename}.{extension}"
|
|
|
|
filename_fmt = "{post_id}_{title}_{num:>03}_{filename}.{extension}"
|
|
|
|
archive_fmt = "{post_id}"
|
|
|
|
archive_fmt = "{post_id}_{num}"
|
|
|
|
root = "http://www.poringa.net"
|
|
|
|
root = "http://www.poringa.net"
|
|
|
|
|
|
|
|
|
|
|
|
def __init__(self, match):
|
|
|
|
def __init__(self, match):
|
|
|
@ -31,17 +31,26 @@ class PoringaExtractor(Extractor):
|
|
|
|
url = "{}/posts/imagenes/{}".format(self.root, post_id)
|
|
|
|
url = "{}/posts/imagenes/{}".format(self.root, post_id)
|
|
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
try:
|
|
|
|
page = self.request(url).text
|
|
|
|
response = self.request(url)
|
|
|
|
except exception.HttpError as exc:
|
|
|
|
except exception.HttpError as exc:
|
|
|
|
self.log.warning(
|
|
|
|
self.log.warning(
|
|
|
|
"Unable to fetch posts for '%s' (%s)", post_id, exc)
|
|
|
|
"Unable to fetch posts for '%s' (%s)", post_id, exc)
|
|
|
|
continue
|
|
|
|
continue
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if "/registro-login?" in response.url:
|
|
|
|
|
|
|
|
self.log.warning("Private post '%s'", post_id)
|
|
|
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
page = response.text
|
|
|
|
title, pos = text.extract(
|
|
|
|
title, pos = text.extract(
|
|
|
|
page, 'property="og:title" content="', '"')
|
|
|
|
page, 'property="og:title" content="', '"')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
pos = page.index('<div class="main-info', pos)
|
|
|
|
pos = page.index('<div class="main-info', pos)
|
|
|
|
user, pos = text.extract(
|
|
|
|
user, pos = text.extract(
|
|
|
|
page, 'href="http://www.poringa.net/', '"', pos)
|
|
|
|
page, 'href="http://www.poringa.net/', '"', pos)
|
|
|
|
|
|
|
|
except ValueError:
|
|
|
|
|
|
|
|
user = None
|
|
|
|
|
|
|
|
|
|
|
|
if not user:
|
|
|
|
if not user:
|
|
|
|
user = "poringa"
|
|
|
|
user = "poringa"
|
|
|
@ -53,14 +62,14 @@ class PoringaExtractor(Extractor):
|
|
|
|
"_http_headers": {"Referer": url},
|
|
|
|
"_http_headers": {"Referer": url},
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
yield Message.Directory, data
|
|
|
|
|
|
|
|
main_post = text.extr(
|
|
|
|
main_post = text.extr(
|
|
|
|
page, 'property="dc:content" role="main">', '</div>')
|
|
|
|
page, 'property="dc:content" role="main">', '</div>')
|
|
|
|
for url in text.extract_iter(
|
|
|
|
urls = list(text.extract_iter(
|
|
|
|
main_post,
|
|
|
|
main_post, '<img class="imagen" border="0" src="', '"'))
|
|
|
|
'<img class="imagen" border="0" src="',
|
|
|
|
data["count"] = len(urls)
|
|
|
|
'"',
|
|
|
|
|
|
|
|
):
|
|
|
|
yield Message.Directory, data
|
|
|
|
|
|
|
|
for data["num"], url in enumerate(urls, 1):
|
|
|
|
yield Message.Url, url, text.nameext_from_url(url, data)
|
|
|
|
yield Message.Url, url, text.nameext_from_url(url, data)
|
|
|
|
|
|
|
|
|
|
|
|
def posts(self):
|
|
|
|
def posts(self):
|
|
|
@ -95,8 +104,8 @@ class PoringaExtractor(Extractor):
|
|
|
|
class PoringaPostExtractor(PoringaExtractor):
|
|
|
|
class PoringaPostExtractor(PoringaExtractor):
|
|
|
|
"""Extractor for posts on poringa.net"""
|
|
|
|
"""Extractor for posts on poringa.net"""
|
|
|
|
subcategory = "post"
|
|
|
|
subcategory = "post"
|
|
|
|
pattern = BASE_PATTERN + r"/posts/imagenes/(\d+)/[a-zA-Z0-9_-]+\.html"
|
|
|
|
pattern = BASE_PATTERN + r"/posts/imagenes/(\d+)"
|
|
|
|
example = "http://www.poringa.net/posts/imagenes/12/TITLE.html"
|
|
|
|
example = "http://www.poringa.net/posts/imagenes/12345/TITLE.html"
|
|
|
|
|
|
|
|
|
|
|
|
def posts(self):
|
|
|
|
def posts(self):
|
|
|
|
return (self.item,)
|
|
|
|
return (self.item,)
|
|
|
@ -104,12 +113,12 @@ class PoringaPostExtractor(PoringaExtractor):
|
|
|
|
|
|
|
|
|
|
|
|
class PoringaUserExtractor(PoringaExtractor):
|
|
|
|
class PoringaUserExtractor(PoringaExtractor):
|
|
|
|
subcategory = "user"
|
|
|
|
subcategory = "user"
|
|
|
|
pattern = BASE_PATTERN + r"/([a-zA-Z0-9_-]+)$"
|
|
|
|
pattern = BASE_PATTERN + r"/(\w+)$"
|
|
|
|
example = "http://www.poringa.net/USER"
|
|
|
|
example = "http://www.poringa.net/USER"
|
|
|
|
|
|
|
|
|
|
|
|
def posts(self):
|
|
|
|
def posts(self):
|
|
|
|
url = "{}/buscar/".format(self.root)
|
|
|
|
url = self.root + "/buscar/"
|
|
|
|
params = {"q": text.unquote(self.item)}
|
|
|
|
params = {"q": self.item}
|
|
|
|
return self._pagination(url, params)
|
|
|
|
return self._pagination(url, params)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@ -120,7 +129,7 @@ class PoringaSearchExtractor(PoringaExtractor):
|
|
|
|
|
|
|
|
|
|
|
|
def posts(self):
|
|
|
|
def posts(self):
|
|
|
|
url = self.root + "/buscar/"
|
|
|
|
url = self.root + "/buscar/"
|
|
|
|
params = {"q": text.unquote(self.item)}
|
|
|
|
params = {"q": self.item}
|
|
|
|
return self._pagination(url, params)
|
|
|
|
return self._pagination(url, params)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|