[poringa] improvements and fixes

- add 'num' and 'count' metadata fields
- prevent crash for "private" posts
- prevent crash when there's no 'main-info'
- update tests

gallery_dl/extractor/poringa.py

@@ -17,8 +17,8 @@ BASE_PATTERN = r"(?:https?://)?(?:www\.)?poringa\.net"
 class PoringaExtractor(Extractor):
     category = "poringa"
     directory_fmt = ("{category}", "{user}", "{post_id}")
-    filename_fmt = "{post_id}_{title}_{filename}.{extension}"
-    archive_fmt = "{post_id}"
+    filename_fmt = "{post_id}_{title}_{num:>03}_{filename}.{extension}"
+    archive_fmt = "{post_id}_{num}"
     root = "http://www.poringa.net"
 
     def __init__(self, match):
@@ -31,36 +31,45 @@ class PoringaExtractor(Extractor):
             url = "{}/posts/imagenes/{}".format(self.root, post_id)
 
             try:
-                page = self.request(url).text
+                response = self.request(url)
             except exception.HttpError as exc:
                 self.log.warning(
                     "Unable to fetch posts for '%s' (%s)", post_id, exc)
                 continue
 
+            if "/registro-login?" in response.url:
+                self.log.warning("Private post '%s'", post_id)
+                continue
+
+            page = response.text
             title, pos = text.extract(
                 page, 'property="og:title" content="', '"')
-            pos = page.index('<div class="main-info', pos)
-            user, pos = text.extract(
-                page, 'href="http://www.poringa.net/', '"', pos)
+
+            try:
+                pos = page.index('<div class="main-info', pos)
+                user, pos = text.extract(
+                    page, 'href="http://www.poringa.net/', '"', pos)
+            except ValueError:
+                user = None
 
             if not user:
                 user = "poringa"
 
             data = {
-                "post_id"     : post_id,
+                "post_id"      : post_id,
                 "title"        : text.unescape(title),
                 "user"         : text.unquote(user),
                 "_http_headers": {"Referer": url},
             }
 
-            yield Message.Directory, data
             main_post = text.extr(
                 page, 'property="dc:content" role="main">', '</div>')
-            for url in text.extract_iter(
-                main_post,
-                '<img class="imagen" border="0" src="',
-                '"',
-            ):
+            urls = list(text.extract_iter(
+                main_post, '<img class="imagen" border="0" src="', '"'))
+            data["count"] = len(urls)
+
+            yield Message.Directory, data
+            for data["num"], url in enumerate(urls, 1):
                 yield Message.Url, url, text.nameext_from_url(url, data)
 
     def posts(self):
@@ -95,8 +104,8 @@ class PoringaExtractor(Extractor):
 class PoringaPostExtractor(PoringaExtractor):
     """Extractor for posts on poringa.net"""
     subcategory = "post"
-    pattern = BASE_PATTERN + r"/posts/imagenes/(\d+)/[a-zA-Z0-9_-]+\.html"
-    example = "http://www.poringa.net/posts/imagenes/12/TITLE.html"
+    pattern = BASE_PATTERN + r"/posts/imagenes/(\d+)"
+    example = "http://www.poringa.net/posts/imagenes/12345/TITLE.html"
 
     def posts(self):
         return (self.item,)
@@ -104,12 +113,12 @@ class PoringaPostExtractor(PoringaExtractor):
 
 class PoringaUserExtractor(PoringaExtractor):
     subcategory = "user"
-    pattern = BASE_PATTERN + r"/([a-zA-Z0-9_-]+)$"
+    pattern = BASE_PATTERN + r"/(\w+)$"
     example = "http://www.poringa.net/USER"
 
     def posts(self):
-        url = "{}/buscar/".format(self.root)
-        params = {"q": text.unquote(self.item)}
+        url = self.root + "/buscar/"
+        params = {"q": self.item}
         return self._pagination(url, params)
 
 
@@ -120,7 +129,7 @@ class PoringaSearchExtractor(PoringaExtractor):
 
     def posts(self):
         url = self.root + "/buscar/"
-        params = {"q": text.unquote(self.item)}
+        params = {"q": self.item}
         return self._pagination(url, params)
 
 

test/results/poringa.py

@@ -12,36 +12,43 @@ __tests__ = (
     "#url"     : "http://www.poringa.net/posts/imagenes/3051081/Turrita-alto-ojete.html",
     "#category": ("", "poringa", "post"),
     "#class"   : poringa.PoringaPostExtractor,
-    "#pattern" : r"http://www\.poringa\.net/posts/imagenes/3051081/[a-zA-Z0-9_-]+\.html",
+    "#count"   : 26,
 
-    "post_id" : "3051081",
-    "title"   : "turrita alto ojete...",
-    "user"    : "vipower1top",
+    "count"    : 26,
+    "num"      : range(1, 26),
+    "post_id"  : "3051081",
+    "title"    : "turrita alto ojete...",
+    "user"     : "vipower1top",
 },
 
 {
     "#url"     : "http://www.poringa.net/posts/imagenes/3095554/Otra-culona-de-instagram.html",
     "#category": ("", "poringa", "post"),
     "#class"   : poringa.PoringaPostExtractor,
-    "#pattern" : r"http://www\.poringa\.net/posts/imagenes/3095554/[a-zA-Z0-9_-]+\.html",
+    "#count"   : 15,
 
-    "post_id" : "3095554",
-    "title"   : "Otra culona de instagram",
-    "user"    : "Expectro007",
+    "count"    : 15,
+    "num"      : range(1, 15),
+    "post_id"  : "3095554",
+    "title"    : "Otra culona de instagram",
+    "user"     : "Expectro007",
 },
 
 {
     "#url"     : "http://www.poringa.net/Expectro007",
     "#category": ("", "poringa", "user"),
     "#class"   : poringa.PoringaUserExtractor,
-    "#pattern" : r"https?://img-[0-9]\.poringa\.net/poringa/img/[a-zA-Z0-9/{2}]{12}[a-zA-Z0-9-_]+/[a-zA-Z0-9-_]+\.jpg",
+    "#pattern" : r"https?://img-\d+\.poringa\.net/poringa/img/././././././Expectro007/\w{3}\.(jpg|png|gif)",
+    "#count"   : range(500, 600),
 },
 
 {
     "#url"     : "http://www.poringa.net/buscar/?&q=yuslopez",
     "#category": ("", "poringa", "search"),
     "#class"   : poringa.PoringaSearchExtractor,
-    "#pattern" : r"https?://img-[0-9]\.poringa\.net/poringa/img/[a-zA-Z0-9/{2}]{12}[a-zA-Z0-9-_]+/[a-zA-Z0-9-_]+\.jpg",
+    "#pattern" : r"https?://img-\d+\.poringa\.net/poringa/img/././././././\w+/\w{3}\.(jpg|png|gif)",
+    "#range"   : "1-50",
+    "#count"   : 50,
 },
 
 )