smaller code and text changes

gallery_dl/config.py

@@ -66,9 +66,8 @@ def load(*files, format="json", strict=False):
             if strict:
                 log.error("Configuration file '%s' not found", path)
                 sys.exit(1)
-        except Exception as exception:
-            log.warning("Could not parse '%s'", path)
-            log.warning(exception)
+        except Exception as exc:
+            log.warning("Could not parse '%s':  %s", path, exc)
             if strict:
                 sys.exit(2)
 

gallery_dl/extractor/common.py

@@ -117,8 +117,8 @@ class AsynchronousExtractor(Extractor):
 
     def __init__(self):
         Extractor.__init__(self)
-        queue_size = int(config.get(("queue-size",), default=5))
-        self.__queue = queue.Queue(maxsize=queue_size)
+        queue_size = int(config.get(("queue-size",), 5))
+        self.__queue = queue.Queue(queue_size)
         self.__thread = threading.Thread(target=self.async_items, daemon=True)
 
     def __iter__(self):
@@ -140,8 +140,8 @@ class AsynchronousExtractor(Extractor):
         try:
             for task in self.items():
                 put(task)
-        except Exception as e:
-            put(e)
+        except Exception as exc:
+            put(exc)
         put(None)
 
 

gallery_dl/extractor/pixiv.py

@@ -268,9 +268,7 @@ class PixivRankingExtractor(PixivExtractor):
                r"ranking\.php(?:\?([^#]*))?"]
     test = [
         (("https://www.pixiv.net/ranking.php"
-          "?mode=daily&content=illust&date=20170818"), {
-            "pattern": r"^https?://i\d*\.pixiv\.net/img-original/img/.+/\d+_p",
-        }),
+          "?mode=daily&content=illust&date=20170818"), None),
         ("https://www.pixiv.net/ranking.php", None),
     ]
 

gallery_dl/job.py

@@ -17,7 +17,7 @@ class Job():
     """Base class for Job-types"""
     ufile = None
 
-    def __init__(self, url):
+    def __init__(self, url, parent=None):
         self.url = url
         self.extractor = extractor.find(url)
         if self.extractor is None:
@@ -46,6 +46,11 @@ class Job():
             predicates.append(util.RangePredicate(chapter["range"]))
         self.pred_queue = util.build_predicate(predicates)
 
+        # category transfer
+        if parent and parent.extractor.categorytransfer:
+            self.extractor.category = parent.extractor.category
+            self.extractor.subcategory = parent.extractor.subcategory
+
     def run(self):
         """Execute or run the job"""
         try:
@@ -121,12 +126,6 @@ class Job():
         kwdict["category"] = self.extractor.category
         kwdict["subcategory"] = self.extractor.subcategory
 
-    def _prepare(self, job):
-        if self.extractor.categorytransfer:
-            job.extractor.category = self.extractor.category
-            job.extractor.subcategory = self.extractor.subcategory
-        return job
-
     def _write_unsupported(self, url):
         if self.ufile:
             print(url, file=self.ufile, flush=True)
@@ -135,7 +134,7 @@ class Job():
 class DownloadJob(Job):
     """Download images into appropriate directory/filename locations"""
 
-    def __init__(self, url):
+    def __init__(self, url, parent=None):
         Job.__init__(self, url)
         self.pathfmt = util.PathFormat(self.extractor)
         self.downloaders = {}
@@ -156,7 +155,7 @@ class DownloadJob(Job):
 
     def handle_queue(self, url, keywords):
         try:
-            self._prepare(DownloadJob(url)).run()
+            DownloadJob(url, self).run()
         except exception.NoExtractorError:
             self._write_unsupported(url)
 
@@ -178,8 +177,8 @@ class KeywordJob(Job):
     """Print available keywords"""
 
     def handle_url(self, url, keywords):
-        print("\nKeywords for filenames:")
-        print("-----------------------")
+        print("\nKeywords for filenames and --filter:")
+        print("------------------------------------")
         self.print_keywords(keywords)
         raise exception.StopExtraction()
 
@@ -200,7 +199,7 @@ class KeywordJob(Job):
             self.print_keywords(keywords)
             if self.extractor.categorytransfer:
                 print()
-                self._prepare(KeywordJob(url)).run()
+                KeywordJob(url, self).run()
         raise exception.StopExtraction()
 
     @staticmethod
@@ -230,8 +229,8 @@ class UrlJob(Job):
     """Print download urls"""
     maxdepth = -1
 
-    def __init__(self, url, depth=1):
-        Job.__init__(self, url)
+    def __init__(self, url, parent=None, depth=1):
+        Job.__init__(self, url, parent)
         self.depth = depth
         if depth == self.maxdepth:
             self.handle_queue = self.handle_url
@@ -242,7 +241,7 @@ class UrlJob(Job):
 
     def handle_queue(self, url, _):
         try:
-            self._prepare(UrlJob(url, self.depth + 1)).run()
+            UrlJob(url, self, self.depth + 1).run()
         except exception.NoExtractorError:
             self._write_unsupported(url)
 

gallery_dl/option.py

@@ -71,7 +71,7 @@ def build_parser():
     general.add_argument(
         "-h", "--help",
         action="help",
-        help="Print this help message and exit"
+        help="Print this help message and exit",
     )
     general.add_argument(
         "--version",
@@ -121,7 +121,8 @@ def build_parser():
     )
     output.add_argument(
         "-K", "--list-keywords", dest="list_keywords", action="store_true",
-        help="Print a list of available keywords for the given URLs",
+        help=("Print a list of available keywords and example values "
+              "for the given URLs"),
     )
     output.add_argument(
         "--list-modules", dest="list_modules", action="store_true",
@@ -148,7 +149,7 @@ def build_parser():
         "--abort-on-skip",
         action=ConfigConstAction, nargs=0, dest="skip", const="abort",
         help=("Abort extractor run if a file download would normally be "
-              "skipped, i.e. if a file with the same filename already exists")
+              "skipped, i.e. if a file with the same filename already exists"),
     )
 
     configuration = parser.add_argument_group("Configuration Options")
@@ -193,13 +194,14 @@ def build_parser():
         metavar="RANGE", dest="image_range",
         help=("Specify which images to download through a comma seperated list"
               " of indices or index-ranges; "
-              "for example '--images -2,4,6-8,10-' will download images with "
+              "for example '--range -2,4,6-8,10-' will download images with "
               "index 1, 2, 4, 6, 7, 8 and 10 up to the last one"),
     )
     selection.add_argument(
         "--chapter-range",
         metavar="RANGE", dest="chapter_range",
-        help="Same as '--range' except for chapters and other delegated URLs",
+        help=("Same as '--range' except for chapters "
+              "and other transferred URLs"),
     )
     selection.add_argument(
         "--filter",
@@ -213,7 +215,8 @@ def build_parser():
     selection.add_argument(
         "--chapter-filter",
         metavar="EXPR", dest="chapter_filter",
-        help="Same as '--filter' except for chapters and other delegated URLs",
+        help=("Same as '--filter' except for chapters "
+              "and other transferred URLs"),
     )
     selection.add_argument(
         "--images", dest="depr_images",

gallery_dl/util.py

@@ -237,7 +237,7 @@ class Formatter():
     Extra Format Specifiers:
     - "?<before>/<after>/":
         Adds <before> and <after> to the actual value if it evaluates to True.
-        Otherwise the whole replacement field beomes an empty string.
+        Otherwise the whole replacement field becomes an empty string.
         Example: {f:?-+/+-/} -> "-+Example+-" (if "f" contains "Example")
                              -> ""            (if "f" is None, 0, "")
     """

test/test_extractors.py

@@ -69,7 +69,6 @@ skip = [
     "imgtrex",
     "luscious",  # "high load"
     "pawoo",
-    "mangahere",  # unstable; sometimes sends empty responses
 ]
 # enable selective testing for direct calls
 if __name__ == '__main__' and len(sys.argv) > 1: