smaller code and text changes

pull/54/head
Mike Fährmann 7 years ago
parent ad4580800c
commit b319f4bab3
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88

@ -66,9 +66,8 @@ def load(*files, format="json", strict=False):
if strict: if strict:
log.error("Configuration file '%s' not found", path) log.error("Configuration file '%s' not found", path)
sys.exit(1) sys.exit(1)
except Exception as exception: except Exception as exc:
log.warning("Could not parse '%s'", path) log.warning("Could not parse '%s': %s", path, exc)
log.warning(exception)
if strict: if strict:
sys.exit(2) sys.exit(2)

@ -117,8 +117,8 @@ class AsynchronousExtractor(Extractor):
def __init__(self): def __init__(self):
Extractor.__init__(self) Extractor.__init__(self)
queue_size = int(config.get(("queue-size",), default=5)) queue_size = int(config.get(("queue-size",), 5))
self.__queue = queue.Queue(maxsize=queue_size) self.__queue = queue.Queue(queue_size)
self.__thread = threading.Thread(target=self.async_items, daemon=True) self.__thread = threading.Thread(target=self.async_items, daemon=True)
def __iter__(self): def __iter__(self):
@ -140,8 +140,8 @@ class AsynchronousExtractor(Extractor):
try: try:
for task in self.items(): for task in self.items():
put(task) put(task)
except Exception as e: except Exception as exc:
put(e) put(exc)
put(None) put(None)

@ -268,9 +268,7 @@ class PixivRankingExtractor(PixivExtractor):
r"ranking\.php(?:\?([^#]*))?"] r"ranking\.php(?:\?([^#]*))?"]
test = [ test = [
(("https://www.pixiv.net/ranking.php" (("https://www.pixiv.net/ranking.php"
"?mode=daily&content=illust&date=20170818"), { "?mode=daily&content=illust&date=20170818"), None),
"pattern": r"^https?://i\d*\.pixiv\.net/img-original/img/.+/\d+_p",
}),
("https://www.pixiv.net/ranking.php", None), ("https://www.pixiv.net/ranking.php", None),
] ]

@ -17,7 +17,7 @@ class Job():
"""Base class for Job-types""" """Base class for Job-types"""
ufile = None ufile = None
def __init__(self, url): def __init__(self, url, parent=None):
self.url = url self.url = url
self.extractor = extractor.find(url) self.extractor = extractor.find(url)
if self.extractor is None: if self.extractor is None:
@ -46,6 +46,11 @@ class Job():
predicates.append(util.RangePredicate(chapter["range"])) predicates.append(util.RangePredicate(chapter["range"]))
self.pred_queue = util.build_predicate(predicates) self.pred_queue = util.build_predicate(predicates)
# category transfer
if parent and parent.extractor.categorytransfer:
self.extractor.category = parent.extractor.category
self.extractor.subcategory = parent.extractor.subcategory
def run(self): def run(self):
"""Execute or run the job""" """Execute or run the job"""
try: try:
@ -121,12 +126,6 @@ class Job():
kwdict["category"] = self.extractor.category kwdict["category"] = self.extractor.category
kwdict["subcategory"] = self.extractor.subcategory kwdict["subcategory"] = self.extractor.subcategory
def _prepare(self, job):
if self.extractor.categorytransfer:
job.extractor.category = self.extractor.category
job.extractor.subcategory = self.extractor.subcategory
return job
def _write_unsupported(self, url): def _write_unsupported(self, url):
if self.ufile: if self.ufile:
print(url, file=self.ufile, flush=True) print(url, file=self.ufile, flush=True)
@ -135,7 +134,7 @@ class Job():
class DownloadJob(Job): class DownloadJob(Job):
"""Download images into appropriate directory/filename locations""" """Download images into appropriate directory/filename locations"""
def __init__(self, url): def __init__(self, url, parent=None):
Job.__init__(self, url) Job.__init__(self, url)
self.pathfmt = util.PathFormat(self.extractor) self.pathfmt = util.PathFormat(self.extractor)
self.downloaders = {} self.downloaders = {}
@ -156,7 +155,7 @@ class DownloadJob(Job):
def handle_queue(self, url, keywords): def handle_queue(self, url, keywords):
try: try:
self._prepare(DownloadJob(url)).run() DownloadJob(url, self).run()
except exception.NoExtractorError: except exception.NoExtractorError:
self._write_unsupported(url) self._write_unsupported(url)
@ -178,8 +177,8 @@ class KeywordJob(Job):
"""Print available keywords""" """Print available keywords"""
def handle_url(self, url, keywords): def handle_url(self, url, keywords):
print("\nKeywords for filenames:") print("\nKeywords for filenames and --filter:")
print("-----------------------") print("------------------------------------")
self.print_keywords(keywords) self.print_keywords(keywords)
raise exception.StopExtraction() raise exception.StopExtraction()
@ -200,7 +199,7 @@ class KeywordJob(Job):
self.print_keywords(keywords) self.print_keywords(keywords)
if self.extractor.categorytransfer: if self.extractor.categorytransfer:
print() print()
self._prepare(KeywordJob(url)).run() KeywordJob(url, self).run()
raise exception.StopExtraction() raise exception.StopExtraction()
@staticmethod @staticmethod
@ -230,8 +229,8 @@ class UrlJob(Job):
"""Print download urls""" """Print download urls"""
maxdepth = -1 maxdepth = -1
def __init__(self, url, depth=1): def __init__(self, url, parent=None, depth=1):
Job.__init__(self, url) Job.__init__(self, url, parent)
self.depth = depth self.depth = depth
if depth == self.maxdepth: if depth == self.maxdepth:
self.handle_queue = self.handle_url self.handle_queue = self.handle_url
@ -242,7 +241,7 @@ class UrlJob(Job):
def handle_queue(self, url, _): def handle_queue(self, url, _):
try: try:
self._prepare(UrlJob(url, self.depth + 1)).run() UrlJob(url, self, self.depth + 1).run()
except exception.NoExtractorError: except exception.NoExtractorError:
self._write_unsupported(url) self._write_unsupported(url)

@ -71,7 +71,7 @@ def build_parser():
general.add_argument( general.add_argument(
"-h", "--help", "-h", "--help",
action="help", action="help",
help="Print this help message and exit" help="Print this help message and exit",
) )
general.add_argument( general.add_argument(
"--version", "--version",
@ -121,7 +121,8 @@ def build_parser():
) )
output.add_argument( output.add_argument(
"-K", "--list-keywords", dest="list_keywords", action="store_true", "-K", "--list-keywords", dest="list_keywords", action="store_true",
help="Print a list of available keywords for the given URLs", help=("Print a list of available keywords and example values "
"for the given URLs"),
) )
output.add_argument( output.add_argument(
"--list-modules", dest="list_modules", action="store_true", "--list-modules", dest="list_modules", action="store_true",
@ -148,7 +149,7 @@ def build_parser():
"--abort-on-skip", "--abort-on-skip",
action=ConfigConstAction, nargs=0, dest="skip", const="abort", action=ConfigConstAction, nargs=0, dest="skip", const="abort",
help=("Abort extractor run if a file download would normally be " help=("Abort extractor run if a file download would normally be "
"skipped, i.e. if a file with the same filename already exists") "skipped, i.e. if a file with the same filename already exists"),
) )
configuration = parser.add_argument_group("Configuration Options") configuration = parser.add_argument_group("Configuration Options")
@ -193,13 +194,14 @@ def build_parser():
metavar="RANGE", dest="image_range", metavar="RANGE", dest="image_range",
help=("Specify which images to download through a comma seperated list" help=("Specify which images to download through a comma seperated list"
" of indices or index-ranges; " " of indices or index-ranges; "
"for example '--images -2,4,6-8,10-' will download images with " "for example '--range -2,4,6-8,10-' will download images with "
"index 1, 2, 4, 6, 7, 8 and 10 up to the last one"), "index 1, 2, 4, 6, 7, 8 and 10 up to the last one"),
) )
selection.add_argument( selection.add_argument(
"--chapter-range", "--chapter-range",
metavar="RANGE", dest="chapter_range", metavar="RANGE", dest="chapter_range",
help="Same as '--range' except for chapters and other delegated URLs", help=("Same as '--range' except for chapters "
"and other transferred URLs"),
) )
selection.add_argument( selection.add_argument(
"--filter", "--filter",
@ -213,7 +215,8 @@ def build_parser():
selection.add_argument( selection.add_argument(
"--chapter-filter", "--chapter-filter",
metavar="EXPR", dest="chapter_filter", metavar="EXPR", dest="chapter_filter",
help="Same as '--filter' except for chapters and other delegated URLs", help=("Same as '--filter' except for chapters "
"and other transferred URLs"),
) )
selection.add_argument( selection.add_argument(
"--images", dest="depr_images", "--images", dest="depr_images",

@ -237,7 +237,7 @@ class Formatter():
Extra Format Specifiers: Extra Format Specifiers:
- "?<before>/<after>/": - "?<before>/<after>/":
Adds <before> and <after> to the actual value if it evaluates to True. Adds <before> and <after> to the actual value if it evaluates to True.
Otherwise the whole replacement field beomes an empty string. Otherwise the whole replacement field becomes an empty string.
Example: {f:?-+/+-/} -> "-+Example+-" (if "f" contains "Example") Example: {f:?-+/+-/} -> "-+Example+-" (if "f" contains "Example")
-> "" (if "f" is None, 0, "") -> "" (if "f" is None, 0, "")
""" """

@ -69,7 +69,6 @@ skip = [
"imgtrex", "imgtrex",
"luscious", # "high load" "luscious", # "high load"
"pawoo", "pawoo",
"mangahere", # unstable; sometimes sends empty responses
] ]
# enable selective testing for direct calls # enable selective testing for direct calls
if __name__ == '__main__' and len(sys.argv) > 1: if __name__ == '__main__' and len(sys.argv) > 1:

Loading…
Cancel
Save