diff --git a/docs/configuration.rst b/docs/configuration.rst index d7709883..06089ad8 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -1432,7 +1432,14 @@ Description Extended logging output configuration. * format * Format string for logging messages - (see `LogRecord attributes `__) + + In addition to the default + `LogRecord attributes `__, + it is also possible to access the current + `extractor `__ + and `job `__ + objects as well as their attributes + (e.g. ``"{extractor.url}"``) * Default: ``"[{name}][{levelname}] {message}"`` * format-date * Format string for ``{asctime}`` fields in logging messages diff --git a/gallery_dl/__init__.py b/gallery_dl/__init__.py index a57ca2b5..bcdd660f 100644 --- a/gallery_dl/__init__.py +++ b/gallery_dl/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2014-2018 Mike Fährmann +# Copyright 2014-2019 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -21,88 +21,9 @@ if sys.hexversion < 0x3040000: import json import logging -from . import version, config, option, extractor, job, util, exception +from . import version, config, option, output, extractor, job, util, exception __version__ = version.__version__ -log = logging.getLogger("gallery-dl") - -LOG_FORMAT = "[{name}][{levelname}] {message}" -LOG_FORMAT_DATE = "%Y-%m-%d %H:%M:%S" -LOG_LEVEL = logging.INFO - -def initialize_logging(loglevel): - """Setup basic logging functionality before configfiles have been loaded""" - # convert levelnames to lowercase - for level in (10, 20, 30, 40, 50): - name = logging.getLevelName(level) - logging.addLevelName(level, name.lower()) - # setup basic logging to stderr - formatter = logging.Formatter(LOG_FORMAT, LOG_FORMAT_DATE, "{") - handler = logging.StreamHandler() - handler.setFormatter(formatter) - handler.setLevel(loglevel) - root = logging.getLogger() - root.setLevel(logging.NOTSET) - root.addHandler(handler) - - -def setup_logging_handler(key, fmt=LOG_FORMAT, lvl=LOG_LEVEL): - """Setup a new logging handler""" - opts = config.interpolate(("output", key)) - if not opts: - return None - if not isinstance(opts, dict): - opts = {"path": opts} - - path = opts.get("path") - mode = opts.get("mode", "w") - encoding = opts.get("encoding", "utf-8") - try: - path = util.expand_path(path) - handler = logging.FileHandler(path, mode, encoding) - except (OSError, ValueError) as exc: - log.warning("%s: %s", key, exc) - return None - except TypeError as exc: - log.warning("%s: missing or invalid path (%s)", key, exc) - return None - - level = opts.get("level", lvl) - logfmt = opts.get("format", fmt) - datefmt = opts.get("format-date", LOG_FORMAT_DATE) - formatter = logging.Formatter(logfmt, datefmt, "{") - handler.setFormatter(formatter) - handler.setLevel(level) - - return handler - - -def configure_logging_handler(key, handler): - """Configure a logging handler""" - opts = config.interpolate(("output", key)) - if not opts: - return - if isinstance(opts, str): - opts = {"format": opts} - if handler.level == LOG_LEVEL and "level" in opts: - handler.setLevel(opts["level"]) - if "format" in opts or "format-date" in opts: - logfmt = opts.get("format", LOG_FORMAT) - datefmt = opts.get("format-date", LOG_FORMAT_DATE) - formatter = logging.Formatter(logfmt, datefmt, "{") - handler.setFormatter(formatter) - - -def replace_std_streams(errors="replace"): - """Replace standard streams and set their error handlers to 'errors'""" - for name in ("stdout", "stdin", "stderr"): - stream = getattr(sys, name) - setattr(sys, name, stream.__class__( - stream.buffer, - errors=errors, - newline=stream.newlines, - line_buffering=stream.line_buffering, - )) def progress(urls, pformat): @@ -115,7 +36,7 @@ def progress(urls, pformat): yield pinfo["url"] -def parse_inputfile(file): +def parse_inputfile(file, log): """Filter and process strings from an input file. Lines starting with '#' and empty lines will be ignored. @@ -187,12 +108,11 @@ def parse_inputfile(file): def main(): try: if sys.stdout.encoding.lower() != "utf-8": - replace_std_streams() + output.replace_std_streams() parser = option.build_parser() args = parser.parse_args() - - initialize_logging(args.loglevel) + log = output.initialize_logging(args.loglevel) # configuration if args.load_config: @@ -205,10 +125,12 @@ def main(): config.set(key, value) # stream logging handler - configure_logging_handler("log", logging.getLogger().handlers[0]) + output.configure_logging_handler( + "log", logging.getLogger().handlers[0]) # file logging handler - handler = setup_logging_handler("logfile", lvl=args.loglevel) + handler = output.setup_logging_handler( + "logfile", lvl=args.loglevel) if handler: logging.getLogger().addHandler(handler) @@ -284,13 +206,14 @@ def main(): file = sys.stdin else: file = open(args.inputfile, encoding="utf-8") - urls += parse_inputfile(file) + urls += parse_inputfile(file, log) file.close() except OSError as exc: log.warning("input file: %s", exc) # unsupported file logging handler - handler = setup_logging_handler("unsupportedfile", fmt="{message}") + handler = output.setup_logging_handler( + "unsupportedfile", fmt="{message}") if handler: ulog = logging.getLogger("unsupported") ulog.addHandler(handler) diff --git a/gallery_dl/job.py b/gallery_dl/job.py index 0a98ce46..035362aa 100644 --- a/gallery_dl/job.py +++ b/gallery_dl/job.py @@ -25,9 +25,11 @@ class Job(): extr = extractor.find(extr) if not extr: raise exception.NoExtractorError() + self.extractor = extr - self.extractor.log.debug( - "Using %s for '%s'", extr.__class__.__name__, extr.url) + extr.log.extractor = extr + extr.log.job = self + extr.log.debug("Using %s for '%s'", extr.__class__.__name__, extr.url) # url predicates self.pred_url = self._prepare_predicates( diff --git a/gallery_dl/output.py b/gallery_dl/output.py index 7c547461..327b69a0 100644 --- a/gallery_dl/output.py +++ b/gallery_dl/output.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2015-2018 Mike Fährmann +# Copyright 2015-2019 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -9,8 +9,120 @@ import os import sys import shutil -from . import config - +import logging +from . import config, util + + +# -------------------------------------------------------------------- +# Logging + +LOG_FORMAT = "[{name}][{levelname}] {message}" +LOG_FORMAT_DATE = "%Y-%m-%d %H:%M:%S" +LOG_LEVEL = logging.INFO + + +class Logger(logging.Logger): + """Custom logger that includes extractor and job info in log records""" + extractor = util.NONE + job = util.NONE + + def makeRecord(self, name, level, fn, lno, msg, args, exc_info, + func=None, extra=None, sinfo=None, + factory=logging._logRecordFactory): + rv = factory(name, level, fn, lno, msg, args, exc_info, func, sinfo) + rv.extractor = self.extractor + rv.job = self.job + return rv + + +def initialize_logging(loglevel): + """Setup basic logging functionality before configfiles have been loaded""" + # convert levelnames to lowercase + for level in (10, 20, 30, 40, 50): + name = logging.getLevelName(level) + logging.addLevelName(level, name.lower()) + + # register custom Logging class + logging.Logger.manager.setLoggerClass(Logger) + + # setup basic logging to stderr + formatter = logging.Formatter(LOG_FORMAT, LOG_FORMAT_DATE, "{") + handler = logging.StreamHandler() + handler.setFormatter(formatter) + handler.setLevel(loglevel) + root = logging.getLogger() + root.setLevel(logging.NOTSET) + root.addHandler(handler) + + return logging.getLogger("gallery-dl") + + +def setup_logging_handler(key, fmt=LOG_FORMAT, lvl=LOG_LEVEL): + """Setup a new logging handler""" + opts = config.interpolate(("output", key)) + if not opts: + return None + if not isinstance(opts, dict): + opts = {"path": opts} + + path = opts.get("path") + mode = opts.get("mode", "w") + encoding = opts.get("encoding", "utf-8") + try: + path = util.expand_path(path) + handler = logging.FileHandler(path, mode, encoding) + except (OSError, ValueError) as exc: + logging.getLogger("gallery-dl").warning( + "%s: %s", key, exc) + return None + except TypeError as exc: + logging.getLogger("gallery-dl").warning( + "%s: missing or invalid path (%s)", key, exc) + return None + + level = opts.get("level", lvl) + logfmt = opts.get("format", fmt) + datefmt = opts.get("format-date", LOG_FORMAT_DATE) + formatter = logging.Formatter(logfmt, datefmt, "{") + handler.setFormatter(formatter) + handler.setLevel(level) + + return handler + + +def configure_logging_handler(key, handler): + """Configure a logging handler""" + opts = config.interpolate(("output", key)) + if not opts: + return + if isinstance(opts, str): + opts = {"format": opts} + if handler.level == LOG_LEVEL and "level" in opts: + handler.setLevel(opts["level"]) + if "format" in opts or "format-date" in opts: + logfmt = opts.get("format", LOG_FORMAT) + datefmt = opts.get("format-date", LOG_FORMAT_DATE) + formatter = logging.Formatter(logfmt, datefmt, "{") + handler.setFormatter(formatter) + + +# -------------------------------------------------------------------- +# Utility functions + +def replace_std_streams(errors="replace"): + """Replace standard streams and set their error handlers to 'errors'""" + for name in ("stdout", "stdin", "stderr"): + stream = getattr(sys, name) + setattr(sys, name, stream.__class__( + stream.buffer, + errors=errors, + newline=stream.newlines, + line_buffering=stream.line_buffering, + )) + + +# -------------------------------------------------------------------- +# Downloader output def select(): """Automatically select a suitable output class""" diff --git a/gallery_dl/util.py b/gallery_dl/util.py index 0a263c01..ccd5d55f 100644 --- a/gallery_dl/util.py +++ b/gallery_dl/util.py @@ -140,6 +140,28 @@ CODES = { SPECIAL_EXTRACTORS = {"oauth", "recursive", "test"} +class UniversalNone(): + """None-style object that also supports __getattr__ and __getitem__""" + def __getattr__(self, _): + return self + + def __getitem__(self, _): + return self + + @staticmethod + def __bool__(): + return False + + @staticmethod + def __str__(): + return "None" + + __repr__ = __str__ + + +NONE = UniversalNone() + + def build_predicate(predicates): if not predicates: return lambda url, kwds: True