diff --git a/gallery_dl/__init__.py b/gallery_dl/__init__.py
index 82bf5520..9e45dca0 100644
--- a/gallery_dl/__init__.py
+++ b/gallery_dl/__init__.py
@@ -108,6 +108,9 @@ def main():
args = parser.parse_args()
log = output.initialize_logging(args.loglevel)
+ if args.config:
+ config._warn_legacy = False
+
# configuration
if args.load_config:
config.load()
@@ -215,6 +218,15 @@ def main():
"Deleted %d %s from '%s'",
cnt, "entry" if cnt == 1 else "entries", cache._path(),
)
+ elif args.config:
+ if args.config == "init":
+ return config.config_init()
+ if args.config == "open":
+ return config.config_open()
+ if args.config == "status":
+ return config.config_status()
+ if args.config == "update":
+ return config.config_update()
else:
if not args.urls and not args.inputfiles:
parser.error(
diff --git a/gallery_dl/config.py b/gallery_dl/config.py
index 10aa8301..7ba48778 100644
--- a/gallery_dl/config.py
+++ b/gallery_dl/config.py
@@ -8,9 +8,9 @@
"""Global configuration module"""
+import os
import sys
import json
-import os.path
import logging
from . import util
@@ -21,6 +21,7 @@ log = logging.getLogger("config")
# internals
_config = {"__global__": {}}
+_warn_legacy = True
if util.WINDOWS:
_default_configs = [
@@ -49,23 +50,84 @@ if getattr(sys, "frozen", False):
# --------------------------------------------------------------------
# public interface
+def get(section, key, default=None, *, conf=_config):
+ """Get the value of property 'key' or a default value"""
+ try:
+ return conf[section][key]
+ except Exception:
+ return default
+
+
+def set(section, key, value, *, conf=_config):
+ """Set the value of property 'key' for this session"""
+ try:
+ conf[section][key] = value
+ except KeyError:
+ conf[section] = {key: value}
+
+
+def setdefault(section, key, value, *, conf=_config):
+ """Set the value of property 'key' if it doesn't exist"""
+ try:
+ conf[section].setdefault(key, value)
+ except KeyError:
+ conf[section] = {key: value}
+
+
+def unset(section, key, *, conf=_config):
+ """Unset the value of property 'key'"""
+ try:
+ del conf[section][key]
+ except Exception:
+ pass
+
+
+def interpolate(sections, key, default=None, *, conf=_config):
+ if key in conf["__global__"]:
+ return conf["__global__"][key]
+ for section in sections:
+ if section in conf and key in conf[section]:
+ default = conf[section][key]
+ return default
+
+
+class apply():
+ """Context Manager: apply a collection of key-value pairs"""
+
+ def __init__(self, kvlist):
+ self.original = []
+ self.kvlist = kvlist
+
+ def __enter__(self):
+ for path, key, value in self.kvlist:
+ self.original.append((path, key, get(path, key, util.SENTINEL)))
+ set(path, key, value)
+
+ def __exit__(self, etype, value, traceback):
+ for path, key, value in self.original:
+ if value is util.SENTINEL:
+ unset(path, key)
+ else:
+ set(path, key, value)
+
+
def load(files=None, strict=False, fmt="json"):
"""Load JSON configuration files"""
if fmt == "yaml":
try:
import yaml
- parsefunc = yaml.safe_load
+ load = yaml.safe_load
except ImportError:
log.error("Could not import 'yaml' module")
return
else:
- parsefunc = json.load
+ load = json.load
for path in files or _default_configs:
path = util.expand_path(path)
try:
with open(path, encoding="utf-8") as fp:
- confdict = parsefunc(fp)
+ config_dict = load(fp)
except OSError as exc:
if strict:
log.error(exc)
@@ -75,10 +137,17 @@ def load(files=None, strict=False, fmt="json"):
if strict:
sys.exit(2)
else:
+ if "extractor" in config_dict:
+ if _warn_legacy:
+ log.warning("Legacy config file found at %s", path)
+ log.warning("Run 'gallery-dl --config-update' or follow "
+ " to update to the new format")
+ config_dict = update_config_dict(config_dict)
+
if not _config:
- _config.update(confdict)
+ _config.update(config_dict)
else:
- util.combine_dict(_config, confdict)
+ util.combine_dict(_config, config_dict)
def clear():
@@ -153,62 +222,166 @@ def build_module_options_dict(extr, package, module, conf=_config):
return build_options_dict(keys)
-def get(section, key, default=None, *, conf=_config):
- """Get the value of property 'key' or a default value"""
- try:
- return conf[section][key]
- except Exception:
- return default
+def config_init():
+ paths = [
+ util.expand_path(path)
+ for path in _default_configs
+ ]
+ for path in paths:
+ if os.access(path, os.R_OK):
+ log.error("There is already a configuration file at %s", path)
+ return 1
-def set(section, key, value, *, conf=_config):
- """Set the value of property 'key' for this session"""
- try:
- conf[section][key] = value
- except KeyError:
- conf[section] = {key: value}
+ for path in paths:
+ try:
+ with open(path, "w", encoding="utf-8") as fp:
+ fp.write("""\
+{
+ "general": {
+ },
+ "downloader": {
-def setdefault(section, key, value, *, conf=_config):
- """Set the value of property 'key' if it doesn't exist"""
- try:
- conf[section].setdefault(key, value)
- except KeyError:
- conf[section] = {key: value}
+ },
+ "output": {
+ }
+}
+""")
+ break
+ except OSError as exc:
+ log.debug(exc)
+ else:
+ log.error("Unable to create a new configuration file "
+ "at any of the default paths")
+ return 1
-def unset(section, key, *, conf=_config):
- """Unset the value of property 'key'"""
- try:
- del conf[section][key]
- except Exception:
- pass
+ log.info("Created a basic configuration file at %s", path)
-def interpolate(sections, key, default=None, *, conf=_config):
- if key in conf["__global__"]:
- return conf["__global__"][key]
- for section in sections:
- if section in conf and key in conf[section]:
- default = conf[section][key]
- return default
+def config_open():
+ for path in _default_configs:
+ path = util.expand_path(path)
+ if os.access(path, os.R_OK | os.W_OK):
+ import subprocess
+ import shutil
+
+ openers = (("explorer", "notepad")
+ if util.WINDOWS else
+ ("xdg-open", "open", os.environ.get("EDITOR", "")))
+ for opener in openers:
+ if opener := shutil.which(opener):
+ break
+ else:
+ log.warning("Unable to find a program to open '%s' with", path)
+ return 1
+ log.info("Running '%s %s'", opener, path)
+ return subprocess.Popen((opener, path)).wait()
-class apply():
- """Context Manager: apply a collection of key-value pairs"""
+ log.warning("Unable to find any writable configuration file")
+ return 1
- def __init__(self, kvlist):
- self.original = []
- self.kvlist = kvlist
- def __enter__(self):
- for path, key, value in self.kvlist:
- self.original.append((path, key, get(path, key, util.SENTINEL)))
- set(path, key, value)
+def config_status():
+ for path in _default_configs:
+ path = util.expand_path(path)
+ try:
+ with open(path, encoding="utf-8") as fp:
+ config_dict = json.load(fp)
+ except FileNotFoundError:
+ status = "NOT PRESENT"
+ except ValueError:
+ status = "INVALID JSON"
+ except Exception as exc:
+ log.debug(exc)
+ status = "UNKNOWN"
+ else:
+ status = "OK"
+ if "extractor" in config_dict:
+ status += " (legacy)"
+ print(f"{path}: {status}")
- def __exit__(self, etype, value, traceback):
- for path, key, value in self.original:
- if value is util.SENTINEL:
- unset(path, key)
+
+def config_update():
+ for path in _default_configs:
+ path = util.expand_path(path)
+ try:
+ with open(path, encoding="utf-8") as fp:
+ config_content = fp.read()
+ config_dict = json.loads(config_content)
+ except Exception as exc:
+ log.debug(exc)
+ else:
+ if "extractor" in config_dict:
+ config_dict = update_config_dict(config_dict)
+
+ # write backup
+ with open(path + ".bak", "w", encoding="utf-8") as fp:
+ fp.write(config_content)
+
+ # overwrite with updated JSON
+ with open(path, "w", encoding="utf-8") as fp:
+ json.dump(
+ config_dict, fp,
+ indent=4,
+ ensure_ascii=get("output", "ascii"),
+ )
+
+ log.info("Updated %s", path)
+ log.info("Backup at %s", path + ".bak")
+
+
+def update_config_dict(config_legacy):
+ # option names that could be a dict
+ optnames = {"filename", "directory", "path-restrict", "cookies",
+ "extension-map", "keywords", "keywords-default", "proxy"}
+
+ config = {"general": {}}
+
+ if extractor := config_legacy.pop("extractor", None):
+ for key, value in extractor.items():
+ if isinstance(value, dict) and key not in optnames:
+ config[key] = value
+
+ delete = []
+ instances = None
+
+ for skey, sval in value.items():
+ if isinstance(sval, dict):
+
+ # basecategory instance
+ if "root" in sval:
+ if instances is None:
+ config[key + ":instances"] = instances = {}
+ instances[skey] = sval
+ delete.append(skey)
+
+ # subcategory options
+ elif skey not in optnames:
+ config[f"{key}:{skey}"] = value[skey]
+ delete.append(skey)
+
+ for skey in delete:
+ del value[skey]
+
+ if not value:
+ del config[key]
else:
- set(path, key, value)
+ config["general"][key] = value
+
+ if downloader := config_legacy.pop("downloader", None):
+ config["downloader"] = downloader
+ for module in ("http", "ytdl", "text"):
+ if opts := downloader.pop(module, None):
+ config["downloader:" + module] = opts
+
+ for section_name in ("output", "postprocessor", "cache"):
+ if section := config_legacy.pop(section_name, None):
+ config[section_name] = section
+
+ for key, value in config_legacy.items():
+ config["general"][key] = value
+
+ return config
diff --git a/gallery_dl/option.py b/gallery_dl/option.py
index 56a2035e..d02108bf 100644
--- a/gallery_dl/option.py
+++ b/gallery_dl/option.py
@@ -290,20 +290,45 @@ def build_parser():
dest="cfgfiles", metavar="FILE", action="append",
help="Additional configuration files",
)
+ configuration.add_argument(
+ "-o", "--option",
+ dest="options", metavar="OPT", action=ParseAction, default=[],
+ help="Additional '=' option values",
+ )
configuration.add_argument(
"--config-yaml",
dest="yamlfiles", metavar="FILE", action="append",
help=argparse.SUPPRESS,
)
configuration.add_argument(
- "-o", "--option",
- dest="options", metavar="OPT", action=ParseAction, default=[],
- help="Additional '=' option values",
+ "--config-init",
+ dest="config", action="store_const", const="init",
+ help="Create a basic, initial configuration file",
+ )
+ configuration.add_argument(
+ "--config-open",
+ dest="config", action="store_const", const="open",
+ help="Open a configuration file in the user's preferred application",
+ )
+ configuration.add_argument(
+ "--config-status",
+ dest="config", action="store_const", const="status",
+ help="Show configuration file status",
+ )
+ configuration.add_argument(
+ "--config-update",
+ dest="config", action="store_const", const="update",
+ help="Convert legacy configuration files",
+ )
+ configuration.add_argument(
+ "--config-ignore",
+ dest="load_config", action="store_false",
+ help="Do not load any default configuration files",
)
configuration.add_argument(
"--ignore-config",
dest="load_config", action="store_false",
- help="Do not read the default configuration files",
+ help=argparse.SUPPRESS,
)
authentication = parser.add_argument_group("Authentication Options")