mirror_maker
/
gallery-dl
mirror of https://github.com/mikf/gallery-dl.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

implement several '--config-*' command-line options

Browse Source 
- --config-init, --config-open, --config-status, --config-update
- add --config-ignore as alternative to --ignore-config
2.0
Mike Fährmann 3 years ago
parent c83847e16a
commit 119a25a59c
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88
3 changed files with 265 additions and 55 deletions
Show Stats Download Patch File Download Diff File

12
gallery_dl/__init__.py
Unescape View File

@ -108,6 +108,9 @@ def main():
args = parser.parse_args()
log = output.initialize_logging(args.loglevel)
if args.config:
config._warn_legacy = False
# configuration
if args.load_config:
config.load()
@ -215,6 +218,15 @@ def main():
"Deleted %d %s from '%s'",
cnt, "entry" if cnt == 1 else "entries", cache._path(),
)
elif args.config:
if args.config == "init":
return config.config_init()
if args.config == "open":
return config.config_open()
if args.config == "status":
return config.config_status()
if args.config == "update":
return config.config_update()
else:
if not args.urls and not args.inputfiles:
parser.error(

275
gallery_dl/config.py
Unescape View File

@ -8,9 +8,9 @@
"""Global configuration module"""
import os
import sys
import json
import os.path
import logging
from . import util
@ -21,6 +21,7 @@ log = logging.getLogger("config")
# internals
_config = {"__global__": {}}
_warn_legacy = True
if util.WINDOWS:
_default_configs = [
@ -49,23 +50,84 @@ if getattr(sys, "frozen", False):
# --------------------------------------------------------------------
# public interface
def get(section, key, default=None, *, conf=_config):
"""Get the value of property 'key' or a default value"""
try:
return conf[section][key]
except Exception:
return default
def set(section, key, value, *, conf=_config):
"""Set the value of property 'key' for this session"""
try:
conf[section][key] = value
except KeyError:
conf[section] = {key: value}
def setdefault(section, key, value, *, conf=_config):
"""Set the value of property 'key' if it doesn't exist"""
try:
conf[section].setdefault(key, value)
except KeyError:
conf[section] = {key: value}
def unset(section, key, *, conf=_config):
"""Unset the value of property 'key'"""
try:
del conf[section][key]
except Exception:
pass
def interpolate(sections, key, default=None, *, conf=_config):
if key in conf["__global__"]:
return conf["__global__"][key]
for section in sections:
if section in conf and key in conf[section]:
default = conf[section][key]
return default
class apply():
"""Context Manager: apply a collection of key-value pairs"""
def __init__(self, kvlist):
self.original = []
self.kvlist = kvlist
def __enter__(self):
for path, key, value in self.kvlist:
self.original.append((path, key, get(path, key, util.SENTINEL)))
set(path, key, value)
def __exit__(self, etype, value, traceback):
for path, key, value in self.original:
if value is util.SENTINEL:
unset(path, key)
else:
set(path, key, value)
def load(files=None, strict=False, fmt="json"):
"""Load JSON configuration files"""
if fmt == "yaml":
try:
import yaml
parsefunc = yaml.safe_load
load = yaml.safe_load
except ImportError:
log.error("Could not import 'yaml' module")
return
else:
parsefunc = json.load
load = json.load
for path in files or _default_configs:
path = util.expand_path(path)
try:
with open(path, encoding="utf-8") as fp:
confdict = parsefunc(fp)
config_dict = load(fp)
except OSError as exc:
if strict:
log.error(exc)
@ -75,10 +137,17 @@ def load(files=None, strict=False, fmt="json"):
if strict:
sys.exit(2)
else:
if "extractor" in config_dict:
if _warn_legacy:
log.warning("Legacy config file found at %s", path)
log.warning("Run 'gallery-dl --config-update' or follow "
"<link> to update to the new format")
config_dict = update_config_dict(config_dict)
if not _config:
_config.update(confdict)
_config.update(config_dict)
else:
util.combine_dict(_config, confdict)
util.combine_dict(_config, config_dict)
def clear():
@ -153,62 +222,166 @@ def build_module_options_dict(extr, package, module, conf=_config):
return build_options_dict(keys)
def get(section, key, default=None, *, conf=_config):
"""Get the value of property 'key' or a default value"""
try:
return conf[section][key]
except Exception:
return default
def config_init():
paths = [
util.expand_path(path)
for path in _default_configs
]
for path in paths:
if os.access(path, os.R_OK):
log.error("There is already a configuration file at %s", path)
return 1
def set(section, key, value, *, conf=_config):
"""Set the value of property 'key' for this session"""
try:
conf[section][key] = value
except KeyError:
conf[section] = {key: value}
for path in paths:
try:
with open(path, "w", encoding="utf-8") as fp:
fp.write("""\
{
"general": {
},
"downloader": {
def setdefault(section, key, value, *, conf=_config):
"""Set the value of property 'key' if it doesn't exist"""
try:
conf[section].setdefault(key, value)
except KeyError:
conf[section] = {key: value}
},
"output": {
}
}
""")
break
except OSError as exc:
log.debug(exc)
else:
log.error("Unable to create a new configuration file "
"at any of the default paths")
return 1
def unset(section, key, *, conf=_config):
"""Unset the value of property 'key'"""
try:
del conf[section][key]
except Exception:
pass
log.info("Created a basic configuration file at %s", path)
def interpolate(sections, key, default=None, *, conf=_config):
if key in conf["__global__"]:
return conf["__global__"][key]
for section in sections:
if section in conf and key in conf[section]:
default = conf[section][key]
return default
def config_open():
for path in _default_configs:
path = util.expand_path(path)
if os.access(path, os.R_OK | os.W_OK):
import subprocess
import shutil
openers = (("explorer", "notepad")
if util.WINDOWS else
("xdg-open", "open", os.environ.get("EDITOR", "")))
for opener in openers:
if opener := shutil.which(opener):
break
else:
log.warning("Unable to find a program to open '%s' with", path)
return 1
log.info("Running '%s %s'", opener, path)
return subprocess.Popen((opener, path)).wait()
class apply():
"""Context Manager: apply a collection of key-value pairs"""
log.warning("Unable to find any writable configuration file")
return 1
def __init__(self, kvlist):
self.original = []
self.kvlist = kvlist
def __enter__(self):
for path, key, value in self.kvlist:
self.original.append((path, key, get(path, key, util.SENTINEL)))
set(path, key, value)
def config_status():
for path in _default_configs:
path = util.expand_path(path)
try:
with open(path, encoding="utf-8") as fp:
config_dict = json.load(fp)
except FileNotFoundError:
status = "NOT PRESENT"
except ValueError:
status = "INVALID JSON"
except Exception as exc:
log.debug(exc)
status = "UNKNOWN"
else:
status = "OK"
if "extractor" in config_dict:
status += " (legacy)"
print(f"{path}: {status}")
def __exit__(self, etype, value, traceback):
for path, key, value in self.original:
if value is util.SENTINEL:
unset(path, key)
def config_update():
for path in _default_configs:
path = util.expand_path(path)
try:
with open(path, encoding="utf-8") as fp:
config_content = fp.read()
config_dict = json.loads(config_content)
except Exception as exc:
log.debug(exc)
else:
if "extractor" in config_dict:
config_dict = update_config_dict(config_dict)
# write backup
with open(path + ".bak", "w", encoding="utf-8") as fp:
fp.write(config_content)
# overwrite with updated JSON
with open(path, "w", encoding="utf-8") as fp:
json.dump(
config_dict, fp,
indent=4,
ensure_ascii=get("output", "ascii"),
)
log.info("Updated %s", path)
log.info("Backup at %s", path + ".bak")
def update_config_dict(config_legacy):
# option names that could be a dict
optnames = {"filename", "directory", "path-restrict", "cookies",
"extension-map", "keywords", "keywords-default", "proxy"}
config = {"general": {}}
if extractor := config_legacy.pop("extractor", None):
for key, value in extractor.items():
if isinstance(value, dict) and key not in optnames:
config[key] = value
delete = []
instances = None
for skey, sval in value.items():
if isinstance(sval, dict):
# basecategory instance
if "root" in sval:
if instances is None:
config[key + ":instances"] = instances = {}
instances[skey] = sval
delete.append(skey)
# subcategory options
elif skey not in optnames:
config[f"{key}:{skey}"] = value[skey]
delete.append(skey)
for skey in delete:
del value[skey]
if not value:
del config[key]
else:
set(path, key, value)
config["general"][key] = value
if downloader := config_legacy.pop("downloader", None):
config["downloader"] = downloader
for module in ("http", "ytdl", "text"):
if opts := downloader.pop(module, None):
config["downloader:" + module] = opts
for section_name in ("output", "postprocessor", "cache"):
if section := config_legacy.pop(section_name, None):
config[section_name] = section
for key, value in config_legacy.items():
config["general"][key] = value
return config

33
gallery_dl/option.py
Unescape View File

@ -290,20 +290,45 @@ def build_parser():
dest="cfgfiles", metavar="FILE", action="append",
help="Additional configuration files",
)
configuration.add_argument(
"-o", "--option",
dest="options", metavar="OPT", action=ParseAction, default=[],
help="Additional '<key>=<value>' option values",
)
configuration.add_argument(
"--config-yaml",
dest="yamlfiles", metavar="FILE", action="append",
help=argparse.SUPPRESS,
)
configuration.add_argument(
"-o", "--option",
dest="options", metavar="OPT", action=ParseAction, default=[],
help="Additional '<key>=<value>' option values",
"--config-init",
dest="config", action="store_const", const="init",
help="Create a basic, initial configuration file",
)
configuration.add_argument(
"--config-open",
dest="config", action="store_const", const="open",
help="Open a configuration file in the user's preferred application",
)
configuration.add_argument(
"--config-status",
dest="config", action="store_const", const="status",
help="Show configuration file status",
)
configuration.add_argument(
"--config-update",
dest="config", action="store_const", const="update",
help="Convert legacy configuration files",
)
configuration.add_argument(
"--config-ignore",
dest="load_config", action="store_false",
help="Do not load any default configuration files",
)
configuration.add_argument(
"--ignore-config",
dest="load_config", action="store_false",
help="Do not read the default configuration files",
help=argparse.SUPPRESS,
)
authentication = parser.add_argument_group("Authentication Options")

Write Preview
Loading…
Cancel
Save