From 089923e3dd812d3e3f301b41900b4fd48c0a16d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Wed, 27 Mar 2019 22:36:46 +0100 Subject: [PATCH] parse configuration.rst to build gallery-dl.conf.5 (#150) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit … a man-page containing all of gallery-dl's configuration file options. This implementation relies on Python dicts preserving their insertion order. Python 3.4 and 3.5 need to use OrderedDict or they produce randomly ordered man-page sections. The man-page formatting is a bit rough around the edges, but it works for the most part. The only real "problem" are inline-links, but it's better if they are left in there. --- Makefile | 15 ++- scripts/man.py | 241 ++++++++++++++++++++++++++++++++++++++++++++----- setup.py | 1 + 3 files changed, 227 insertions(+), 30 deletions(-) diff --git a/Makefile b/Makefile index f8f8e2e1..51f1a402 100644 --- a/Makefile +++ b/Makefile @@ -8,20 +8,16 @@ PYTHON ?= /usr/bin/env python3 # set SYSCONFDIR to /etc if PREFIX=/usr or PREFIX=/usr/local SYSCONFDIR = $(shell if [ $(PREFIX) = /usr -o $(PREFIX) = /usr/local ]; then echo /etc; else echo $(PREFIX)/etc; fi) -all: gallery-dl.1 gallery-dl.bash_completion docs/supportedsites.rst +all: gallery-dl.1 gallery-dl.conf.5 gallery-dl.bash_completion docs/supportedsites.rst clean: - $(RM) gallery-dl.1 gallery-dl.bash_completion + $(RM) gallery-dl.1 gallery-dl.conf.5 gallery-dl.bash_completion $(RM) -r build/ -install: gallery-dl.1 gallery-dl.bash_completion +install: gallery-dl.1 gallery-dl.conf.5 gallery-dl.bash_completion $(PYTHON) setup.py install - install -d $(DESTDIR)$(MANDIR)/man1 - install -m 644 youtube-dl.1 $(DESTDIR)$(MANDIR)/man1 - install -d $(DESTDIR)$(SYSCONFDIR)/bash_completion.d - install -m 644 youtube-dl.bash_completion $(DESTDIR)$(SYSCONFDIR)/bash_completion.d/youtube-dl -release: docs/supportedsites.rst +release: gallery-dl.1 gallery-dl.conf.5 gallery-dl.bash_completion docs/supportedsites.rst scripts/release.sh test: @@ -35,5 +31,8 @@ docs/supportedsites.rst: gallery_dl/*/*.py scripts/supportedsites.py gallery-dl.1: gallery_dl/option.py scripts/man.py $(PYTHON) scripts/man.py +gallery-dl.conf.5: docs/configuration.rst scripts/man.py + $(PYTHON) scripts/man.py + gallery-dl.bash_completion: gallery_dl/option.py scripts/bash_completion.py $(PYTHON) scripts/bash_completion.py diff --git a/scripts/man.py b/scripts/man.py index 8e658a28..b191faab 100755 --- a/scripts/man.py +++ b/scripts/man.py @@ -7,6 +7,7 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. +import re import sys import os.path import datetime @@ -18,7 +19,12 @@ import gallery_dl.option # noqa import gallery_dl.version # noqa -TEMPLATE = r""".TH "gallery-dl" "1" "$(date)s" "$(version)s" "" +def build_gallery_dl_1(path=None): + + OPTS_FMT = """.TP\n.B "{}" {}\n{}""" + + TEMPLATE = r""" +.TH "GALLERY-DL" "1" "$(date)s" "$(version)s" "gallery-dl Manual" .\" disable hyphenation .nh @@ -56,7 +62,7 @@ Scan \f[I]URL\f[] for other URLs and invoke \f[B]gallery-dl\f[] on them. gallery-dl oauth:\f[I]SITE\-NAME\f[] Gain OAuth authentication tokens for .IR deviantart , -.IR flickr , +.IR flickr , .IR reddit , .IR smugmug ", and" .IR tumblr . @@ -84,26 +90,217 @@ and https://github.com/mikf/gallery-dl/graphs/contributors .BR gallery-dl.conf (5) """ -OPTS_FMT = r""".TP -.B "{}" {} -{} + options = [] + for action in gallery_dl.option.build_parser()._actions: + if action.help.startswith("=="): + continue + options.append(OPTS_FMT.format( + ", ".join(action.option_strings).replace("-", r"\-"), + r"\f[I]{}\f[]".format(action.metavar) if action.metavar else "", + action.help, + )) + + if not path: + path = os.path.join(ROOTDIR, "gallery-dl.1") + with open(path, "w", encoding="utf-8") as file: + file.write(TEMPLATE.lstrip() % { + "options": "\n".join(options), + "version": gallery_dl.version.__version__, + "date" : datetime.datetime.now(), + }) + + +def build_gallery_dl_conf_5(path=None): + + TEMPLATE = r""" +.TH "GALLERY-DL.CONF" "5" "$(date)s" "$(version)s" "gallery-dl Manual" +.\" disable hyphenation +.nh +.\" disable justification (adjust text to left margin only) +.ad l + +.SH NAME +gallery-dl.conf \- gallery-dl configuration file + +.SH DESCRIPTION +gallery-dl will search for configuration files in the following places +every time it is started, unless +.B --ignore-config +is specified: +.PP +.RS 4 +.nf +.I /etc/gallery-dl.conf +.I $HOME/.config/gallery-dl/config.json +.I $HOME/.gallery-dl.conf +.fi +.RE +.PP +It is also possible to specify additional configuration files with the +.B -c/--config +command-line option or to add further option values with +.B -o/--option +as = pairs, + +Configuration files are JSON-based and therefore don't allow any ordinary +comments, but, since unused keys are simply ignored, it is possible to utilize +those as makeshift comments by settings their values to arbitrary strings. + +.SH EXAMPLE +{ +.RS 4 +"base-directory": "/tmp/", +.br +"extractor": { +.RS 4 +"pixiv": { +.RS 4 +"directory": ["Pixiv", "Works", "{user[id]}"], +.br +"filename": "{id}{num}.{extension}", +.br +"username": "foo", +.br +"password": "bar" +.RE +}, +.br +"flickr": { +.RS 4 +"_comment": "OAuth keys for account 'foobar'", +.br +"access-token": "0123456789-0123456789abcdef", +.br +"access-token-secret": "fedcba9876543210" +.RE +} +.RE +}, +.br +"downloader": { +.RS 4 +"retries": 3, +.br +"timeout": 2.5 +.RE +} +.RE +} + +%(options)s + +.SH BUGS +https://github.com/mikf/gallery-dl/issues + +.SH AUTHORS +Mike Fährmann +.br +and https://github.com/mikf/gallery-dl/graphs/contributors + +.SH "SEE ALSO" +.BR gallery-dl (1) """ + sections = parse_docs_configuration() + content = [] + + for sec_name, section in sections.items(): + content.append(".SH " + sec_name.upper()) + + for opt_name, option in section.items(): + content.append(".SS " + opt_name) + + for field, text in option.items(): + if field in ("Type", "Default"): + content.append('.IP "{}:" {}'.format(field, len(field)+2)) + content.append(strip_rst(text)) + else: + content.append('.IP "{}:" 4'.format(field)) + content.append(strip_rst(text, field != "Example")) + + if not path: + path = os.path.join(ROOTDIR, "gallery-dl.conf.5") + with open(path, "w", encoding="utf-8") as file: + file.write(TEMPLATE.lstrip() % { + "options": "\n".join(content), + "version": gallery_dl.version.__version__, + "date" : datetime.datetime.now(), + }) + + +def parse_docs_configuration(): + + doc_path = os.path.join(ROOTDIR, "docs", "configuration.rst") + with open(doc_path, encoding="utf-8") as file: + doc_lines = file.readlines() + + sections = {} + sec_name = None + options = None + opt_name = None + opt_desc = None + name = None + last = last2 = None + for line in doc_lines: + + # start of new section + if re.match(r"^=+$", line): + if sec_name and options: + sections[sec_name] = options + sec_name = last.strip() + options = {} + + elif re.match(r"^=+ =+$", line): + # start of option table + if re.match(r"^-+$", last): + opt_name = last2.strip() + opt_desc = {} + # end of option table + elif opt_desc: + options[opt_name] = opt_desc + opt_name = None + name = None + + # inside option table + elif opt_name: + if line[0].isalpha(): + name, _, line = line.partition(" ") + opt_desc[name] = "" + line = line.strip() + if line.startswith(("* ", "- ")): + line = "\n" + line + elif line.startswith("| "): + line = line[2:] + "\n.br" + opt_desc[name] += line + "\n" + + last2 = last + last = line + sections[sec_name] = options + + return sections + + +def strip_rst(text, extended=True, *, ITALIC=r"\\f[I]\1\\f[]", REGULAR=r"\1"): + + text = text.replace("\\", "\\\\") + + # ``foo`` + repl = ITALIC if extended else REGULAR + text = re.sub(r"``([^`]+)``", repl, text) + # |foo|_ + text = re.sub(r"\|([^|]+)\|_*", ITALIC, text) + # `foo`_ + text = re.sub(r"`([^`]+)`_+", ITALIC, text) + # `foo` + text = re.sub(r"`([^`]+)`", REGULAR, text) + # foo_ + text = re.sub(r"([A-Za-z0-9-]+)_+(?=\s)", ITALIC, text) + # ------- + text = re.sub(r"---+", "", text) + + return text + -options = [] -for action in gallery_dl.option.build_parser()._actions: - if action.help.startswith("=="): - continue - options.append(OPTS_FMT.format( - ", ".join(action.option_strings).replace("-", r"\-"), - r"\f[I]{}\f[]".format(action.metavar) if action.metavar else "", - action.help, - )) - -PATH = os.path.join(ROOTDIR, "gallery-dl.1") -with open(PATH, "w", encoding="utf-8") as file: - file.write(TEMPLATE % { - "options": "\n".join(options), - "version": gallery_dl.version.__version__, - "date" : datetime.datetime.now(), - }) +if __name__ == "__main__": + build_gallery_dl_1() + build_gallery_dl_conf_5() diff --git a/setup.py b/setup.py index df55f0ec..54042752 100644 --- a/setup.py +++ b/setup.py @@ -83,6 +83,7 @@ data_files = [ for (path, files) in [ ('etc/bash_completion.d', ['gallery-dl.bash_completion']), ('share/man/man1' , ['gallery-dl.1']), + ('share/man/man5' , ['gallery-dl.conf.5']), ] ]