From 9e88e7a344f9e8a15b8057691e739c0be86efff2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Sun, 3 Nov 2019 21:45:45 +0100 Subject: [PATCH] [postprocessor:exec] improve (#421, #413) - add 'final' option - include job status in pp finalization - improve and extend documentation --- docs/configuration.rst | 27 +++++++++++++++++++++++---- gallery_dl/job.py | 13 +++++++++---- gallery_dl/postprocessor/common.py | 4 ++-- gallery_dl/postprocessor/exec.py | 22 +++++++++++++++++++--- gallery_dl/postprocessor/zip.py | 2 +- test/test_postprocessor.py | 4 ++-- 6 files changed, 56 insertions(+), 16 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 3b566067..5ed5a495 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -1388,12 +1388,31 @@ Description Controls whether to wait for a subprocess to finish exec.command ------------ =========== ===== -Type ``list`` of ``strings`` -Example ``["echo", "{user[account]}", "{id}"]`` +Type ``string`` or ``list`` of ``strings`` +Example * ``"convert {} {}.png && rm {}"`` + * ``["echo", "{user[account]}", "{id}"]`` Description The command to run. - Each element of this list is treated as a `format string`_ using - the files' metadata. + * If this is a ``string``, it will be executed using the system's + shell, e.g. ``/bin/sh``. Any ``{}`` will be replaced + with the full path of a file or target directory, depending on + `exec.final`_ + + * If this is a ``list``, the first element specifies the program + name and any further elements its arguments. + Each element of this list is treated as a `format string`_ using + the files' metadata as well as ``{_path}``, ``{_directory}``, + and ``{_filename}``. +=========== ===== + +exec.final +---------- +=========== ===== +Type ``bool`` +Default ``false`` +Description Controls whether to execute `exec.command`_ for each + downloaded file or only once after all files + have been downloaded successfully. =========== ===== diff --git a/gallery_dl/job.py b/gallery_dl/job.py index 9e8535a2..9c763368 100644 --- a/gallery_dl/job.py +++ b/gallery_dl/job.py @@ -67,6 +67,9 @@ class Job(): exc.__class__.__name__, exc) log.debug("", exc_info=True) self.status |= 1 + except BaseException: + self.status |= 1 + raise finally: self.handle_finalize() return self.status @@ -255,13 +258,15 @@ class DownloadJob(Job): self._write_unsupported(url) def handle_finalize(self): - if self.postprocessors: - for pp in self.postprocessors: - pp.finalize() + pathfmt = self.pathfmt if self.archive: self.archive.close() - if self.pathfmt: + if pathfmt: self.extractor._store_cookies() + if self.postprocessors: + status = self.status + for pp in self.postprocessors: + pp.run_final(pathfmt, status) def handle_skip(self): self.out.skip(self.pathfmt.path) diff --git a/gallery_dl/postprocessor/common.py b/gallery_dl/postprocessor/common.py index 71ef932a..83b42eb6 100644 --- a/gallery_dl/postprocessor/common.py +++ b/gallery_dl/postprocessor/common.py @@ -31,8 +31,8 @@ class PostProcessor(): """Execute postprocessor after moving a file to its target location""" @staticmethod - def finalize(): - """Cleanup""" + def run_final(pathfmt, status): + """Postprocessor finalization after all files have been downloaded""" def __repr__(self): return self.__class__.__name__ diff --git a/gallery_dl/postprocessor/exec.py b/gallery_dl/postprocessor/exec.py index 19a9b87d..0a56281b 100644 --- a/gallery_dl/postprocessor/exec.py +++ b/gallery_dl/postprocessor/exec.py @@ -26,17 +26,26 @@ class ExecPP(PostProcessor): def __init__(self, pathfmt, options): PostProcessor.__init__(self) args = options["command"] + final = options.get("final", False) if isinstance(args, str): + if final: + self._format = self._format_args_directory + else: + self._format = self._format_args_path if "{}" not in args: args += " {}" self.args = args self.shell = True - self._format = self._format_args_string else: + self._format = self._format_args_list self.args = [util.Formatter(arg) for arg in args] self.shell = False - self._format = self._format_args_list + + if final: + self.run_after = PostProcessor.run_after + else: + self.run_final = PostProcessor.run_final if options.get("async", False): self._exec = self._exec_async @@ -44,9 +53,16 @@ class ExecPP(PostProcessor): def run_after(self, pathfmt): self._exec(self._format(pathfmt)) - def _format_args_string(self, pathfmt): + def run_final(self, pathfmt, status): + if status == 0: + self._exec(self._format(pathfmt)) + + def _format_args_path(self, pathfmt): return self.args.replace("{}", quote(pathfmt.realpath)) + def _format_args_directory(self, pathfmt): + return self.args.replace("{}", quote(pathfmt.realdirectory)) + def _format_args_list(self, pathfmt): kwdict = pathfmt.kwdict kwdict["_directory"] = pathfmt.realdirectory diff --git a/gallery_dl/postprocessor/zip.py b/gallery_dl/postprocessor/zip.py index 6659a8de..42f76082 100644 --- a/gallery_dl/postprocessor/zip.py +++ b/gallery_dl/postprocessor/zip.py @@ -59,7 +59,7 @@ class ZipPP(PostProcessor): with zipfile.ZipFile(*self.args) as zfile: self._write(pathfmt, zfile) - def finalize(self): + def run_final(self, pathfmt, status): if self.zfile: self.zfile.close() diff --git a/test/test_postprocessor.py b/test/test_postprocessor.py index 78b98380..0ab89dbf 100644 --- a/test/test_postprocessor.py +++ b/test/test_postprocessor.py @@ -327,7 +327,7 @@ class ZipTest(BasePostprocessorTest): self.assertEqual(len(pp.zfile.NameToInfo), 3) # close file - pp.finalize() + pp.run_final(self.pathfmt, 0) # reopen to check persistence with zipfile.ZipFile(pp.zfile.filename) as file: @@ -360,7 +360,7 @@ class ZipTest(BasePostprocessorTest): pp.prepare(self.pathfmt) pp.run(self.pathfmt) - pp.finalize() + pp.run_final(self.pathfmt, 0) self.assertEqual(pp.zfile.write.call_count, 3) for call in pp.zfile.write.call_args_list: