From 523ebc9b0bc0b1109b92df3fa4b659e494a6fd58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Thu, 9 May 2019 16:22:06 +0200 Subject: [PATCH] Fix serialization of 'datetime' objects in '--write-metadata' Simplified universal serialization support in json.dump() can be achieved by passing 'default=str', which was already the case in DataJob.run() for -j/--dump-json, but not for the 'metadata' post-processor. This commit introduces util.dump_json() that (more or less) unifies the JSON output procedure of both --write-metadata and --dump-json. (#251, #252) --- gallery_dl/job.py | 8 ++------ gallery_dl/postprocessor/metadata.py | 9 +-------- gallery_dl/util.py | 13 +++++++++++++ 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/gallery_dl/job.py b/gallery_dl/job.py index b8e9a6d2..667b9b37 100644 --- a/gallery_dl/job.py +++ b/gallery_dl/job.py @@ -8,7 +8,6 @@ import sys import time -import json import logging from . import extractor, downloader, postprocessor from . import config, text, util, output, exception @@ -469,16 +468,13 @@ class DataJob(Job): except BaseException: pass + # convert numbers to string if config.get(("output", "num-to-str"), False): for msg in self.data: util.transform_dict(msg[-1], util.number_to_string) # dump to 'file' - json.dump( - self.data, self.file, - sort_keys=True, indent=2, ensure_ascii=self.ascii, default=str, - ) - self.file.write("\n") + util.dump_json(self.data, self.file, self.ascii, 2) def handle_url(self, url, kwdict): self.data.append((Message.Url, url, self._filter(kwdict))) diff --git a/gallery_dl/postprocessor/metadata.py b/gallery_dl/postprocessor/metadata.py index 3aae9431..4c6ca8c2 100644 --- a/gallery_dl/postprocessor/metadata.py +++ b/gallery_dl/postprocessor/metadata.py @@ -10,7 +10,6 @@ from .common import PostProcessor from .. import util -import json class MetadataPP(PostProcessor): @@ -61,13 +60,7 @@ class MetadataPP(PostProcessor): file.write("\n") def _write_json(self, file, pathfmt): - json.dump( - pathfmt.keywords, - file, - sort_keys=True, - indent=self.indent, - ensure_ascii=self.ascii, - ) + util.dump_json(pathfmt.keywords, file, self.ascii, self.indent) __postprocessor__ = MetadataPP diff --git a/gallery_dl/util.py b/gallery_dl/util.py index 063816be..1c2b1606 100644 --- a/gallery_dl/util.py +++ b/gallery_dl/util.py @@ -11,6 +11,7 @@ import re import os import sys +import json import shutil import string import _string @@ -92,6 +93,18 @@ def to_string(value): return str(value) +def dump_json(obj, fp=sys.stdout, ensure_ascii=True, indent=4): + """Serialize 'obj' as JSON and write it to 'fp'""" + json.dump( + obj, fp, + ensure_ascii=ensure_ascii, + indent=indent, + default=str, + sort_keys=True, + ) + fp.write("\n") + + def expand_path(path): """Expand environment variables and tildes (~)""" if not path: