mirror_maker
/
gallery-dl
mirror of https://github.com/mikf/gallery-dl.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[archive] move DownloadArchive into its own module

Browse Source
pull/5580/head
Mike Fährmann 4 months ago
parent 2ee9ffeed6
commit 88f94190f4
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88
4 changed files with 95 additions and 68 deletions
Show Stats Download Patch File Download Diff File

56
gallery_dl/archive.py
Unescape View File

@ -0,0 +1,56 @@
# -*- coding: utf-8 -*-
# Copyright 2024 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
"""Download Archives"""
import os
import sqlite3
from . import formatter
class DownloadArchive():
def __init__(self, path, format_string, pragma=None,
cache_key="_archive_key"):
try:
con = sqlite3.connect(path, timeout=60, check_same_thread=False)
except sqlite3.OperationalError:
os.makedirs(os.path.dirname(path))
con = sqlite3.connect(path, timeout=60, check_same_thread=False)
con.isolation_level = None
self.keygen = formatter.parse(format_string).format_map
self.connection = con
self.close = con.close
self.cursor = cursor = con.cursor()
self._cache_key = cache_key
if pragma:
for stmt in pragma:
cursor.execute("PRAGMA " + stmt)
try:
cursor.execute("CREATE TABLE IF NOT EXISTS archive "
"(entry TEXT PRIMARY KEY) WITHOUT ROWID")
except sqlite3.OperationalError:
# fallback for missing WITHOUT ROWID support (#553)
cursor.execute("CREATE TABLE IF NOT EXISTS archive "
"(entry TEXT PRIMARY KEY)")
def add(self, kwdict):
"""Add item described by 'kwdict' to archive"""
key = kwdict.get(self._cache_key) or self.keygen(kwdict)
self.cursor.execute(
"INSERT OR IGNORE INTO archive (entry) VALUES (?)", (key,))
def check(self, kwdict):
"""Return True if the item described by 'kwdict' exists in archive"""
key = kwdict[self._cache_key] = self.keygen(kwdict)
self.cursor.execute(
"SELECT 1 FROM archive WHERE entry=? LIMIT 1", (key,))
return self.cursor.fetchone()

38
gallery_dl/job.py
Unescape View File

@ -11,10 +11,23 @@ import errno
import logging import logging
import functools import functools
import collections import collections
from . import extractor, downloader, postprocessor
from . import config, text, util, path, formatter, output, exception, version from . import (
extractor,
downloader,
postprocessor,
archive,
config,
exception,
formatter,
output,
path,
text,
util,
version,
)
from .extractor.message import Message from .extractor.message import Message
from .output import stdout_write stdout_write = output.stdout_write
class Job(): class Job():
@ -507,23 +520,24 @@ class DownloadJob(Job):
# monkey-patch method to do nothing and always return True # monkey-patch method to do nothing and always return True
self.download = pathfmt.fix_extension self.download = pathfmt.fix_extension
archive = cfg("archive") archive_path = cfg("archive")
if archive: if archive_path:
archive = util.expand_path(archive) archive_path = util.expand_path(archive_path)
archive_format = (cfg("archive-prefix", extr.category) + archive_format = (cfg("archive-prefix", extr.category) +
cfg("archive-format", extr.archive_fmt)) cfg("archive-format", extr.archive_fmt))
archive_pragma = (cfg("archive-pragma")) archive_pragma = (cfg("archive-pragma"))
try: try:
if "{" in archive: if "{" in archive_path:
archive = formatter.parse(archive).format_map(kwdict) archive_path = formatter.parse(
self.archive = util.DownloadArchive( archive_path).format_map(kwdict)
archive, archive_format, archive_pragma) self.archive = archive.DownloadArchive(
archive_path, archive_format, archive_pragma)
except Exception as exc: except Exception as exc:
extr.log.warning( extr.log.warning(
"Failed to open download archive at '%s' (%s: %s)", "Failed to open download archive at '%s' (%s: %s)",
archive, exc.__class__.__name__, exc) archive_path, exc.__class__.__name__, exc)
else: else:
extr.log.debug("Using download archive '%s'", archive) extr.log.debug("Using download archive '%s'", archive_path)
skip = cfg("skip", True) skip = cfg("skip", True)
if skip: if skip:

23
gallery_dl/postprocessor/common.py
Unescape View File

@ -8,7 +8,7 @@
"""Common classes and constants used by postprocessor modules.""" """Common classes and constants used by postprocessor modules."""
from .. import util, formatter from .. import util, formatter, archive
class PostProcessor(): class PostProcessor():
@ -22,30 +22,31 @@ class PostProcessor():
return self.__class__.__name__ return self.__class__.__name__
def _init_archive(self, job, options, prefix=None): def _init_archive(self, job, options, prefix=None):
archive = options.get("archive") archive_path = options.get("archive")
if archive: if archive_path:
extr = job.extractor extr = job.extractor
archive = util.expand_path(archive) archive_path = util.expand_path(archive_path)
if not prefix: if not prefix:
prefix = "_" + self.name.upper() + "_" prefix = "_" + self.name.upper() + "_"
archive_format = ( archive_format = (
options.get("archive-prefix", extr.category) + options.get("archive-prefix", extr.category) +
options.get("archive-format", prefix + extr.archive_fmt)) options.get("archive-format", prefix + extr.archive_fmt))
try: try:
if "{" in archive: if "{" in archive_path:
archive = formatter.parse(archive).format_map( archive_path = formatter.parse(archive_path).format_map(
job.pathfmt.kwdict) job.pathfmt.kwdict)
self.archive = util.DownloadArchive( self.archive = archive.DownloadArchive(
archive, archive_format, archive_path, archive_format,
options.get("archive-pragma"), options.get("archive-pragma"),
"_archive_" + self.name) "_archive_" + self.name)
except Exception as exc: except Exception as exc:
self.log.warning( self.log.warning(
"Failed to open %s archive at '%s' (%s: %s)", "Failed to open %s archive at '%s' (%s: %s)",
self.name, archive, exc.__class__.__name__, exc) self.name, archive_path, exc.__class__.__name__, exc)
else: else:
self.log.debug("Using %s archive '%s'", self.name, archive) self.log.debug(
"Using %s archive '%s'", self.name, archive_path)
return True return True
else:
self.archive = None self.archive = None
return False return False

44
gallery_dl/util.py
Unescape View File

@ -16,7 +16,6 @@ import time
import random import random
import getpass import getpass
import hashlib import hashlib
import sqlite3
import binascii import binascii
import datetime import datetime
import functools import functools
@ -852,46 +851,3 @@ class FilterPredicate():
raise raise
except Exception as exc: except Exception as exc:
raise exception.FilterError(exc) raise exception.FilterError(exc)
class DownloadArchive():
def __init__(self, path, format_string, pragma=None,
cache_key="_archive_key"):
try:
con = sqlite3.connect(path, timeout=60, check_same_thread=False)
except sqlite3.OperationalError:
os.makedirs(os.path.dirname(path))
con = sqlite3.connect(path, timeout=60, check_same_thread=False)
con.isolation_level = None
from . import formatter
self.keygen = formatter.parse(format_string).format_map
self.close = con.close
self.cursor = cursor = con.cursor()
self._cache_key = cache_key
if pragma:
for stmt in pragma:
cursor.execute("PRAGMA " + stmt)
try:
cursor.execute("CREATE TABLE IF NOT EXISTS archive "
"(entry TEXT PRIMARY KEY) WITHOUT ROWID")
except sqlite3.OperationalError:
# fallback for missing WITHOUT ROWID support (#553)
cursor.execute("CREATE TABLE IF NOT EXISTS archive "
"(entry TEXT PRIMARY KEY)")
def check(self, kwdict):
"""Return True if the item described by 'kwdict' exists in archive"""
key = kwdict[self._cache_key] = self.keygen(kwdict)
self.cursor.execute(
"SELECT 1 FROM archive WHERE entry=? LIMIT 1", (key,))
return self.cursor.fetchone()
def add(self, kwdict):
"""Add item described by 'kwdict' to archive"""
key = kwdict.get(self._cache_key) or self.keygen(kwdict)
self.cursor.execute(
"INSERT OR IGNORE INTO archive (entry) VALUES (?)", (key,))

Write Preview
Loading…
Cancel
Save