less complexity, better performance, but some duplicate code here and therepull/197/head
Mike Fährmann
6 years ago
parent
12482553bd
commit
34ea0d6a10
@ -1,220 +1,184 @@
|
|||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
# Copyright 2016-2018 Mike Fährmann
|
# Copyright 2016-2019 Mike Fährmann
|
||||||
#
|
#
|
||||||
# This program is free software; you can redistribute it and/or modify
|
# This program is free software; you can redistribute it and/or modify
|
||||||
# it under the terms of the GNU General Public License version 2 as
|
# it under the terms of the GNU General Public License version 2 as
|
||||||
# published by the Free Software Foundation.
|
# published by the Free Software Foundation.
|
||||||
|
|
||||||
"""Decorator to keep function results in a in-memory and database cache"""
|
"""Decorators to keep function results in an in-memory and database cache"""
|
||||||
|
|
||||||
import sqlite3
|
import sqlite3
|
||||||
import pickle
|
import pickle
|
||||||
import time
|
import time
|
||||||
import tempfile
|
|
||||||
import os.path
|
|
||||||
import functools
|
import functools
|
||||||
from . import config, util
|
from . import config, util
|
||||||
|
|
||||||
|
|
||||||
class CacheInvalidError(Exception):
|
class CacheDecorator():
|
||||||
"""A cache entry is either expired or does not exist"""
|
"""Simplified in-memory cache"""
|
||||||
pass
|
def __init__(self, func, keyarg):
|
||||||
|
self.func = func
|
||||||
|
self.cache = {}
|
||||||
class CacheModule():
|
self.keyarg = keyarg
|
||||||
"""Base class for cache modules"""
|
|
||||||
def __init__(self):
|
|
||||||
pass
|
|
||||||
|
|
||||||
def __getitem__(self, key):
|
|
||||||
raise CacheInvalidError()
|
|
||||||
|
|
||||||
def __setitem__(self, key, item):
|
|
||||||
pass
|
|
||||||
|
|
||||||
def __delitem__(self, key):
|
def __get__(self, instance, cls):
|
||||||
pass
|
return functools.partial(self.__call__, instance)
|
||||||
|
|
||||||
def __enter__(self):
|
def __call__(self, *args, **kwargs):
|
||||||
pass
|
key = "" if self.keyarg is None else args[self.keyarg]
|
||||||
|
try:
|
||||||
|
value = self.cache[key]
|
||||||
|
except KeyError:
|
||||||
|
value = self.cache[key] = self.func(*args, **kwargs)
|
||||||
|
return value
|
||||||
|
|
||||||
def __exit__(self, *exc_info):
|
def update(self, key, value):
|
||||||
pass
|
self.cache[key] = value
|
||||||
|
|
||||||
|
def invalidate(self, key):
|
||||||
|
try:
|
||||||
|
del self.cache[key]
|
||||||
|
except KeyError:
|
||||||
|
pass
|
||||||
|
|
||||||
class CacheChain(CacheModule):
|
|
||||||
|
|
||||||
def __init__(self, modules=[]):
|
class MemoryCacheDecorator(CacheDecorator):
|
||||||
CacheModule.__init__(self)
|
"""In-memory cache"""
|
||||||
self.modules = modules
|
def __init__(self, func, keyarg, maxage):
|
||||||
|
CacheDecorator.__init__(self, func, keyarg)
|
||||||
|
self.maxage = maxage
|
||||||
|
|
||||||
def __getitem__(self, key):
|
def __call__(self, *args, **kwargs):
|
||||||
num = 0
|
key = "" if self.keyarg is None else args[self.keyarg]
|
||||||
for module in self.modules:
|
timestamp = int(time.time())
|
||||||
try:
|
try:
|
||||||
value = module[key]
|
value, expires = self.cache[key]
|
||||||
break
|
except KeyError:
|
||||||
except CacheInvalidError:
|
expires = 0
|
||||||
num += 1
|
if expires < timestamp:
|
||||||
else:
|
value = self.func(*args, **kwargs)
|
||||||
raise CacheInvalidError()
|
expires = timestamp + self.maxage
|
||||||
while num:
|
self.cache[key] = value, expires
|
||||||
num -= 1
|
|
||||||
self.modules[num][key[0]] = value
|
|
||||||
return value
|
return value
|
||||||
|
|
||||||
def __setitem__(self, key, item):
|
def update(self, key, value):
|
||||||
for module in self.modules:
|
self.cache[key] = value, int(time.time()) + self.maxage
|
||||||
module.__setitem__(key, item)
|
|
||||||
|
|
||||||
def __delitem__(self, key):
|
|
||||||
for module in self.modules:
|
|
||||||
module.__delitem__(key)
|
|
||||||
|
|
||||||
def __exit__(self, exc_type, exc_value, exc_traceback):
|
class DatabaseCacheDecorator():
|
||||||
for module in self.modules:
|
"""Database cache"""
|
||||||
module.__exit__(exc_type, exc_value, exc_traceback)
|
db = None
|
||||||
|
_init = True
|
||||||
|
|
||||||
|
def __init__(self, func, keyarg, maxage):
|
||||||
class MemoryCache(CacheModule):
|
self.key = "%s.%s" % (func.__module__, func.__name__)
|
||||||
"""In-memory cache module"""
|
self.func = func
|
||||||
def __init__(self):
|
|
||||||
CacheModule.__init__(self)
|
|
||||||
self.cache = {}
|
self.cache = {}
|
||||||
|
self.keyarg = keyarg
|
||||||
|
self.maxage = maxage
|
||||||
|
|
||||||
def __getitem__(self, key):
|
def __get__(self, obj, objtype):
|
||||||
key, timestamp = key
|
return functools.partial(self.__call__, obj)
|
||||||
|
|
||||||
|
def __call__(self, *args, **kwargs):
|
||||||
|
key = "" if self.keyarg is None else args[self.keyarg]
|
||||||
|
timestamp = int(time.time())
|
||||||
|
|
||||||
|
# in-memory cache lookup
|
||||||
try:
|
try:
|
||||||
value, expires = self.cache[key]
|
value, expires = self.cache[key]
|
||||||
if timestamp < expires:
|
if expires > timestamp:
|
||||||
return value, expires
|
return value
|
||||||
except KeyError:
|
except KeyError:
|
||||||
pass
|
pass
|
||||||
raise CacheInvalidError()
|
|
||||||
|
|
||||||
def __setitem__(self, key, item):
|
# database lookup
|
||||||
self.cache[key] = item
|
fullkey = "%s-%s" % (self.key, key)
|
||||||
|
cursor = self.cursor()
|
||||||
|
try:
|
||||||
|
cursor.execute("BEGIN EXCLUSIVE")
|
||||||
|
except sqlite3.OperationalError:
|
||||||
|
pass # Silently swallow exception - workaround for Python 3.6
|
||||||
|
try:
|
||||||
|
cursor.execute(
|
||||||
|
"SELECT value, expires FROM data WHERE key=? LIMIT 1",
|
||||||
|
(fullkey,),
|
||||||
|
)
|
||||||
|
result = cursor.fetchone()
|
||||||
|
|
||||||
|
if result and result[1] > timestamp:
|
||||||
|
value, expires = result
|
||||||
|
value = pickle.loads(value)
|
||||||
|
else:
|
||||||
|
value = self.func(*args, **kwargs)
|
||||||
|
expires = timestamp + self.maxage
|
||||||
|
cursor.execute(
|
||||||
|
"INSERT OR REPLACE INTO data VALUES (?,?,?)",
|
||||||
|
(fullkey, pickle.dumps(value), expires),
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
self.db.commit()
|
||||||
|
self.cache[key] = value, expires
|
||||||
|
return value
|
||||||
|
|
||||||
|
def update(self, key, value):
|
||||||
|
expires = int(time.time()) + self.maxage
|
||||||
|
self.cache[key] = value, expires
|
||||||
|
self.cursor().execute(
|
||||||
|
"INSERT OR REPLACE INTO data VALUES (?,?,?)",
|
||||||
|
("%s-%s" % (self.key, key), pickle.dumps(value), expires),
|
||||||
|
)
|
||||||
|
|
||||||
def __delitem__(self, key):
|
def invalidate(self, key):
|
||||||
try:
|
try:
|
||||||
del self.cache[key]
|
del self.cache[key]
|
||||||
except KeyError:
|
except KeyError:
|
||||||
pass
|
pass
|
||||||
|
self.cursor().execute(
|
||||||
|
"DELETE FROM data WHERE key=? LIMIT 1",
|
||||||
class DatabaseCache(CacheModule):
|
("%s-%s" % (self.key, key),),
|
||||||
"""Database cache module"""
|
|
||||||
def __init__(self):
|
|
||||||
CacheModule.__init__(self)
|
|
||||||
path_default = os.path.join(tempfile.gettempdir(), ".gallery-dl.cache")
|
|
||||||
path = config.get(("cache", "file"), path_default)
|
|
||||||
if path is None:
|
|
||||||
raise RuntimeError()
|
|
||||||
path = util.expand_path(path)
|
|
||||||
self.db = sqlite3.connect(path, timeout=30, check_same_thread=False)
|
|
||||||
self.db.execute(
|
|
||||||
"CREATE TABLE IF NOT EXISTS data ("
|
|
||||||
"key TEXT PRIMARY KEY,"
|
|
||||||
"value TEXT,"
|
|
||||||
"expires INTEGER"
|
|
||||||
")"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
def __getitem__(self, key):
|
def cursor(self):
|
||||||
key, timestamp = key
|
if self._init:
|
||||||
try:
|
self.db.execute(
|
||||||
cursor = self.db.cursor()
|
"CREATE TABLE IF NOT EXISTS data "
|
||||||
try:
|
"(key TEXT PRIMARY KEY, value TEXT, expires INTEGER)"
|
||||||
cursor.execute("BEGIN EXCLUSIVE")
|
|
||||||
except sqlite3.OperationalError:
|
|
||||||
"""workaround for python 3.6"""
|
|
||||||
cursor.execute(
|
|
||||||
"SELECT value, expires "
|
|
||||||
"FROM data "
|
|
||||||
"WHERE key=?",
|
|
||||||
(key,)
|
|
||||||
)
|
)
|
||||||
value, expires = cursor.fetchone()
|
DatabaseCacheDecorator._init = False
|
||||||
if timestamp < expires:
|
return self.db.cursor()
|
||||||
self.commit()
|
|
||||||
return pickle.loads(value), expires
|
|
||||||
except TypeError:
|
|
||||||
pass
|
|
||||||
raise CacheInvalidError()
|
|
||||||
|
|
||||||
def __setitem__(self, key, item):
|
|
||||||
value, expires = item
|
|
||||||
self.db.execute("INSERT OR REPLACE INTO data VALUES (?,?,?)",
|
|
||||||
(key, pickle.dumps(value), expires))
|
|
||||||
|
|
||||||
def __delitem__(self, key):
|
|
||||||
self.db.execute("DELETE FROM data WHERE key=?", (key,))
|
|
||||||
|
|
||||||
def __exit__(self, *exc_info):
|
|
||||||
self.commit()
|
|
||||||
|
|
||||||
def commit(self):
|
|
||||||
self.db.commit()
|
|
||||||
|
|
||||||
|
|
||||||
class CacheDecorator():
|
|
||||||
|
|
||||||
def __init__(self, func, module, maxage, keyarg):
|
|
||||||
self.func = func
|
|
||||||
self.key = "%s.%s" % (func.__module__, func.__name__)
|
|
||||||
self.cache = module
|
|
||||||
self.maxage = maxage
|
|
||||||
self.keyarg = keyarg
|
|
||||||
|
|
||||||
def __call__(self, *args, **kwargs):
|
|
||||||
timestamp = time.time()
|
|
||||||
if self.keyarg is None:
|
|
||||||
key = self.key
|
|
||||||
else:
|
|
||||||
key = "%s-%s" % (self.key, args[self.keyarg])
|
|
||||||
try:
|
|
||||||
result, _ = self.cache[key, timestamp]
|
|
||||||
except CacheInvalidError:
|
|
||||||
with self.cache:
|
|
||||||
result = self.func(*args, **kwargs)
|
|
||||||
expires = int(timestamp + self.maxage)
|
|
||||||
self.cache[key] = result, expires
|
|
||||||
return result
|
|
||||||
|
|
||||||
def __get__(self, obj, objtype):
|
def memcache(maxage=None, keyarg=None):
|
||||||
"""Support instance methods."""
|
if maxage:
|
||||||
return functools.partial(self.__call__, obj)
|
def wrap(func):
|
||||||
|
return MemoryCacheDecorator(func, keyarg, maxage)
|
||||||
|
else:
|
||||||
|
def wrap(func):
|
||||||
|
return CacheDecorator(func, keyarg)
|
||||||
|
return wrap
|
||||||
|
|
||||||
def invalidate(self, key=None):
|
|
||||||
key = "%s-%s" % (self.key, key) if key else self.key
|
|
||||||
del self.cache[key]
|
|
||||||
|
|
||||||
def update(self, key, result):
|
def cache(maxage=3600, keyarg=None):
|
||||||
key = "%s-%s" % (self.key, key) if key else self.key
|
def wrap(func):
|
||||||
expires = int(time.time() + self.maxage)
|
return DatabaseCacheDecorator(func, keyarg, maxage)
|
||||||
self.cache[key] = result, expires
|
return wrap
|
||||||
|
|
||||||
|
|
||||||
def build_cache_decorator(*modules):
|
try:
|
||||||
if len(modules) > 1:
|
path = config.get(("cache", "file"), "")
|
||||||
module = CacheChain(modules)
|
if path is None:
|
||||||
|
raise RuntimeError()
|
||||||
|
elif not path:
|
||||||
|
import tempfile
|
||||||
|
import os.path
|
||||||
|
path = os.path.join(tempfile.gettempdir(), ".gallery-dl.cache")
|
||||||
else:
|
else:
|
||||||
module = modules[0]
|
path = util.expand_path(path)
|
||||||
|
|
||||||
def decorator(maxage=3600, keyarg=None):
|
|
||||||
def wrap(func):
|
|
||||||
return CacheDecorator(func, module, maxage, keyarg)
|
|
||||||
return wrap
|
|
||||||
return decorator
|
|
||||||
|
|
||||||
|
|
||||||
MEMCACHE = MemoryCache()
|
DatabaseCacheDecorator.db = sqlite3.connect(
|
||||||
memcache = build_cache_decorator(MEMCACHE)
|
path, timeout=30, check_same_thread=False)
|
||||||
|
|
||||||
try:
|
|
||||||
DBCACHE = DatabaseCache()
|
|
||||||
cache = build_cache_decorator(MEMCACHE, DBCACHE)
|
|
||||||
except (RuntimeError, sqlite3.OperationalError):
|
except (RuntimeError, sqlite3.OperationalError):
|
||||||
DBCACHE = None
|
cache = memcache # noqa: F811
|
||||||
cache = memcache
|
|
||||||
|
|||||||
Loading…
Reference in new issue