mirror of
https://github.com/mikf/gallery-dl.git
synced 2024-11-22 10:42:34 +01:00
rewrite cache module
less complexity, better performance, but some duplicate code here and there
This commit is contained in:
Mike Fährmann
parent
12482553bd
commit
34ea0d6a10
@ -1,220 +1,184 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2016-2018 Mike Fährmann
|
||||
# Copyright 2016-2019 Mike Fährmann
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License version 2 as
|
||||
# published by the Free Software Foundation.
|
||||
|
||||
"""Decorator to keep function results in a in-memory and database cache"""
|
||||
"""Decorators to keep function results in an in-memory and database cache"""
|
||||
|
||||
import sqlite3
|
||||
import pickle
|
||||
import time
|
||||
import tempfile
|
||||
import os.path
|
||||
import functools
|
||||
from . import config, util
|
||||
|
||||
|
||||
class CacheInvalidError(Exception):
|
||||
"""A cache entry is either expired or does not exist"""
|
||||
pass
|
||||
class CacheDecorator():
|
||||
"""Simplified in-memory cache"""
|
||||
def __init__(self, func, keyarg):
|
||||
self.func = func
|
||||
self.cache = {}
|
||||
self.keyarg = keyarg
|
||||
|
||||
def __get__(self, instance, cls):
|
||||
return functools.partial(self.__call__, instance)
|
||||
|
||||
class CacheModule():
|
||||
"""Base class for cache modules"""
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def __getitem__(self, key):
|
||||
raise CacheInvalidError()
|
||||
|
||||
def __setitem__(self, key, item):
|
||||
pass
|
||||
|
||||
def __delitem__(self, key):
|
||||
pass
|
||||
|
||||
def __enter__(self):
|
||||
pass
|
||||
|
||||
def __exit__(self, *exc_info):
|
||||
pass
|
||||
|
||||
|
||||
class CacheChain(CacheModule):
|
||||
|
||||
def __init__(self, modules=[]):
|
||||
CacheModule.__init__(self)
|
||||
self.modules = modules
|
||||
|
||||
def __getitem__(self, key):
|
||||
num = 0
|
||||
for module in self.modules:
|
||||
try:
|
||||
value = module[key]
|
||||
break
|
||||
except CacheInvalidError:
|
||||
num += 1
|
||||
else:
|
||||
raise CacheInvalidError()
|
||||
while num:
|
||||
num -= 1
|
||||
self.modules[num][key[0]] = value
|
||||
def __call__(self, *args, **kwargs):
|
||||
key = "" if self.keyarg is None else args[self.keyarg]
|
||||
try:
|
||||
value = self.cache[key]
|
||||
except KeyError:
|
||||
value = self.cache[key] = self.func(*args, **kwargs)
|
||||
return value
|
||||
|
||||
def __setitem__(self, key, item):
|
||||
for module in self.modules:
|
||||
module.__setitem__(key, item)
|
||||
def update(self, key, value):
|
||||
self.cache[key] = value
|
||||
|
||||
def __delitem__(self, key):
|
||||
for module in self.modules:
|
||||
module.__delitem__(key)
|
||||
|
||||
def __exit__(self, exc_type, exc_value, exc_traceback):
|
||||
for module in self.modules:
|
||||
module.__exit__(exc_type, exc_value, exc_traceback)
|
||||
|
||||
|
||||
class MemoryCache(CacheModule):
|
||||
"""In-memory cache module"""
|
||||
def __init__(self):
|
||||
CacheModule.__init__(self)
|
||||
self.cache = {}
|
||||
|
||||
def __getitem__(self, key):
|
||||
key, timestamp = key
|
||||
try:
|
||||
value, expires = self.cache[key]
|
||||
if timestamp < expires:
|
||||
return value, expires
|
||||
except KeyError:
|
||||
pass
|
||||
raise CacheInvalidError()
|
||||
|
||||
def __setitem__(self, key, item):
|
||||
self.cache[key] = item
|
||||
|
||||
def __delitem__(self, key):
|
||||
def invalidate(self, key):
|
||||
try:
|
||||
del self.cache[key]
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
|
||||
class DatabaseCache(CacheModule):
|
||||
"""Database cache module"""
|
||||
def __init__(self):
|
||||
CacheModule.__init__(self)
|
||||
path_default = os.path.join(tempfile.gettempdir(), ".gallery-dl.cache")
|
||||
path = config.get(("cache", "file"), path_default)
|
||||
if path is None:
|
||||
raise RuntimeError()
|
||||
path = util.expand_path(path)
|
||||
self.db = sqlite3.connect(path, timeout=30, check_same_thread=False)
|
||||
self.db.execute(
|
||||
"CREATE TABLE IF NOT EXISTS data ("
|
||||
"key TEXT PRIMARY KEY,"
|
||||
"value TEXT,"
|
||||
"expires INTEGER"
|
||||
")"
|
||||
)
|
||||
|
||||
def __getitem__(self, key):
|
||||
key, timestamp = key
|
||||
try:
|
||||
cursor = self.db.cursor()
|
||||
try:
|
||||
cursor.execute("BEGIN EXCLUSIVE")
|
||||
except sqlite3.OperationalError:
|
||||
"""workaround for python 3.6"""
|
||||
cursor.execute(
|
||||
"SELECT value, expires "
|
||||
"FROM data "
|
||||
"WHERE key=?",
|
||||
(key,)
|
||||
)
|
||||
value, expires = cursor.fetchone()
|
||||
if timestamp < expires:
|
||||
self.commit()
|
||||
return pickle.loads(value), expires
|
||||
except TypeError:
|
||||
pass
|
||||
raise CacheInvalidError()
|
||||
|
||||
def __setitem__(self, key, item):
|
||||
value, expires = item
|
||||
self.db.execute("INSERT OR REPLACE INTO data VALUES (?,?,?)",
|
||||
(key, pickle.dumps(value), expires))
|
||||
|
||||
def __delitem__(self, key):
|
||||
self.db.execute("DELETE FROM data WHERE key=?", (key,))
|
||||
|
||||
def __exit__(self, *exc_info):
|
||||
self.commit()
|
||||
|
||||
def commit(self):
|
||||
self.db.commit()
|
||||
|
||||
|
||||
class CacheDecorator():
|
||||
|
||||
def __init__(self, func, module, maxage, keyarg):
|
||||
self.func = func
|
||||
self.key = "%s.%s" % (func.__module__, func.__name__)
|
||||
self.cache = module
|
||||
class MemoryCacheDecorator(CacheDecorator):
|
||||
"""In-memory cache"""
|
||||
def __init__(self, func, keyarg, maxage):
|
||||
CacheDecorator.__init__(self, func, keyarg)
|
||||
self.maxage = maxage
|
||||
self.keyarg = keyarg
|
||||
|
||||
def __call__(self, *args, **kwargs):
|
||||
timestamp = time.time()
|
||||
if self.keyarg is None:
|
||||
key = self.key
|
||||
else:
|
||||
key = "%s-%s" % (self.key, args[self.keyarg])
|
||||
key = "" if self.keyarg is None else args[self.keyarg]
|
||||
timestamp = int(time.time())
|
||||
try:
|
||||
result, _ = self.cache[key, timestamp]
|
||||
except CacheInvalidError:
|
||||
with self.cache:
|
||||
result = self.func(*args, **kwargs)
|
||||
expires = int(timestamp + self.maxage)
|
||||
self.cache[key] = result, expires
|
||||
return result
|
||||
value, expires = self.cache[key]
|
||||
except KeyError:
|
||||
expires = 0
|
||||
if expires < timestamp:
|
||||
value = self.func(*args, **kwargs)
|
||||
expires = timestamp + self.maxage
|
||||
self.cache[key] = value, expires
|
||||
return value
|
||||
|
||||
def update(self, key, value):
|
||||
self.cache[key] = value, int(time.time()) + self.maxage
|
||||
|
||||
|
||||
class DatabaseCacheDecorator():
|
||||
"""Database cache"""
|
||||
db = None
|
||||
_init = True
|
||||
|
||||
def __init__(self, func, keyarg, maxage):
|
||||
self.key = "%s.%s" % (func.__module__, func.__name__)
|
||||
self.func = func
|
||||
self.cache = {}
|
||||
self.keyarg = keyarg
|
||||
self.maxage = maxage
|
||||
|
||||
def __get__(self, obj, objtype):
|
||||
"""Support instance methods."""
|
||||
return functools.partial(self.__call__, obj)
|
||||
|
||||
def invalidate(self, key=None):
|
||||
key = "%s-%s" % (self.key, key) if key else self.key
|
||||
del self.cache[key]
|
||||
def __call__(self, *args, **kwargs):
|
||||
key = "" if self.keyarg is None else args[self.keyarg]
|
||||
timestamp = int(time.time())
|
||||
|
||||
def update(self, key, result):
|
||||
key = "%s-%s" % (self.key, key) if key else self.key
|
||||
expires = int(time.time() + self.maxage)
|
||||
self.cache[key] = result, expires
|
||||
# in-memory cache lookup
|
||||
try:
|
||||
value, expires = self.cache[key]
|
||||
if expires > timestamp:
|
||||
return value
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
# database lookup
|
||||
fullkey = "%s-%s" % (self.key, key)
|
||||
cursor = self.cursor()
|
||||
try:
|
||||
cursor.execute("BEGIN EXCLUSIVE")
|
||||
except sqlite3.OperationalError:
|
||||
pass # Silently swallow exception - workaround for Python 3.6
|
||||
try:
|
||||
cursor.execute(
|
||||
"SELECT value, expires FROM data WHERE key=? LIMIT 1",
|
||||
(fullkey,),
|
||||
)
|
||||
result = cursor.fetchone()
|
||||
|
||||
if result and result[1] > timestamp:
|
||||
value, expires = result
|
||||
value = pickle.loads(value)
|
||||
else:
|
||||
value = self.func(*args, **kwargs)
|
||||
expires = timestamp + self.maxage
|
||||
cursor.execute(
|
||||
"INSERT OR REPLACE INTO data VALUES (?,?,?)",
|
||||
(fullkey, pickle.dumps(value), expires),
|
||||
)
|
||||
finally:
|
||||
self.db.commit()
|
||||
self.cache[key] = value, expires
|
||||
return value
|
||||
|
||||
def update(self, key, value):
|
||||
expires = int(time.time()) + self.maxage
|
||||
self.cache[key] = value, expires
|
||||
self.cursor().execute(
|
||||
"INSERT OR REPLACE INTO data VALUES (?,?,?)",
|
||||
("%s-%s" % (self.key, key), pickle.dumps(value), expires),
|
||||
)
|
||||
|
||||
def invalidate(self, key):
|
||||
try:
|
||||
del self.cache[key]
|
||||
except KeyError:
|
||||
pass
|
||||
self.cursor().execute(
|
||||
"DELETE FROM data WHERE key=? LIMIT 1",
|
||||
("%s-%s" % (self.key, key),),
|
||||
)
|
||||
|
||||
def cursor(self):
|
||||
if self._init:
|
||||
self.db.execute(
|
||||
"CREATE TABLE IF NOT EXISTS data "
|
||||
"(key TEXT PRIMARY KEY, value TEXT, expires INTEGER)"
|
||||
)
|
||||
DatabaseCacheDecorator._init = False
|
||||
return self.db.cursor()
|
||||
|
||||
|
||||
def build_cache_decorator(*modules):
|
||||
if len(modules) > 1:
|
||||
module = CacheChain(modules)
|
||||
else:
|
||||
module = modules[0]
|
||||
|
||||
def decorator(maxage=3600, keyarg=None):
|
||||
def memcache(maxage=None, keyarg=None):
|
||||
if maxage:
|
||||
def wrap(func):
|
||||
return CacheDecorator(func, module, maxage, keyarg)
|
||||
return wrap
|
||||
return decorator
|
||||
return MemoryCacheDecorator(func, keyarg, maxage)
|
||||
else:
|
||||
def wrap(func):
|
||||
return CacheDecorator(func, keyarg)
|
||||
return wrap
|
||||
|
||||
|
||||
MEMCACHE = MemoryCache()
|
||||
memcache = build_cache_decorator(MEMCACHE)
|
||||
def cache(maxage=3600, keyarg=None):
|
||||
def wrap(func):
|
||||
return DatabaseCacheDecorator(func, keyarg, maxage)
|
||||
return wrap
|
||||
|
||||
|
||||
try:
|
||||
DBCACHE = DatabaseCache()
|
||||
cache = build_cache_decorator(MEMCACHE, DBCACHE)
|
||||
path = config.get(("cache", "file"), "")
|
||||
if path is None:
|
||||
raise RuntimeError()
|
||||
elif not path:
|
||||
import tempfile
|
||||
import os.path
|
||||
path = os.path.join(tempfile.gettempdir(), ".gallery-dl.cache")
|
||||
else:
|
||||
path = util.expand_path(path)
|
||||
|
||||
DatabaseCacheDecorator.db = sqlite3.connect(
|
||||
path, timeout=30, check_same_thread=False)
|
||||
|
||||
except (RuntimeError, sqlite3.OperationalError):
|
||||
DBCACHE = None
|
||||
cache = memcache
|
||||
cache = memcache # noqa: F811
|
||||
|
||||
Loading…
Reference in New Issue
Block a user