mirror of
https://github.com/matrix-org/synapse-s3-storage-provider.git
synced 2024-10-23 07:29:40 +00:00
Generalize for usage in other S3-like services.
This commit is contained in:
parent
54c58922de
commit
503fd7ba38
1 changed files with 15 additions and 5 deletions
|
@ -46,12 +46,15 @@ class S3StorageProviderBackend(StorageProvider):
|
||||||
self.cache_directory = hs.config.media_store_path
|
self.cache_directory = hs.config.media_store_path
|
||||||
self.bucket = config["bucket"]
|
self.bucket = config["bucket"]
|
||||||
self.storage_class = config["storage_class"]
|
self.storage_class = config["storage_class"]
|
||||||
|
self.api_kwargs = {}
|
||||||
|
if "endpoint_url" in config:
|
||||||
|
self.api_kwargs["endpoint_url"] = config["endpoint_url"]
|
||||||
|
|
||||||
def store_file(self, path, file_info):
|
def store_file(self, path, file_info):
|
||||||
"""See StorageProvider.store_file"""
|
"""See StorageProvider.store_file"""
|
||||||
|
|
||||||
def _store_file():
|
def _store_file():
|
||||||
boto3.resource('s3').Bucket(self.bucket).upload_file(
|
boto3.resource('s3', **self.api_kwargs).Bucket(self.bucket).upload_file(
|
||||||
Filename=os.path.join(self.cache_directory, path),
|
Filename=os.path.join(self.cache_directory, path),
|
||||||
Key=path,
|
Key=path,
|
||||||
ExtraArgs={"StorageClass": self.storage_class},
|
ExtraArgs={"StorageClass": self.storage_class},
|
||||||
|
@ -64,7 +67,7 @@ class S3StorageProviderBackend(StorageProvider):
|
||||||
def fetch(self, path, file_info):
|
def fetch(self, path, file_info):
|
||||||
"""See StorageProvider.fetch"""
|
"""See StorageProvider.fetch"""
|
||||||
d = defer.Deferred()
|
d = defer.Deferred()
|
||||||
_S3DownloadThread(self.bucket, path, d).start()
|
_S3DownloadThread(self.bucket, self.api_kwargs, path, d).start()
|
||||||
return make_deferred_yieldable(d)
|
return make_deferred_yieldable(d)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
@ -82,17 +85,23 @@ class S3StorageProviderBackend(StorageProvider):
|
||||||
assert isinstance(bucket, basestring)
|
assert isinstance(bucket, basestring)
|
||||||
assert storage_class in _VALID_STORAGE_CLASSES
|
assert storage_class in _VALID_STORAGE_CLASSES
|
||||||
|
|
||||||
return {
|
result = {
|
||||||
"bucket": bucket,
|
"bucket": bucket,
|
||||||
"storage_class": storage_class,
|
"storage_class": storage_class,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if "endpoint_url" in config:
|
||||||
|
result["endpoint_url"] = config["endpoint_url"]
|
||||||
|
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
class _S3DownloadThread(threading.Thread):
|
class _S3DownloadThread(threading.Thread):
|
||||||
"""Attempts to download a file from S3.
|
"""Attempts to download a file from S3.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
bucket (str): The S3 bucket which may have the file
|
bucket (str): The S3 bucket which may have the file
|
||||||
|
api_kwargs (dict): Keyword arguments to pass when invoking the API. Generally `endpoint_url`.
|
||||||
key (str): The key of the file
|
key (str): The key of the file
|
||||||
deferred (Deferred[_S3Responder|None]): If file exists
|
deferred (Deferred[_S3Responder|None]): If file exists
|
||||||
resolved with an _S3Responder instance, if it doesn't
|
resolved with an _S3Responder instance, if it doesn't
|
||||||
|
@ -105,15 +114,16 @@ class _S3DownloadThread(threading.Thread):
|
||||||
|
|
||||||
READ_CHUNK_SIZE = 16 * 1024
|
READ_CHUNK_SIZE = 16 * 1024
|
||||||
|
|
||||||
def __init__(self, bucket, key, deferred):
|
def __init__(self, bucket, api_kwargs, key, deferred):
|
||||||
super(_S3DownloadThread, self).__init__(name="s3-download")
|
super(_S3DownloadThread, self).__init__(name="s3-download")
|
||||||
self.bucket = bucket
|
self.bucket = bucket
|
||||||
|
self.api_kwargs = api_kwargs
|
||||||
self.key = key
|
self.key = key
|
||||||
self.deferred = deferred
|
self.deferred = deferred
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
session = boto3.session.Session()
|
session = boto3.session.Session()
|
||||||
s3 = session.client('s3')
|
s3 = session.client('s3', **self.api_kwargs)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
resp = s3.get_object(Bucket=self.bucket, Key=self.key)
|
resp = s3.get_object(Bucket=self.bucket, Key=self.key)
|
||||||
|
|
Loading…
Reference in a new issue