mirror_maker/synapse-s3-storage-provider
1
0
Fork 1
mirror of https://github.com/matrix-org/synapse-s3-storage-provider.git synced 2024-10-23 07:29:40 +00:00
Code Issues Projects Releases Packages Wiki Activity

Merge remote-tracking branch 'djmaze/master' into feature/improvements

Browse source
This commit is contained in:
Adam Hellberg 2019-03-14 22:43:28 +01:00
commit 30c30447e7
No known key found for this signature in database
GPG key ID: CFC01CAD18FBA436
1 changed files with 16 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
s3_storage_provider.py
View file

@ -50,12 +50,16 @@ class S3StorageProviderBackend(StorageProvider):
self.cache_directory = hs.config.media_store_path self.cache_directory = hs.config.media_store_path
self.bucket = config["bucket"] self.bucket = config["bucket"]
self.storage_class = config["storage_class"] self.storage_class = config["storage_class"]
self.api_kwargs = {}
if "endpoint_url" in config:
self.api_kwargs["endpoint_url"] = config["endpoint_url"]
def store_file(self, path, file_info): def store_file(self, path, file_info):
"""See StorageProvider.store_file""" """See StorageProvider.store_file"""
def _store_file(): def _store_file():
boto3.resource('s3').Bucket(self.bucket).upload_file( session = boto3.session.Session()
session.resource('s3', **self.api_kwargs).Bucket(self.bucket).upload_file(
Filename=os.path.join(self.cache_directory, path), Filename=os.path.join(self.cache_directory, path),
Key=path, Key=path,
ExtraArgs={"StorageClass": self.storage_class}, ExtraArgs={"StorageClass": self.storage_class},
@ -68,7 +72,7 @@ class S3StorageProviderBackend(StorageProvider):
def fetch(self, path, file_info): def fetch(self, path, file_info):
"""See StorageProvider.fetch""" """See StorageProvider.fetch"""
d = defer.Deferred() d = defer.Deferred()
_S3DownloadThread(self.bucket, path, d).start() _S3DownloadThread(self.bucket, self.api_kwargs, path, d).start()
return make_deferred_yieldable(d) return make_deferred_yieldable(d)
@staticmethod @staticmethod
@ -86,32 +90,39 @@ class S3StorageProviderBackend(StorageProvider):
assert isinstance(bucket, string_types) assert isinstance(bucket, string_types)
assert storage_class in _VALID_STORAGE_CLASSES assert storage_class in _VALID_STORAGE_CLASSES
return { result = {
"bucket": bucket, "bucket": bucket,
"storage_class": storage_class, "storage_class": storage_class,
} }
if "endpoint_url" in config:
result["endpoint_url"] = config["endpoint_url"]
return result
class _S3DownloadThread(threading.Thread): class _S3DownloadThread(threading.Thread):
"""Attempts to download a file from S3. """Attempts to download a file from S3.
Args: Args:
bucket (str): The S3 bucket which may have the file bucket (str): The S3 bucket which may have the file
api_kwargs (dict): Keyword arguments to pass when invoking the API. Generally `endpoint_url`.
key (str): The key of the file key (str): The key of the file
deferred (Deferred[_S3Responder|None]): If file exists deferred (Deferred[_S3Responder|None]): If file exists
resolved with an _S3Responder instance, if it doesn't resolved with an _S3Responder instance, if it doesn't
exist then resolves with None. exist then resolves with None.
""" """
def __init__(self, bucket, key, deferred): def __init__(self, bucket, api_kwargs, key, deferred):
super(_S3DownloadThread, self).__init__(name="s3-download") super(_S3DownloadThread, self).__init__(name="s3-download")
self.bucket = bucket self.bucket = bucket
self.api_kwargs = api_kwargs
self.key = key self.key = key
self.deferred = deferred self.deferred = deferred
def run(self): def run(self):
session = boto3.session.Session() session = boto3.session.Session()
s3 = session.client('s3') s3 = session.client('s3', **self.api_kwargs)
try: try:
resp = s3.get_object(Bucket=self.bucket, Key=self.key) resp = s3.get_object(Bucket=self.bucket, Key=self.key)