mirror of
https://github.com/matrix-org/synapse-s3-storage-provider.git
synced 2024-10-23 07:29:40 +00:00
Merge pull request #48 from matrix-org/michaelkaye/allow_option_not_to_use_upload
Option to not show progressbar.
This commit is contained in:
commit
887ee24d76
1 changed files with 23 additions and 6 deletions
|
@ -10,6 +10,7 @@ import humanize
|
||||||
import psycopg2
|
import psycopg2
|
||||||
import tqdm
|
import tqdm
|
||||||
import yaml
|
import yaml
|
||||||
|
import sys
|
||||||
|
|
||||||
# Schema for our sqlite database cache
|
# Schema for our sqlite database cache
|
||||||
SCHEMA = """
|
SCHEMA = """
|
||||||
|
@ -26,6 +27,7 @@ SCHEMA = """
|
||||||
CREATE INDEX IF NOT EXISTS deleted_idx ON media(known_deleted);
|
CREATE INDEX IF NOT EXISTS deleted_idx ON media(known_deleted);
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
progress = True
|
||||||
|
|
||||||
def parse_duration(string):
|
def parse_duration(string):
|
||||||
"""Parse a string into a duration supports suffix of d, m or y.
|
"""Parse a string into a duration supports suffix of d, m or y.
|
||||||
|
@ -108,11 +110,17 @@ def run_check_delete(sqlite_conn, base_path):
|
||||||
"""Entry point for check-deleted command
|
"""Entry point for check-deleted command
|
||||||
"""
|
"""
|
||||||
deleted = []
|
deleted = []
|
||||||
for origin, media_id, filesystem_id, m_type in tqdm.tqdm(
|
if progress:
|
||||||
get_not_deleted(sqlite_conn),
|
it = tqdm.tqdm(
|
||||||
unit="files",
|
get_not_deleted(sqlite_conn),
|
||||||
total=get_not_deleted_count(sqlite_conn),
|
unit="files",
|
||||||
):
|
total=get_not_deleted_count(sqlite_conn)
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
it = get_not_deleted(sqlite_conn)
|
||||||
|
print("Checking on ", get_not_deleted_count(sqlite_conn), " undeleted files")
|
||||||
|
|
||||||
|
for origin, media_id, filesystem_id, m_type in it:
|
||||||
if m_type == "local":
|
if m_type == "local":
|
||||||
file_path = os.path.join(
|
file_path = os.path.join(
|
||||||
base_path,
|
base_path,
|
||||||
|
@ -246,7 +254,12 @@ def run_upload(
|
||||||
deleted_bytes = 0
|
deleted_bytes = 0
|
||||||
|
|
||||||
# This is a progress bar
|
# This is a progress bar
|
||||||
it = tqdm.tqdm(get_not_deleted(sqlite_conn), unit="files", total=total)
|
if progress:
|
||||||
|
it = tqdm.tqdm(get_not_deleted(sqlite_conn), unit="files", total=total)
|
||||||
|
else:
|
||||||
|
print("Uploading ", total, " files")
|
||||||
|
it = get_not_deleted(sqlite_conn)
|
||||||
|
|
||||||
for origin, media_id, filesystem_id, m_type in it:
|
for origin, media_id, filesystem_id, m_type in it:
|
||||||
rel_file_path = to_path(origin, filesystem_id, m_type)
|
rel_file_path = to_path(origin, filesystem_id, m_type)
|
||||||
|
|
||||||
|
@ -327,6 +340,7 @@ def get_postgres_conn(parser):
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
parser = argparse.ArgumentParser(prog="s3_media_upload")
|
parser = argparse.ArgumentParser(prog="s3_media_upload")
|
||||||
|
parser.add_argument("--no-progress", help="do not show progress bars", action="store_true", dest="no_progress")
|
||||||
subparsers = parser.add_subparsers(help="command to run", dest="cmd")
|
subparsers = parser.add_subparsers(help="command to run", dest="cmd")
|
||||||
|
|
||||||
update_db_parser = subparsers.add_parser(
|
update_db_parser = subparsers.add_parser(
|
||||||
|
@ -412,6 +426,9 @@ def main():
|
||||||
)
|
)
|
||||||
|
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
if args.no_progress:
|
||||||
|
global progress
|
||||||
|
progress = False
|
||||||
|
|
||||||
if args.cmd == "write":
|
if args.cmd == "write":
|
||||||
sqlite_conn = get_sqlite_conn(parser)
|
sqlite_conn = get_sqlite_conn(parser)
|
||||||
|
|
Loading…
Reference in a new issue