mirror_maker/synapse-s3-storage-provider
1
0
Fork 1
mirror of https://github.com/matrix-org/synapse-s3-storage-provider.git synced 2024-10-23 07:29:40 +00:00
Code Issues Projects Releases Packages Wiki Activity

Option to not show progressbar.

Browse source
This commit is contained in:
Michael Kaye 2020-04-23 10:07:03 +01:00
parent d58c93d7ca
commit 03d21a79df
1 changed files with 22 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
scripts/s3_media_upload
View file

@ -10,6 +10,7 @@ import humanize
import psycopg2 import psycopg2
import tqdm import tqdm
import yaml import yaml
import sys
# Schema for our sqlite database cache # Schema for our sqlite database cache
SCHEMA = """ SCHEMA = """
@ -26,6 +27,7 @@ SCHEMA = """
CREATE INDEX IF NOT EXISTS deleted_idx ON media(known_deleted); CREATE INDEX IF NOT EXISTS deleted_idx ON media(known_deleted);
""" """
progress=True
def parse_duration(string): def parse_duration(string):
"""Parse a string into a duration supports suffix of d, m or y. """Parse a string into a duration supports suffix of d, m or y.
@ -108,11 +110,17 @@ def run_check_delete(sqlite_conn, base_path):
"""Entry point for check-deleted command """Entry point for check-deleted command
""" """
deleted = [] deleted = []
for origin, media_id, filesystem_id, m_type in tqdm.tqdm( if progress:
it = tqdm.tqdm(
get_not_deleted(sqlite_conn), get_not_deleted(sqlite_conn),
unit="files", unit="files",
total=get_not_deleted_count(sqlite_conn), total=get_not_deleted_count(sqlite_conn)
): )
else:
it = get_not_deleted(sqlite_conn)
print("Checking on ", get_not_deleted_count(sqlite_conn), " undeleted files")
for origin, media_id, filesystem_id, m_type in it:
if m_type == "local": if m_type == "local":
file_path = os.path.join( file_path = os.path.join(
base_path, base_path,
@ -246,7 +254,12 @@ def run_upload(
deleted_bytes = 0 deleted_bytes = 0
# This is a progress bar # This is a progress bar
if progress:
it = tqdm.tqdm(get_not_deleted(sqlite_conn), unit="files", total=total) it = tqdm.tqdm(get_not_deleted(sqlite_conn), unit="files", total=total)
else:
print("Uploading ", total, " files")
it = get_not_deleted(sqlite_conn)
for origin, media_id, filesystem_id, m_type in it: for origin, media_id, filesystem_id, m_type in it:
rel_file_path = to_path(origin, filesystem_id, m_type) rel_file_path = to_path(origin, filesystem_id, m_type)
@ -327,6 +340,7 @@ def get_postgres_conn(parser):
def main(): def main():
parser = argparse.ArgumentParser(prog="s3_media_upload") parser = argparse.ArgumentParser(prog="s3_media_upload")
parser.add_argument("--no-progress", help="do not show progress bars", action="store_true", dest="no_progress")
subparsers = parser.add_subparsers(help="command to run", dest="cmd") subparsers = parser.add_subparsers(help="command to run", dest="cmd")
update_db_parser = subparsers.add_parser( update_db_parser = subparsers.add_parser(
@ -406,6 +420,8 @@ def main():
) )
args = parser.parse_args() args = parser.parse_args()
if args.no_progress:
progress=False
if args.cmd == "write": if args.cmd == "write":
sqlite_conn = get_sqlite_conn(parser) sqlite_conn = get_sqlite_conn(parser)