"src/vscode:/vscode.git/clone" did not exist on "ec831b6a72ae8e511b011b9a590449a02b214928"
Unverified Commit f08343d4 authored by Nicolas Patry's avatar Nicolas Patry Committed by GitHub
Browse files

fix(server): Removes the parallelism in file convertion (during download) (#275)

parent b4fe248b
import concurrent import concurrent
import time import time
import datetime
import torch import torch
from concurrent.futures import ThreadPoolExecutor from concurrent.futures import ThreadPoolExecutor
...@@ -78,17 +79,9 @@ def convert_file(pt_file: Path, st_file: Path): ...@@ -78,17 +79,9 @@ def convert_file(pt_file: Path, st_file: Path):
def convert_files(pt_files: List[Path], st_files: List[Path]): def convert_files(pt_files: List[Path], st_files: List[Path]):
assert len(pt_files) == len(st_files) assert len(pt_files) == len(st_files)
executor = ThreadPoolExecutor(max_workers=5) N = len(pt_files)
futures = [
executor.submit(convert_file, pt_file=pt_file, st_file=st_file)
for pt_file, st_file in zip(pt_files, st_files)
]
# We do this instead of using tqdm because we want to parse the logs with the launcher # We do this instead of using tqdm because we want to parse the logs with the launcher
start_time = time.time() start = datetime.datetime.now()
for i, future in enumerate(concurrent.futures.as_completed(futures)): for i, (pt_file, sf_file) in enumerate(zip(pt_files, st_files)):
elapsed = timedelta(seconds=int(time.time() - start_time)) elapsed = datetime.datetime.now() - start
remaining = len(futures) - (i + 1) logger.info(f"Convert: [{i + 1}/{N}] -- Took: {elapsed}")
eta = (elapsed / (i + 1)) * remaining if remaining > 0 else 0
logger.info(f"Convert: [{i + 1}/{len(futures)}] -- ETA: {eta}")
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment