Unverified Commit 13fd3dca authored by Shaden Smith's avatar Shaden Smith Committed by GitHub
Browse files

Removes DeepSpeedDataSource (#100)



* Removes DeepSpeedDataSource

* dropping unused imports
Co-authored-by: default avatarJeff Rasley <jerasley@microsoft.com>
parent 6d602065
......@@ -3,26 +3,8 @@ Copyright 2019 The Microsoft DeepSpeed Team
'''
import torch
import logging
from torch.utils.data import DataLoader, RandomSampler
from torch.utils.data.distributed import DistributedSampler
from tqdm import tqdm
class DeepSpeedDataSource(object):
def __init__(self, filenames):
all_lines = []
for filename in filenames:
logging.info("Start reading file %s" % filename)
with open(filename, "r") as f:
for i, line in enumerate(tqdm(f)):
all_lines.append(line.strip())
self.all_lines = all_lines
self.len = len(self.all_lines)
def __len__(self):
return self.len
class DeepSpeedDataLoader(object):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment