Unverified Commit 13fd3dca authored by Shaden Smith's avatar Shaden Smith Committed by GitHub
Browse files

Removes DeepSpeedDataSource (#100)



* Removes DeepSpeedDataSource

* dropping unused imports
Co-authored-by: default avatarJeff Rasley <jerasley@microsoft.com>
parent 6d602065
...@@ -3,26 +3,8 @@ Copyright 2019 The Microsoft DeepSpeed Team ...@@ -3,26 +3,8 @@ Copyright 2019 The Microsoft DeepSpeed Team
''' '''
import torch import torch
import logging
from torch.utils.data import DataLoader, RandomSampler from torch.utils.data import DataLoader, RandomSampler
from torch.utils.data.distributed import DistributedSampler from torch.utils.data.distributed import DistributedSampler
from tqdm import tqdm
class DeepSpeedDataSource(object):
def __init__(self, filenames):
all_lines = []
for filename in filenames:
logging.info("Start reading file %s" % filename)
with open(filename, "r") as f:
for i, line in enumerate(tqdm(f)):
all_lines.append(line.strip())
self.all_lines = all_lines
self.len = len(self.all_lines)
def __len__(self):
return self.len
class DeepSpeedDataLoader(object): class DeepSpeedDataLoader(object):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment