_single()` function
-5. Add new comm op to `run_all.py`
diff --git a/benchmarks/communication/__init__.py b/benchmarks/communication/__init__.py
deleted file mode 100644
index fcb45ab2b68516814a4bfbffebf2e01cbfefd527..0000000000000000000000000000000000000000
--- a/benchmarks/communication/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-'''Copyright The Microsoft DeepSpeed Team'''
diff --git a/benchmarks/communication/all_gather.py b/benchmarks/communication/all_gather.py
deleted file mode 100644
index dc97267b384020e408739017c3b9051434211b34..0000000000000000000000000000000000000000
--- a/benchmarks/communication/all_gather.py
+++ /dev/null
@@ -1,159 +0,0 @@
-'''Copyright The Microsoft DeepSpeed Team'''
-
-from benchmarks.communication.utils import *
-from benchmarks.communication.constants import *
-from deepspeed.accelerator import get_accelerator
-
-import time
-
-
-# Run all_gather and print metrics
-def timed_all_gather(input, output, args):
- if args.dist == 'torch':
- import torch.distributed as dist
- elif args.dist == 'deepspeed':
- import deepspeed.comm as dist
-
- sync_all()
- # Warmups, establish connections, etc.
- for i in range(args.warmups):
- # use all_gather_base if available
- if args.dist == 'torch':
- if hasattr(torch.distributed, "_all_gather_base"):
- dist._all_gather_base(output, input, group=None, async_op=args.async_op)
- else:
- output_tensors = list(
- torch.chunk(output_tensor,
- cdb.get_world_size(group)))
- dist.all_gather(output_tensors, input_tensor, group=group, async_op=True)
- elif args.dist == 'deepspeed':
- dist.allgather_fn(output, input, group=None, async_op=args.async_op)
- sync_all()
-
- # time the actual comm op trials times and average it
- pre = time.perf_counter()
- for i in range(args.trials):
- # use all_gather_base if available
- if args.dist == 'torch':
- if hasattr(torch.distributed, "_all_gather_base"):
- dist._all_gather_base(output, input, group=None, async_op=args.async_op)
- else:
- output_tensors = list(
- torch.chunk(output_tensor,
- cdb.get_world_size(group)))
- dist.all_gather(output_tensors, input_tensor, group=group, async_op=True)
- elif args.dist == 'deepspeed':
- dist.allgather_fn(output, input, group=None, async_op=args.async_op)
- sync_all()
- duration = time.perf_counter() - pre
-
- # maintain and clean performance data
- avg_duration = duration / args.trials
- size = input.element_size() * input.nelement()
- n = dist.get_world_size()
- tput, busbw = get_bw('all_gather', size, avg_duration, args)
- tput_str, busbw_str, duration_str = get_metric_strings(args, tput, busbw, avg_duration)
- desc = f'{input.nelement()}x{input.element_size()}'
-
- if not args.raw:
- size = convert_size(size)
-
- print_rank_0(
- f"{size:<20} {desc:25s} {duration_str:20s} {tput_str:20s} {busbw_str:20s}")
-
-
-def run_all_gather(local_rank, args):
- if args.dist == 'torch':
- import torch.distributed as dist
- elif args.dist == 'deepspeed':
- import deepspeed.comm as dist
-
- # Prepare benchmark header
- print_header(args, 'all_gather')
- global_rank = dist.get_rank()
- world_size = dist.get_world_size()
-
- if args.scan:
- # Create list of message sizes
- M_LIST = []
- for x in (2**p for p in range(1, args.maxsize)):
- M_LIST.append(x)
-
- sync_all()
- # loop over various tensor sizes
- for M in M_LIST:
- global_rank = dist.get_rank()
- try:
- mat = torch.ones(world_size,
- M,
- dtype=getattr(
- torch,
- args.dtype)).to(
- get_accelerator().device_name(local_rank))
- sync_all()
- input = ((mat.mul_(float(global_rank))).view(-1))
- # Delete original mat to avoid OOM
- del mat
- get_accelerator().empty_cache()
- output = torch.zeros(input.nelement() * world_size,
- dtype=getattr(
- torch,
- args.dtype)).to(
- get_accelerator().device_name(local_rank))
- except RuntimeError as e:
- if 'out of memory' in str(e):
- if dist.get_rank() == 0:
- print('WARNING: Ran out of GPU memory. Exiting comm op.')
- sync_all()
- break
- sync_all()
- timed_all_gather(input, output, args)
- else:
- # all_gather_base saves memory
- if (args.dist == 'torch'
- and hasattr(torch.distributed,
- "_all_gather_base")) or (args.dist == 'deepspeed'
- and dist.has_allgather_base):
- mem_factor = args.mem_factor + 0.2
- else:
- mem_factor = args.mem_factor
- # Send the biggest message size our GPUs can fit. If you're facing OOM errors, reduce the mem_factor
- sync_all()
- elements_per_gpu = max_numel(comm_op='all_gather',
- dtype=getattr(torch,
- args.dtype),
- mem_factor=mem_factor,
- local_rank=local_rank,
- args=args)
- try:
- mat = torch.ones(elements_per_gpu,
- dtype=getattr(torch,
- args.dtype)).to(
- get_accelerator().device_name(local_rank))
- # multiply each GPU's tensor by the rank to ease debugging
- input = ((mat.mul_(float(global_rank))).view(-1))
- # Delete original mat to avoid OOM
- del mat
- get_accelerator().empty_cache()
- output = torch.zeros(
- elements_per_gpu * world_size,
- dtype=getattr(torch,
- args.dtype)).to(get_accelerator().device_name(local_rank))
- except RuntimeError as e:
- if 'out of memory' in str(e):
- if dist.get_rank() == 0:
- print(
- 'WARNING: Ran out of GPU memory. Try to reduce the --mem-factor argument!'
- )
- sync_all()
- return
-
- sync_all()
- timed_all_gather(input, output, args)
-
-
-if __name__ == "__main__":
- args = benchmark_parser().parse_args()
- rank = args.local_rank
- init_processes(local_rank=rank, args=args)
- run_all_gather(local_rank=rank, args=args)
diff --git a/benchmarks/communication/all_reduce.py b/benchmarks/communication/all_reduce.py
deleted file mode 100644
index edc1b99301c06e7c8c4b5807e35ad2afa39bf17b..0000000000000000000000000000000000000000
--- a/benchmarks/communication/all_reduce.py
+++ /dev/null
@@ -1,113 +0,0 @@
-'''Copyright The Microsoft DeepSpeed Team'''
-
-from benchmarks.communication.utils import *
-from benchmarks.communication.constants import *
-from deepspeed.accelerator import get_accelerator
-
-import time
-
-
-def timed_all_reduce(input, args):
- if args.dist == 'torch':
- import torch.distributed as dist
- elif args.dist == 'deepspeed':
- import deepspeed.comm as dist
-
- sync_all()
- # Warmups, establish connections, etc.
- for i in range(args.warmups):
- dist.all_reduce(input, async_op=args.async_op)
- sync_all()
-
- # time the actual comm op trials times and average it
- pre = time.perf_counter()
- for i in range(args.trials):
- dist.all_reduce(input, async_op=args.async_op)
- sync_all()
- duration = time.perf_counter() - pre
-
- # maintain and clean performance data
- avg_duration = duration / args.trials
- size = input.element_size() * input.nelement()
- n = dist.get_world_size()
- tput, busbw = get_bw('all_reduce', size, avg_duration, args)
- tput_str, busbw_str, duration_str = get_metric_strings(args, tput, busbw, avg_duration)
- desc = f'{input.nelement()}x{input.element_size()}'
-
- if not args.raw:
- size = convert_size(size)
-
- print_rank_0(
- f"{size:<20} {desc:25s} {duration_str:20s} {tput_str:20s} {busbw_str:20s}")
-
-
-def run_all_reduce(local_rank, args):
- if args.dist == 'torch':
- import torch.distributed as dist
- elif args.dist == 'deepspeed':
- import deepspeed.comm as dist
-
- # Prepare benchmark header
- print_header(args, 'all_reduce')
-
- world_size = dist.get_world_size()
- global_rank = dist.get_rank()
-
- if args.scan:
- M_LIST = []
- for x in (2**p for p in range(1, args.maxsize)):
- M_LIST.append(x)
-
- sync_all()
- # loop over various tensor sizes
- for M in M_LIST:
- global_rank = dist.get_rank()
- try:
- mat = torch.ones(world_size,
- M,
- dtype=getattr(
- torch,
- args.dtype)).to(
- get_accelerator().device_name(local_rank))
- sync_all()
- input = ((mat.mul_(float(global_rank))).view(-1))
- except RuntimeError as e:
- if 'out of memory' in str(e):
- if dist.get_rank() == 0:
- print('WARNING: Ran out of GPU memory. Exiting comm op.')
- sync_all()
- break
- sync_all()
- timed_all_reduce(input, args)
- else:
- # Send the biggest message size our GPUs can fit. If you're facing OOM errors, reduce the mem_factor
- # Don't need output tensor, so we double mem_factor
- elements_per_gpu = max_numel(comm_op='all_reduce',
- dtype=getattr(torch,
- args.dtype),
- mem_factor=args.mem_factor * 2,
- local_rank=local_rank,
- args=args)
- try:
- mat = torch.ones(elements_per_gpu,
- dtype=getattr(torch,
- args.dtype)).to(
- get_accelerator().device_name(local_rank))
- input = ((mat.mul_(float(global_rank))).view(-1))
- except RuntimeError as e:
- if 'out of memory' in str(e):
- if dist.get_rank() == 0:
- print(
- 'WARNING: Ran out of GPU memory. Try to reduce the --mem-factor argument!'
- )
- sync_all()
- return
- sync_all()
- timed_all_reduce(input, args)
-
-
-if __name__ == "__main__":
- args = benchmark_parser().parse_args()
- rank = args.local_rank
- init_processes(local_rank=rank, args=args)
- run_all_reduce(local_rank=rank, args=args)
diff --git a/benchmarks/communication/all_to_all.py b/benchmarks/communication/all_to_all.py
deleted file mode 100644
index bd35cf290e4c0f35b2b2ce4b2c4ea17f876e52ec..0000000000000000000000000000000000000000
--- a/benchmarks/communication/all_to_all.py
+++ /dev/null
@@ -1,134 +0,0 @@
-'''Copyright The Microsoft DeepSpeed Team'''
-
-from benchmarks.communication.utils import *
-from benchmarks.communication.constants import *
-from deepspeed.accelerator import get_accelerator
-
-import time
-
-
-def timed_all_to_all(input, output, args):
- if args.dist == 'torch':
- import torch.distributed as dist
- elif args.dist == 'deepspeed':
- import deepspeed.comm as dist
-
- sync_all()
- # Warmups, establish connections, etc.
- for i in range(args.warmups):
- dist.all_to_all_single(output, input, async_op=args.async_op)
- sync_all()
-
- # time the actual comm op trials times and average it
- pre = time.perf_counter()
- for i in range(args.trials):
- dist.all_to_all_single(output, input, async_op=args.async_op)
- sync_all()
- duration = time.perf_counter() - pre
-
- # maintain and clean performance data
- avg_duration = duration / args.trials
- size = input.element_size() * input.nelement()
- n = dist.get_world_size()
- tput, busbw = get_bw('all_to_all', size, avg_duration, args)
- tput_str, busbw_str, duration_str = get_metric_strings(args, tput, busbw, avg_duration)
- desc = f'{input.nelement()}x{input.element_size()}'
-
- if not args.raw:
- size = convert_size(size)
-
- print_rank_0(
- f"{size:<20} {desc:25s} {duration_str:20s} {tput_str:20s} {busbw_str:20s}")
-
-
-def run_all_to_all(local_rank, args):
- if args.dist == 'torch':
- import torch.distributed as dist
- elif args.dist == 'deepspeed':
- import deepspeed.comm as dist
-
- world_size = dist.get_world_size()
- global_rank = dist.get_rank()
- # Prepare benchmark header
- print_header(args, 'all_to_all')
-
- if args.scan:
- M_LIST = []
- for x in (2**p for p in range(1, args.maxsize)):
- M_LIST.append(x)
-
- sync_all()
- # loop over various tensor sizes
- for M in M_LIST:
- global_rank = dist.get_rank()
- try:
- mat = torch.ones(world_size,
- M,
- dtype=getattr(
- torch,
- args.dtype)).to(
- get_accelerator().device_name(local_rank))
- assert mat.numel() % world_size == 0, f"tensor cannot be divided in {world_size} chunks"
- sync_all()
- input = ((mat.mul_(float(global_rank))).view(-1))
- output = (mat.clone().view(-1))
- except RuntimeError as e:
- if 'out of memory' in str(e):
- if dist.get_rank() == 0:
- print('WARNING: Ran out of GPU memory. Exiting comm op.')
- sync_all()
- break
- sync_all()
- timed_all_to_all(input, output, args)
- else:
- # Send the biggest message size our GPUs can fit. If you're facing OOM errors, reduce the mem_factor
- elements_per_gpu = max_numel(comm_op='all_to_all',
- dtype=getattr(torch,
- args.dtype),
- mem_factor=args.mem_factor,
- local_rank=local_rank,
- args=args)
- try:
- mat = torch.ones(elements_per_gpu,
- dtype=getattr(torch,
- args.dtype)).to(
- get_accelerator().device_name(local_rank))
- assert mat.numel() % world_size == 0, f"tensor with {mat.numel()} elements cannot be divided in {world_size} chunks"
- input = ((mat.mul_(float(global_rank))).view(-1))
- # Delete original mat to avoid OOM
- del mat
- get_accelerator().empty_cache()
- output = torch.zeros(
- elements_per_gpu,
- dtype=getattr(torch,
- args.dtype)).to(get_accelerator().device_name(local_rank))
- except RuntimeError as e:
- if 'out of memory' in str(e):
- if dist.get_rank() == 0:
- print(
- 'WARNING: Ran out of GPU memory. Try to reduce the --mem-factor argument!'
- )
- sync_all()
- return
- sync_all()
-
- if args.debug:
- for i in range(world_size):
- if i == global_rank:
- print(f"Before AllToAll Input List at rank {global_rank}: {input}")
- dist.barrier()
-
- timed_all_to_all(input, output, args)
-
- if args.debug:
- for i in range(world_size):
- if i == global_rank:
- print(f"AllToAll Results at rank {global_rank}: {output}")
- dist.barrier()
-
-
-if __name__ == "__main__":
- args = benchmark_parser().parse_args()
- rank = args.local_rank
- init_processes(local_rank=rank, args=args)
- run_all_to_all(local_rank=rank, args=args)
diff --git a/benchmarks/communication/broadcast.py b/benchmarks/communication/broadcast.py
deleted file mode 100644
index 633e46638fac775920b89b0fc2ab9b0f4401dc79..0000000000000000000000000000000000000000
--- a/benchmarks/communication/broadcast.py
+++ /dev/null
@@ -1,114 +0,0 @@
-'''Copyright The Microsoft DeepSpeed Team'''
-
-import torch
-from benchmarks.communication.utils import *
-from benchmarks.communication.constants import *
-from deepspeed.accelerator import get_accelerator
-
-import time
-
-
-def timed_broadcast(input, args):
- if args.dist == 'torch':
- import torch.distributed as dist
- elif args.dist == 'deepspeed':
- import deepspeed.comm as dist
-
- sync_all()
- # Warmups, establish connections, etc.
- for i in range(args.warmups):
- dist.broadcast(input, 0, async_op=args.async_op)
- sync_all()
-
- # time the actual comm op trials times and average it
- pre = time.perf_counter()
- for i in range(args.trials):
- dist.broadcast(input, 0, async_op=args.async_op)
- sync_all()
- duration = time.perf_counter() - pre
-
- # maintain and clean performance data
- avg_duration = duration / args.trials
- size = input.element_size() * input.nelement()
- n = dist.get_world_size()
- tput, busbw = get_bw('broadcast', size, avg_duration, args)
- tput_str, busbw_str, duration_str = get_metric_strings(args, tput, busbw, avg_duration)
- desc = f'{input.nelement()}x{input.element_size()}'
-
- if not args.raw:
- size = convert_size(size)
-
- print_rank_0(
- f"{size:<20} {desc:25s} {duration_str:20s} {tput_str:20s} {busbw_str:20s}")
-
-
-def run_broadcast(local_rank, args):
- if args.dist == 'torch':
- import torch.distributed as dist
- elif args.dist == 'deepspeed':
- import deepspeed.comm as dist
-
- # Prepare benchmark header
- print_header(args, 'broadcast')
-
- world_size = dist.get_world_size()
- global_rank = dist.get_rank()
-
- if args.scan:
- M_LIST = []
- for x in (2**p for p in range(1, args.maxsize)):
- M_LIST.append(x)
-
- sync_all()
- # loop over various tensor sizes
- for M in M_LIST:
- global_rank = dist.get_rank()
- try:
- mat = torch.ones(world_size,
- M,
- dtype=getattr(
- torch,
- args.dtype)).to(
- get_accelerator().device_name(local_rank))
- sync_all()
- input = ((mat.mul_(float(global_rank))).view(-1))
- except RuntimeError as e:
- if 'out of memory' in str(e):
- if dist.get_rank() == 0:
- print('WARNING: Ran out of GPU memory. Exiting comm op.')
- sync_all()
- break
- sync_all()
- timed_broadcast(input, args)
- else:
- # Send the biggest message size our GPUs can fit. If you're facing OOM errors, reduce the mem_factor
- # Don't need output tensor, so we double mem_factor
- elements_per_gpu = max_numel(comm_op='broadcast',
- dtype=getattr(torch,
- args.dtype),
- mem_factor=args.mem_factor * 2,
- local_rank=local_rank,
- args=args)
- try:
- mat = torch.ones(elements_per_gpu,
- dtype=getattr(torch,
- args.dtype)).to(
- get_accelerator().device_name(local_rank))
- input = ((mat.mul_(float(global_rank))).view(-1))
- except RuntimeError as e:
- if 'out of memory' in str(e):
- if dist.get_rank() == 0:
- print(
- 'WARNING: Ran out of GPU memory. Try to reduce the --mem-factor argument!'
- )
- sync_all()
- return
- sync_all()
- timed_broadcast(input, args)
-
-
-if __name__ == "__main__":
- args = benchmark_parser().parse_args()
- rank = args.local_rank
- init_processes(local_rank=rank, args=args)
- run_broadcast(local_rank=rank, args=args)
diff --git a/benchmarks/communication/constants.py b/benchmarks/communication/constants.py
deleted file mode 100644
index 935927acd174256fe7cd552a3181977e2dfdb7d8..0000000000000000000000000000000000000000
--- a/benchmarks/communication/constants.py
+++ /dev/null
@@ -1,10 +0,0 @@
-'''Copyright The Microsoft DeepSpeed Team'''
-from deepspeed.accelerator import get_accelerator
-
-DEFAULT_WARMUPS = 5
-DEFAULT_TRIALS = 50
-DEFAULT_TYPE = 'float'
-DEFAULT_BACKEND = get_accelerator().communication_backend_name()
-DEFAULT_UNIT = 'Gbps'
-DEFAULT_DIST = 'deepspeed'
-DEFAULT_MAXSIZE = 24
diff --git a/benchmarks/communication/pt2pt.py b/benchmarks/communication/pt2pt.py
deleted file mode 100644
index 1c890fc42e93585a067ef815f163a96d069096ef..0000000000000000000000000000000000000000
--- a/benchmarks/communication/pt2pt.py
+++ /dev/null
@@ -1,132 +0,0 @@
-'''Copyright The Microsoft DeepSpeed Team'''
-
-from benchmarks.communication.utils import *
-from benchmarks.communication.constants import *
-from deepspeed.accelerator import get_accelerator
-
-import time
-
-
-def timed_pt2pt(input, args):
- if args.dist == 'torch':
- import torch.distributed as dist
- elif args.dist == 'deepspeed':
- import deepspeed.comm as dist
-
- sync_all()
- # Warmups, establish connections, etc.
- for i in range(args.warmups):
- if dist.get_rank() == 0:
- if args.async_op:
- dist.isend(input, 1)
- else:
- dist.send(input, 1)
- if dist.get_rank() == 1:
- if args.async_op:
- dist.irecv(input, src=0)
- else:
- dist.recv(input, src=0)
- sync_all()
-
- # time the actual comm op trials times and average it
- pre = time.perf_counter()
- for i in range(args.trials):
- if dist.get_rank() == 0:
- if args.async_op:
- dist.isend(input, 1)
- else:
- dist.send(input, 1)
- if dist.get_rank() == 1:
- if args.async_op:
- dist.irecv(input, src=0)
- else:
- dist.recv(input, src=0)
-
- sync_all()
- duration = time.perf_counter() - pre
-
- # maintain and clean performance data
- avg_duration = duration / args.trials
- size = input.element_size() * input.nelement()
- n = dist.get_world_size()
- tput, busbw = get_bw('pt2pt', size, avg_duration, args)
- tput_str, busbw_str, duration_str = get_metric_strings(args, tput, busbw, avg_duration)
- desc = f'{input.nelement()}x{input.element_size()}'
-
- if not args.raw:
- size = convert_size(size)
-
- print_rank_0(
- f"{size:<20} {desc:25s} {duration_str:20s} {tput_str:20s} {busbw_str:20s}")
-
-
-def run_pt2pt(local_rank, args):
- if args.dist == 'torch':
- import torch.distributed as dist
- elif args.dist == 'deepspeed':
- import deepspeed.comm as dist
-
- # Prepare benchmark header
- print_header(args, 'pt2pt')
- global_rank = dist.get_rank()
- world_size = dist.get_world_size()
-
- if args.scan:
- # Create list of message sizes
- M_LIST = []
- for x in (2**p for p in range(1, args.maxsize)):
- M_LIST.append(x)
-
- sync_all()
- # loop over various tensor sizes
- for M in M_LIST:
- global_rank = dist.get_rank()
- try:
- mat = torch.ones(world_size,
- M,
- dtype=getattr(
- torch,
- args.dtype)).to(
- get_accelerator().device_name(local_rank))
- sync_all()
- input = ((mat.mul_(float(global_rank))).view(-1))
- except RuntimeError as e:
- if 'out of memory' in str(e):
- if dist.get_rank() == 0:
- print('WARNING: Ran out of GPU memory. Exiting comm op.')
- sync_all()
- break
- sync_all()
- timed_pt2pt(input, args)
- else:
- # Send the biggest message size our GPUs can fit. If you're facing OOM errors, reduce the mem_factor
- # Don't need output tensor, so double mem_factor
- elements_per_gpu = max_numel(comm_op='pt2pt',
- dtype=getattr(torch,
- args.dtype),
- mem_factor=args.mem_factor * 2,
- local_rank=local_rank,
- args=args)
- try:
- mat = torch.ones(elements_per_gpu,
- dtype=getattr(torch,
- args.dtype)).to(
- get_accelerator().device_name(local_rank))
- input = ((mat.mul_(float(global_rank))).view(-1))
- except RuntimeError as e:
- if 'out of memory' in str(e):
- if dist.get_rank() == 0:
- print(
- 'WARNING: Ran out of GPU memory. Try to reduce the --mem-factor argument!'
- )
- sync_all()
- return
- sync_all()
- timed_pt2pt(input, args)
-
-
-if __name__ == "__main__":
- args = benchmark_parser().parse_args()
- rank = args.local_rank
- init_processes(local_rank=rank, args=args)
- run_pt2pt(local_rank=rank, args=args)
diff --git a/benchmarks/communication/run_all.py b/benchmarks/communication/run_all.py
deleted file mode 100644
index 7ec562cc9ae0dcc101477117a6158b54a1f4272a..0000000000000000000000000000000000000000
--- a/benchmarks/communication/run_all.py
+++ /dev/null
@@ -1,49 +0,0 @@
-'''Copyright The Microsoft DeepSpeed Team'''
-
-from benchmarks.communication.utils import *
-from benchmarks.communication.all_reduce import run_all_reduce
-from benchmarks.communication.all_gather import run_all_gather
-from benchmarks.communication.all_to_all import run_all_to_all
-from benchmarks.communication.pt2pt import run_pt2pt
-from benchmarks.communication.broadcast import run_broadcast
-from benchmarks.communication.constants import *
-
-
-# For importing
-def main(args, rank):
-
- init_processes(local_rank=rank, args=args)
-
- ops_to_run = []
- if args.all_reduce:
- ops_to_run.append('all_reduce')
- if args.all_gather:
- ops_to_run.append('all_gather')
- if args.broadcast:
- ops_to_run.append('broadcast')
- if args.pt2pt:
- ops_to_run.append('pt2pt')
- if args.all_to_all:
- ops_to_run.append('all_to_all')
-
- if len(ops_to_run) == 0:
- ops_to_run = ['all_reduce', 'all_gather', 'all_to_all', 'broadcast', 'pt2pt']
-
- for comm_op in ops_to_run:
- if comm_op == 'all_reduce':
- run_all_reduce(local_rank=rank, args=args)
- if comm_op == 'all_gather':
- run_all_gather(local_rank=rank, args=args)
- if comm_op == 'all_to_all':
- run_all_to_all(local_rank=rank, args=args)
- if comm_op == 'pt2pt':
- run_pt2pt(local_rank=rank, args=args)
- if comm_op == 'broadcast':
- run_broadcast(local_rank=rank, args=args)
-
-
-# For directly calling benchmark
-if __name__ == "__main__":
- args = benchmark_parser().parse_args()
- rank = args.local_rank
- main(args, rank)
diff --git a/benchmarks/communication/utils.py b/benchmarks/communication/utils.py
deleted file mode 100644
index b913dda14fe552cd7dc2f9fb46d878f5366d4c2a..0000000000000000000000000000000000000000
--- a/benchmarks/communication/utils.py
+++ /dev/null
@@ -1,220 +0,0 @@
-'''Copyright The Microsoft DeepSpeed Team'''
-
-import torch
-import os
-import math
-import argparse
-from benchmarks.communication.constants import *
-from deepspeed.accelerator import get_accelerator
-
-global dist
-
-
-def init_torch_distributed(backend):
- global dist
- import torch.distributed as dist
- torch.distributed.init_process_group(backend)
- local_rank = int(os.environ['LOCAL_RANK'])
- get_accelerator().set_device(local_rank)
-
-
-def init_deepspeed_comm(backend):
- global dist
- import deepspeed
- import deepspeed.comm as dist
- deepspeed.init_distributed(dist_backend=backend)
- local_rank = int(os.environ['LOCAL_RANK'])
- get_accelerator().set_device(local_rank)
-
-
-def init_processes(local_rank, args):
- if args.dist == 'deepspeed':
- init_deepspeed_comm(args.backend)
- elif args.dist == 'torch':
- init_torch_distributed(args.backend)
- else:
- print_rank_0(f"distributed framework {args.dist} not supported")
- exit(0)
-
-
-def print_rank_0(message):
- if dist.get_rank() == 0:
- print(message)
-
-
-def print_header(args, comm_op):
- if comm_op == 'pt2pt':
- world_size = 2
- else:
- world_size = dist.get_world_size()
- tput = f'Throughput ({args.bw_unit})'
- busbw = f'BusBW ({args.bw_unit})'
- header = f"\n---- Performance of {comm_op} on {world_size} devices ---------------------------------------------------------\n"
- duration_str = 'Duration'
- if args.raw:
- duration_str += ' (us)'
- header += f"{'Size (Bytes)':20s} {'Description':25s} {duration_str:20s} {tput:20s} {busbw:20s}\n"
- header += "----------------------------------------------------------------------------------------------------"
- print_rank_0(header)
-
-
-def get_bw(comm_op, size, duration, args):
- n = dist.get_world_size()
- tput = 0
- busbw = 0
- if comm_op == "all_to_all":
- tput = (size / duration)
- busbw = (size / duration) * ((n - 1) / n)
- elif comm_op == "all_gather":
- size *= n
- tput = (size / duration)
- busbw = (size / duration) * ((n - 1) / n)
- elif comm_op == "all_reduce":
- tput = (size * 2 / duration)
- busbw = (size / duration) * (2 * (n - 1) / n)
- elif comm_op == "pt2pt" or comm_op == "broadcast":
- tput = (size / duration)
- busbw = tput
- else:
- print_rank_0("wrong comm_op specified")
- exit(0)
-
- if args.bw_unit == 'Gbps':
- tput *= 8
- busbw *= 8
-
- return tput, busbw
-
-
-def get_metric_strings(args, tput, busbw, duration):
- duration_ms = duration * 1e3
- duration_us = duration * 1e6
- tput = f'{tput / 1e9:.3f}'
- busbw = f'{busbw /1e9:.3f}'
-
- if duration_us < 1e3 or args.raw:
- duration = f'{duration_us:.3f}'
- if not args.raw:
- duration += ' us'
- else:
- duration = f'{duration_ms:.3f} ms'
- return tput, busbw, duration
-
-
-def sync_all():
- get_accelerator().synchronize()
- dist.barrier()
-
-
-def max_numel(comm_op, dtype, mem_factor, local_rank, args):
- dtype_size = _element_size(dtype)
- max_memory_per_gpu = get_accelerator().total_memory(local_rank) * mem_factor
- if comm_op == 'all_reduce' or comm_op == 'pt2pt' or comm_op == 'broadcast':
- elements_per_gpu = int(max_memory_per_gpu // dtype_size)
- elif comm_op == 'all_gather':
- # all_gather performance is lower for non-powers of two, and the output buffer size scales with world size
- # Therefore, divide by world size and round down to nearest power of 2
- elements_per_gpu = int(max_memory_per_gpu // dtype_size // dist.get_world_size())
- elements_per_gpu = int(pow(2, int(math.log(elements_per_gpu, 2))))
- elif comm_op == 'all_to_all':
- # Number of elements must be divisible by world_size
- # all_to_all performance is lower for non-powers of two. Round down like all_gather.
- elements_per_gpu = int(max_memory_per_gpu // dtype_size)
- elements_per_gpu = int(dist.get_world_size() *
- round(elements_per_gpu / dist.get_world_size()))
- elements_per_gpu = int(pow(2, int(math.log(elements_per_gpu, 2))))
- else:
- print(f"This communication operation: {comm_op} is not supported yet")
- exit(0)
- return elements_per_gpu
-
-
-# Helper function to pretty-print message sizes
-def convert_size(size_bytes):
- if size_bytes == 0:
- return "0B"
- size_name = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB")
- i = int(math.floor(math.log(size_bytes, 1024)))
- p = math.pow(1024, i)
- s = round(size_bytes / p, 2)
- return "%s %s" % (s, size_name[i])
-
-
-# Copied from torch. Need to add the func here for old torch compatibility.
-def _element_size(dtype):
- """
- Returns the element size for a dtype, in bytes
- """
- if not isinstance(dtype, torch.dtype):
- raise RuntimeError(f'expected torch.dtype, but got {type(dtype)}')
-
- if dtype.is_complex:
- return torch.finfo(dtype).bits >> 2
- elif dtype.is_floating_point:
- return torch.finfo(dtype).bits >> 3
- elif dtype == torch.bool:
- # NOTE: torch.bool is not supported in torch.iinfo()
- return 1
- else:
- return torch.iinfo(dtype).bits >> 3
-
-
-def benchmark_parser():
- parser = argparse.ArgumentParser()
- parser.add_argument("--local_rank", type=int)
- parser.add_argument("--trials",
- type=int,
- default=DEFAULT_TRIALS,
- help='Number of timed iterations')
- parser.add_argument("--warmups",
- type=int,
- default=DEFAULT_WARMUPS,
- help='Number of warmup (non-timed) iterations')
- parser.add_argument("--maxsize",
- type=int,
- default=24,
- help='Max message size as a power of 2')
- parser.add_argument("--async-op",
- action="store_true",
- help='Enables non-blocking communication')
- parser.add_argument("--bw-unit",
- type=str,
- default=DEFAULT_UNIT,
- choices=['Gbps',
- 'GBps'])
- parser.add_argument("--backend",
- type=str,
- default=DEFAULT_BACKEND,
- choices=['nccl',
- 'ccl'],
- help='Communication library to use')
- parser.add_argument("--dist",
- type=str,
- default=DEFAULT_DIST,
- choices=['deepspeed',
- 'torch'],
- help='Distributed DL framework to use')
- parser.add_argument("--scan",
- action="store_true",
- help='Enables scanning all message sizes')
- parser.add_argument("--raw",
- action="store_true",
- help='Print the message size and latency without units')
- parser.add_argument("--all-reduce", action="store_true", help='Run all_reduce')
- parser.add_argument("--all-gather", action="store_true", help='Run all_gather')
- parser.add_argument("--all-to-all", action="store_true", help='Run all_to_all')
- parser.add_argument("--pt2pt", action="store_true", help='Run pt2pt')
- parser.add_argument("--broadcast", action="store_true", help='Run broadcast')
- parser.add_argument("--dtype",
- type=str,
- default=DEFAULT_TYPE,
- help='PyTorch tensor dtype')
- parser.add_argument(
- "--mem-factor",
- type=float,
- default=.4,
- help='Proportion of max available GPU memory to use for single-size evals')
- parser.add_argument("--debug",
- action="store_true",
- help='Enables all_to_all debug prints')
- return parser
diff --git a/benchmarks/inference/bert-bench.py b/benchmarks/inference/bert-bench.py
deleted file mode 100644
index 9d586d033cd7b375f5e82dfef53445199846630f..0000000000000000000000000000000000000000
--- a/benchmarks/inference/bert-bench.py
+++ /dev/null
@@ -1,92 +0,0 @@
-'''Copyright The Microsoft DeepSpeed Team'''
-
-import torch
-import time
-import deepspeed
-import argparse
-from transformers import pipeline
-from deepspeed.accelerator import get_accelerator
-
-parser = argparse.ArgumentParser()
-parser.add_argument("--model", "-m", type=str, help="hf model name")
-parser.add_argument("--deepspeed", action="store_true", help="use deepspeed inference")
-parser.add_argument("--dtype", type=str, default="fp16", help="fp16 or fp32")
-parser.add_argument("--max-tokens", type=int, default=50, help="max new tokens")
-parser.add_argument("--local_rank", type=int, default=0, help="local rank")
-parser.add_argument("--trials", type=int, default=30, help="number of trials")
-parser.add_argument("--kernel-inject", action="store_true", help="inject kernels on")
-parser.add_argument("--graphs", action="store_true", help="CUDA Graphs on")
-args = parser.parse_args()
-
-
-def print_latency(latency_set, title, warmup=3):
- # trim warmup queries
- latency_set = latency_set[warmup:]
- count = len(latency_set)
- if count > 0:
- latency_set.sort()
- n50 = (count - 1) * 0.5 + 1
- n90 = (count - 1) * 0.9 + 1
- n95 = (count - 1) * 0.95 + 1
- n99 = (count - 1) * 0.99 + 1
- n999 = (count - 1) * 0.999 + 1
-
- avg = sum(latency_set) / count
- p50 = latency_set[int(n50) - 1]
- p90 = latency_set[int(n90) - 1]
- p95 = latency_set[int(n95) - 1]
- p99 = latency_set[int(n99) - 1]
- p999 = latency_set[int(n999) - 1]
-
- print(f"====== latency stats {title} ======")
- print("\tAvg Latency: {0:8.2f} ms".format(avg * 1000))
- print("\tP50 Latency: {0:8.2f} ms".format(p50 * 1000))
- print("\tP90 Latency: {0:8.2f} ms".format(p90 * 1000))
- print("\tP95 Latency: {0:8.2f} ms".format(p95 * 1000))
- print("\tP99 Latency: {0:8.2f} ms".format(p99 * 1000))
- print("\t999 Latency: {0:8.2f} ms".format(p999 * 1000))
-
-
-deepspeed.init_distributed()
-
-print(args.model, args.max_tokens, args.dtype)
-
-if args.dtype.lower() == "fp16":
- dtype = torch.float16
-else:
- dtype = torch.float32
-
-pipe = pipeline("fill-mask", model=args.model, framework="pt", device=args.local_rank)
-
-if dtype == torch.half:
- pipe.model.half()
-
-mask = pipe.tokenizer.mask_token
-
-br = pipe(f"Hello I'm a {mask} model")
-if args.deepspeed:
- pipe.model = deepspeed.init_inference(pipe.model,
- dtype=dtype,
- mp_size=1,
- replace_with_kernel_inject=args.kernel_inject,
- enable_cuda_graph=args.graphs)
- pipe.model.profile_model_time()
-
-responses = []
-times = []
-mtimes = []
-for i in range(args.trials):
- get_accelerator().synchronize()
- start = time.time()
- r = pipe(f"Hello I'm a {mask} model")
- get_accelerator().synchronize()
- end = time.time()
- responses.append(r)
- times.append((end - start))
- mtimes += pipe.model.model_times()
- #print(f"{pipe.model.model_times()=}")
-
-print_latency(times, "e2e latency")
-print_latency(mtimes, "model latency")
-
-print(responses[0:3])
diff --git a/benchmarks/inference/collect_results.py b/benchmarks/inference/collect_results.py
deleted file mode 100644
index 0e51033114db848d2d2ff14b2f33b009a2090672..0000000000000000000000000000000000000000
--- a/benchmarks/inference/collect_results.py
+++ /dev/null
@@ -1,147 +0,0 @@
-'''Copyright The Microsoft DeepSpeed Team'''
-
-import os
-import re
-import argparse
-import pandas as pd
-
-parser = argparse.ArgumentParser()
-parser.add_argument(
- "--results-dir",
- "-r",
- type=str,
- default="./results",
- help="directory containing sweep results",
-)
-parser.add_argument("--version",
- "-v",
- type=int,
- default=0,
- help="version to be collected")
-parser.add_argument("--gen-text-n",
- "-n",
- type=int,
- default=1,
- help="expected number of generated text")
-parser.add_argument("--output",
- "-o",
- type=str,
- default="./results.csv",
- help="output file")
-args = parser.parse_args()
-
-
-def get_branch(file_path):
- match = re.match(r".*\/(.*)\.log", file_path)
- if match is None:
- return False
- else:
- return match.groups()[0]
-
-
-def get_benchmark_params(root_dir, file_path):
- match = re.match(
- rf"{root_dir}\/(.+?)_(fp\d+)_(true|false)_(true|false)_(\d+)gpus_v(\d+)\/",
- file_path,
- )
- if match is None:
- return False
- else:
- model, dtype, graphs, kernel, gpus, version = match.groups()
- bool_dict = {"true": True, "false": False}
- return {
- "model": model,
- "dtype": dtype,
- "graphs": bool_dict[graphs.lower()],
- "kernel": bool_dict[kernel.lower()],
- "gpus": int(gpus),
- "version": int(version),
- }
-
-
-def get_perf_data(file_content):
- matches = re.findall(r"\s+(.+?)\sLatency:\s+(\d+\.\d+)\sms", file_content)
- if matches is []:
- return False
- else:
- return {f"latency-{key}": float(val) for key, val in matches}
-
-
-def get_generated_text(file_content, gen_text_n):
- file_content = file_content.replace("\n", " ")
- file_content = file_content.replace("\t", " ")
- matches = re.findall(r"RESPONSE\s(\d+):\s+[-]{30}\s+(.+?)\s+[-]{30}", file_content)
- if len(matches) != gen_text_n:
- return False
- else:
- return {f"generated-text-{key}": val for key, val in matches}
-
-
-def get_error(file_content):
- matches = re.findall(r"Error:\s+(.+?)\n", file_content)
- if matches is []:
- return False
- else:
- return {f"error": val for val in matches}
-
-
-if __name__ == "__main__":
- # List to collect data from all benchmarks
- benchmarks_data = []
-
- # Walk through directory of results from sweep.sh
- for root, dirs, files in os.walk(args.results_dir):
- # Because of how some models are named, the dir structure for results can vary, e.g.:
- # "EleutherAI/gpt-neo_*/baseline.log" versus "gpt2_*/baseline.log"
- if dirs:
- continue
-
- # Get data from baseline and each tested branch
- for name in files:
- file_path = os.path.join(root, name)
-
- branch = get_branch(file_path)
- if not branch:
- print(f"WARNING: Could not detect branch for file {file_path}, skipping")
- continue
-
- params = get_benchmark_params(args.results_dir, file_path)
- if not params:
- print(
- f"WARNING: Could not detect benchmark settings for file {file_path}, skipping"
- )
- continue
-
- # Verify that the version matches that which we want to collect
- if params["version"] != args.version:
- continue
-
- with open(file_path, "r") as f:
- file_content = f.read()
-
- perf_data = get_perf_data(file_content)
- if not perf_data:
- print(
- f"WARNING: Could not detect benchmark performance data for file {file_path}"
- )
-
- generated_text = get_generated_text(file_content, args.gen_text_n)
- if not generated_text:
- print(f"WARNING: Could not detect generated text for file {file_path}")
-
- error = get_error(file_content)
- if error:
- print(f"Error found in {file_path}, collecting error info...")
- benchmarks_data.append({"branch": branch, **params, **error})
- continue
-
- benchmarks_data.append({
- "branch": branch,
- **params,
- **perf_data,
- **generated_text
- })
-
- # Convert to a DataFrame and save
- benchmarks_df = pd.DataFrame(benchmarks_data)
- benchmarks_df.to_csv(args.output)
diff --git a/benchmarks/inference/gpt-bench.py b/benchmarks/inference/gpt-bench.py
deleted file mode 100644
index 29578b30cf1faf1638bbd0a84865ee9c283e9443..0000000000000000000000000000000000000000
--- a/benchmarks/inference/gpt-bench.py
+++ /dev/null
@@ -1,124 +0,0 @@
-'''Copyright The Microsoft DeepSpeed Team'''
-
-import os
-import torch
-import time
-import deepspeed
-import argparse
-from transformers import pipeline
-from deepspeed.accelerator import get_accelerator
-
-parser = argparse.ArgumentParser()
-parser.add_argument("--model", "-m", type=str, help="hf model name")
-parser.add_argument("--deepspeed", action="store_true", help="use deepspeed inference")
-parser.add_argument("--dtype",
- type=str,
- default="fp16",
- choices=["fp16",
- "fp32",
- "int8"],
- help="int8, fp16, or fp32")
-parser.add_argument("--graphs", action="store_true", help="CUDA Graphs on")
-parser.add_argument("--kernel-inject", action="store_true", help="inject kernels on")
-parser.add_argument("--max-tokens", type=int, default=50, help="max new tokens")
-parser.add_argument("--local_rank",
- type=int,
- default=int(os.getenv("LOCAL_RANK",
- "0")),
- help="local rank")
-parser.add_argument("--world_size",
- type=int,
- default=int(os.getenv("WORLD_SIZE",
- "1")),
- help="world size")
-parser.add_argument("--trials", type=int, default=30, help="number of trials")
-args = parser.parse_args()
-
-
-def print_latency(latency_set, title, warmup=3):
- # trim warmup queries
- latency_set = list(latency_set)
- latency_set = latency_set[warmup:]
- count = len(latency_set)
- if count > 0:
- latency_set.sort()
- n50 = (count - 1) * 0.5 + 1
- n90 = (count - 1) * 0.9 + 1
- n95 = (count - 1) * 0.95 + 1
- n99 = (count - 1) * 0.99 + 1
- n999 = (count - 1) * 0.999 + 1
-
- avg = sum(latency_set) / count
- p50 = latency_set[int(n50) - 1]
- p90 = latency_set[int(n90) - 1]
- p95 = latency_set[int(n95) - 1]
- p99 = latency_set[int(n99) - 1]
- p999 = latency_set[int(n999) - 1]
-
- print(f"====== latency stats {title} ======")
- print("\tAvg Latency: {0:8.2f} ms".format(avg * 1000))
- print("\tP50 Latency: {0:8.2f} ms".format(p50 * 1000))
- print("\tP90 Latency: {0:8.2f} ms".format(p90 * 1000))
- print("\tP95 Latency: {0:8.2f} ms".format(p95 * 1000))
- print("\tP99 Latency: {0:8.2f} ms".format(p99 * 1000))
- print("\t999 Latency: {0:8.2f} ms".format(p999 * 1000))
-
-
-deepspeed.init_distributed()
-
-if args.local_rank == 0:
- print("BENCHMARK SETTINGS:")
- print(f"\tMODEL: {args.model}")
- print(f"\tMAX_TOKENS: {args.max_tokens}")
- print(f"\tDTYPE: {args.dtype}")
- print(f"\tCUDA_GRAPHS: {args.graphs}")
- print(f"\tKERNEL_INJECT: {args.kernel_inject}")
-
-if args.dtype == "int8":
- dtype = torch.int8
-elif args.dtype == "fp16":
- dtype = torch.float16
-else:
- dtype = torch.float32
-
-pipe = pipeline("text-generation",
- model=args.model,
- framework="pt",
- device=args.local_rank)
-
-if dtype == torch.float16:
- pipe.model.half()
-
-if args.deepspeed:
- pipe.model = deepspeed.init_inference(
- pipe.model,
- dtype=dtype,
- mp_size=args.world_size,
- replace_with_kernel_inject=args.kernel_inject,
- enable_cuda_graph=args.graphs,
- )
- pipe.model.profile_model_time()
-
-responses = []
-times = []
-mtimes = []
-for i in range(args.trials):
- get_accelerator().synchronize()
- start = time.time()
- r = pipe("DeepSpeed is", do_sample=False, max_new_tokens=args.max_tokens)
- get_accelerator().synchronize()
- end = time.time()
- responses.append(r)
- times.append(end - start) # / (args.max_tokens - 3))
- mtimes.append(sum(pipe.model.model_times()))
-
-if args.local_rank == 0:
- print_latency(times, "(e2e) latency")
- print_latency(mtimes, "(model-only) latency")
- print_latency(map(lambda t: t / (args.max_tokens - 3),
- times),
- "(e2e) per token latency")
- print(f"RESPONSE 0:")
- print("-" * 30)
- print(responses[0][0]["generated_text"])
- print("-" * 30)
diff --git a/benchmarks/inference/requirements.txt b/benchmarks/inference/requirements.txt
deleted file mode 100644
index 00899dd5f4858229e4115fd2b80b7807636892bd..0000000000000000000000000000000000000000
--- a/benchmarks/inference/requirements.txt
+++ /dev/null
@@ -1 +0,0 @@
-transformers>=4.21.3
diff --git a/benchmarks/inference/run_model.sh b/benchmarks/inference/run_model.sh
deleted file mode 100644
index 8e5fe3ac0133150a5de05f76da4951a2ead6be58..0000000000000000000000000000000000000000
--- a/benchmarks/inference/run_model.sh
+++ /dev/null
@@ -1,36 +0,0 @@
-set -x
-
-model=$1
-branch1=$2
-branch2=$3
-dtype=$4
-graphs=$5
-kernel=$6
-gpus=$7
-
-version=0
-log_path=results/${model}_${dtype}_${graphs}_${kernel}_${gpus}gpus_v${version}
-mkdir -p ${log_path}
-
-params="--dtype $dtype "
-if [[ "$graphs" == "true" ]]; then
- params+="--graphs "
-fi
-if [[ "$kernel" == "true" ]]; then
- params+="--kernel "
-fi
-
-echo "baseline $log_path"
-deepspeed --num_gpus 1 gpt-bench.py -m "${model}" $params &> ${log_path}/baseline.log
-
-cd ../../
-git checkout ${branch1}
-cd -
-echo "ds ${branch1} $log_path"
-deepspeed --num_gpus $gpus gpt-bench.py --deepspeed -m "${model}" $params &> ${log_path}/ds-${branch1}.log
-
-cd ../../
-git checkout ${branch2}
-cd -
-echo "ds ${branch2} $log_path"
-deepspeed --num_gpus $gpus gpt-bench.py --deepspeed -m "${model}" $params&> ${log_path}/ds-${branch2}.log
diff --git a/benchmarks/inference/sweep.sh b/benchmarks/inference/sweep.sh
deleted file mode 100644
index aabcb0bfdbd89e2eedd97d3e6afa74c0e50e7803..0000000000000000000000000000000000000000
--- a/benchmarks/inference/sweep.sh
+++ /dev/null
@@ -1,41 +0,0 @@
-set -x
-
-export TRANSFORMERS_CACHE=/tmp/hf-cache
-
-branch1=$1
-branch2=$2
-
-gptneo_models="EleutherAI/gpt-neo-2.7B EleutherAI/gpt-neo-1.3B EleutherAI/gpt-neo-125M"
-gpt2_models="gpt2 gpt2-large gpt2-xl"
-gptj_models="EleutherAI/gpt-j-6B"
-opt_models="facebook/opt-125m facebook/opt-1.3b facebook/opt-2.7b facebook/opt-6.7b facebook/opt-13b"
-bloom_models="bigscience/bloom-560m bigscience/bloom-1b7 bigscience/bloom-3b bigscience/bloom-7b1"
-
-for gpus in `echo "1 2 4 8"`; do
- for dtype in `echo "fp16 fp32"`; do
- for graphs in `echo "true false"`; do
- for kernel in `echo "true false"`; do
- params="$dtype $graphs $kernel $gpus"
- for m in `echo "$gptneo_models"`; do
- bash run_model.sh $m $branch1 $branch2 $params
- done
-
- for m in `echo "$gpt2_models"`; do
- bash run_model.sh $m $branch1 $branch2 $params
- done
-
- for m in `echo "$gptj_models"`; do
- bash run_model.sh $m $branch1 $branch2 $params
- done
-
- for m in `echo "$opt_models"`; do
- bash run_model.sh $m $branch1 $branch2 $params
- done
-
- for m in `echo "$bloom_models"`; do
- bash run_model.sh $m $branch1 $branch2 $params
- done
- done
- done
- done
-done
diff --git a/bin/ds_elastic b/bin/ds_elastic
index c9987d4565da3cb4c7e32b8342c201ba0165e030..1c78aea888946756d2f586f7be604c8140b5d09c 100755
--- a/bin/ds_elastic
+++ b/bin/ds_elastic
@@ -9,11 +9,7 @@ from deepspeed.elasticity import compute_elastic_config
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('-c', '--config', type=str, help="DeepSpeed config json")
- parser.add_argument('-w',
- '--world-size',
- type=int,
- default=0,
- help="Intended/current world size")
+ parser.add_argument('-w', '--world-size', type=int, default=0, help="Intended/current world size")
args = parser.parse_args()
ds_config = json.load(open(args.config, 'r'))
@@ -26,7 +22,9 @@ if __name__ == '__main__':
print(json.dumps(elastic_config, indent=4, sort_keys=True))
if args.world_size > 0:
- final_batch_size, valid_gpus, micro_batch_size = compute_elastic_config(ds_config=ds_config, target_deepspeed_version=ds_version, world_size=args.world_size)
+ final_batch_size, valid_gpus, micro_batch_size = compute_elastic_config(ds_config=ds_config,
+ target_deepspeed_version=ds_version,
+ world_size=args.world_size)
print('------------------------------------------')
print(f"Calculated results for world size {args.world_size}:")
print('------------------------------------------')
diff --git a/blogs/README.md b/blogs/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..c5c72fd4efb021390295788a3a19cf4710d9d68c
--- /dev/null
+++ b/blogs/README.md
@@ -0,0 +1 @@
+All DeepSpeed blogs are linked here:
diff --git a/blogs/assets/images/Throughputs-OPT.png b/blogs/assets/images/Throughputs-OPT.png
new file mode 100755
index 0000000000000000000000000000000000000000..9b968319f759d161ca6bbee7cc3524f58fcce933
Binary files /dev/null and b/blogs/assets/images/Throughputs-OPT.png differ
diff --git a/blogs/assets/images/ds-chat-overview.png b/blogs/assets/images/ds-chat-overview.png
new file mode 100755
index 0000000000000000000000000000000000000000..479ca4bb2ad72319dc293340ca9dce9bf59085c8
Binary files /dev/null and b/blogs/assets/images/ds-chat-overview.png differ
diff --git a/blogs/assets/images/figure3.png b/blogs/assets/images/figure3.png
new file mode 100755
index 0000000000000000000000000000000000000000..44b8e67bd6b950ad072443b7969c82aed90267f6
Binary files /dev/null and b/blogs/assets/images/figure3.png differ
diff --git a/blogs/assets/images/figure4.png b/blogs/assets/images/figure4.png
new file mode 100755
index 0000000000000000000000000000000000000000..dca56637049cd80759bc39798bb8fa9cbf1d0921
Binary files /dev/null and b/blogs/assets/images/figure4.png differ
diff --git a/blogs/assets/images/figure5.png b/blogs/assets/images/figure5.png
new file mode 100755
index 0000000000000000000000000000000000000000..6282c0d19ed1841bcaf1b86e335db806bdcf484c
Binary files /dev/null and b/blogs/assets/images/figure5.png differ
diff --git a/blogs/assets/images/figure6.png b/blogs/assets/images/figure6.png
new file mode 100755
index 0000000000000000000000000000000000000000..8e60773b5709bd39ac400e1a1c68f827207e48d8
Binary files /dev/null and b/blogs/assets/images/figure6.png differ
diff --git a/blogs/assets/images/figure7.png b/blogs/assets/images/figure7.png
new file mode 100755
index 0000000000000000000000000000000000000000..9b400b11efb0d38c3d3241ac9a8615934a9bb7eb
Binary files /dev/null and b/blogs/assets/images/figure7.png differ
diff --git a/blogs/assets/images/hero-figure-black-ja.png b/blogs/assets/images/hero-figure-black-ja.png
new file mode 100644
index 0000000000000000000000000000000000000000..5c6cf05d3e7a2d47c2d0dfb7ac25653024bcf8b4
Binary files /dev/null and b/blogs/assets/images/hero-figure-black-ja.png differ
diff --git a/blogs/assets/images/hero-figure-black.png b/blogs/assets/images/hero-figure-black.png
new file mode 100755
index 0000000000000000000000000000000000000000..6a05f4b27bb9532aed6eae373488aa67825bc069
Binary files /dev/null and b/blogs/assets/images/hero-figure-black.png differ
diff --git a/blogs/assets/images/hybrid-engine.png b/blogs/assets/images/hybrid-engine.png
new file mode 100755
index 0000000000000000000000000000000000000000..5e501108a5b7a29b137ec231f5bf83c51b3f0480
Binary files /dev/null and b/blogs/assets/images/hybrid-engine.png differ
diff --git a/blogs/assets/images/mascot.png b/blogs/assets/images/mascot.png
new file mode 100755
index 0000000000000000000000000000000000000000..e9f7a354fc85ede378c7007692fc5ae6e0258636
Binary files /dev/null and b/blogs/assets/images/mascot.png differ
diff --git a/blogs/deepspeed-chat/README.md b/blogs/deepspeed-chat/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..de3b886e88d7961d1ff95533da07b8777b3daf7b
--- /dev/null
+++ b/blogs/deepspeed-chat/README.md
@@ -0,0 +1,321 @@
+
+
+# DeepSpeed Chat: Easy, Fast and Affordable RLHF Training of ChatGPT-like Models at All Scales
+
+
+
+
+
+
+
+
+
+| GPUs | OPT-6.7B | OPT-13B | OPT-30B | OPT-66B |
+|-------------|:--------:|:--------------:|:-------------:|:-----------:|
+| 8x A100-40GB | 5.7 hours | 10.8 hours | 1.85 days | NA |
+| 8x A100-80GB | 4.1 hours ($132) | 9 hours ($290) | 18 hours ($580) | 2.1 days ($1620) |
+
+*Table 1. Single-Node 8x A100: Training Time and Corresponding Approximate Cost on Azure.*
+
+
+
+
+***Excellent Scalability***: DeepSpeed-HE supports models with hundreds of billions of parameters and can achieve excellent scalability on multi-node multi-GPU systems. As a result, even a 13B model can be trained in 1.25 hours and a massive 175B model can be trained with DeepSpeed-HE in under a day.
+
+
+
+
+| GPUs | OPT-13B | OPT-30B | OPT-66B | OPT-175B |
+|---------------|:-----------------:|:---------------:|:-------------:|:-------------:|
+| 64x A100-80G | 1.25 hours ($320) | 4 hours ($1024) | 7.5 hours ($1920) | 20 hours ($5120)|
+
+*Table 2. Multi-Node 64x A100-80GB: Training Time and Corresponding Approximate Cost on Azure.*
+
+
+> ***Very Important Details***: The numbers in both tables above are for Step 3 of the training and based on actual measured training throughput on DeepSpeed-RLHF curated dataset and training recipe which trains for one epoch on a total of 135M tokens. We have in total 67.5M query tokens (131.9k queries with sequence length 256) and 67.5M generated tokens (131.9k answers with sequence length 256), and a maximum global batch size per step of 0.5M tokens (1024 query-answer pairs). We urge readers to pay attention to these specifications before making any cost and e2e time comparisons with DeepSpeed-RLHF. See our [benchmark settings](https://github.com/microsoft/DeepSpeedExamples/tree/master/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/BenckmarkSetting.md) page for more details.
+
+
+***Democratizing RLHF Training***: With just a single GPU, DeepSpeed-HE supports training models with over 13 billion parameters, enabling data scientists without access to multi-GPU systems to create not just toy RLHF models but large and powerful ones that can be used in real-world scenarios.
+
+
+
+| | V100 32G | A6000 48G | A100 40G | A100 80G |
+|------------|:---------:|:----------:|:--------:|:---------:|
+| Model Size | OPT-2.7B | OPT-6.7B | OPT-6.7B | OPT-13B |
+
+Table 3. Max Model Size Supported by DeepSpeed-HE on a Single GPU
+
+
+
+Next, we dive deeper into the three capabilities of DeepSpeed-Chat introduced above.
+
+# 2. Easy-to-use ChatGPT Training and Inference Experience
+
+We start with the easy-to-use experience by showing how you can train OPT-13B and then OPT-66B models with DeepSpeed-RLHF system. If you are short on time, you can even train an OPT-1.3B model on a single consumer-grade GPU in just two hours. We also demonstrate how you can use our DeepSpeed-chat RLHF API to develop your own custom pipelines.
+
+## Training your first ChatGPT-Style model is so easy with DeepSpeed-Chat’s RLHF examples
+
+*a) One single script completes all three stages of RLHF training and generates your first ChatGPT model!*
+
+We use an example of pretrained OPT-13B as the actor model and OPT-350M as the reward model in the following single script to generate a final 13B ChatGPT-style model:
+
+```python
+pip install deepspeed>=0.9.0
+
+git clone https://github.com/microsoft/DeepSpeedExamples.git
+cd DeepSpeedExamples/applications/DeepSpeed-Chat/
+pip install -r requirements.txt
+
+python train.py --actor-model facebook/opt-13b --reward-model facebook/opt-350m --deployment-type single_node
+```
+
+In about half a day, your 13B model would be fully trained with its checkpoints ready. The following table demonstrates a breakdown of the training time for each of the three steps:
+
+
+
+| Model Sizes | Step 1 | Step 2 | Step 3 | Total |
+|--------------------------------- |:------:|:------:|:------:|:------:|
+| Actor: OPT-13B, Reward: OPT-350M | 2.5hr | 0.25hr | 10.8hr | 13.6hr |
+
+Table 4. E2E time breakdown for training a 13 billion parameter ChatGPT model via DeepSpeed-Chat on a single DGX node with 8 NVIDIA A100-40G GPUs.
+
+
+
+*b) Test your final model with DeepSpeed-Chat’s Inference API*
+
+Now it is time to plug-in and test your final model trained by DeepSpeed-Chat! You can play with not only Q&A sessions but also conversation-style interactions.
+
+```
+Human: Do you know Microsoft?
+Assistant: Yes, I’m a big fan of Microsoft. What about them?
+Human: Can you explain it to a 6-year old child? I wonder how I should describe it
+Assistant: Sure, I can try. Microsoft is a company that makes computers, and they make a program called “Windows” which
+ is the operating system that runs on the computer. It’s like the “brain” of the computer. It’s where all the
+ programs and files are stored. It’s also where you’ll find the “start” button, which is where you’ll start to
+ use the computer.
+```
+
+## Want to try different model sizes and configurations? You got it!
+We understand users often like to play with different model sizes and configurations to meet their training time, resources, and quality requirements. With DeepSpeed-Chat, users can easily do that. For example, if you want to train a larger and higher-quality model on your GPU cluster for your research or business, you can simply use the same script with your desired model size e.g., 66B and GPU counts e.g., 64 GPUs:
+
+```python
+python train.py --actor-model facebook/opt-66b --reward-model facebook/opt-350m --deployment-type multi_node
+```
+
+Within 9 hours, you can have your 66 billion parameters ChatGPT model ready to be served in your favorite front-end GUI:
+
+
+
+
+| Model Sizes | Step 1 | Step 2 | Step 3 | Total |
+|---------------------------------- |:-------:|:------:|:------:|:------:|
+| Actor: OPT-66B, Reward: OPT-350M | 82 mins | 5 mins | 7.5hr | 9hr |
+
+Table 5. E2E time breakdown for training a 66 billion parameter ChatGPT model via DeepSpeed-Chat on 8 DGX nodes with 8 NVIDIA A100-80G GPUs/node.
+
+
+
+If you only have around 1-2 hours for coffee or lunch break, you can also try to train a small/toy model with DeepSpeed-Chat. For example, we prepared a training example for a 1.3B model with a single dataset to test our framework on your consumer-grade GPUs. The best part is that you will have your model checkpoint ready to play with when you are back from your lunch break!
+
+```python
+python train.py --actor-model facebook/opt-1.3b --reward-model facebook/opt-350m --deployment-type single_gpu
+```
+
+
+
+| Model Sizes | Step 1 | Step 2 | Step 3 | Total |
+|--------------------------------- |:---------:|:--------:|:------:|:------:|
+| Actor: OPT-1.3B, Reward: OPT-350M | 2900 secs | 670 secs | 1.2hr | 2.2hr |
+
+*Table 6. E2E time breakdown for training a 1.3 billion parameter ChatGPT model via DeepSpeed-Chat on a single commodity NVIDIA A6000 GPU with 48GB memory.*
+
+
+
+## Customizing your own RLHF training pipeline using DeepSpeed-Chat’s RLHF APIs
+DeepSpeed-Chat allows users to build their very own RLHF training pipeline using our flexible APIs shown below, which users can use to reconstruct their own RLHF training strategy. This enables a general interface and backend for creating a wide range of RLHF algorithms for research exploration.
+
+```python
+
+engine = DeepSpeedRLHFEngine(
+ actor_model_name_or_path=args.actor_model_name_or_path,
+ critic_model_name_or_path=args.critic_model_name_or_path,
+ tokenizer=tokenizer,
+ num_total_iters=num_total_iters,
+ args=args)
+
+trainer = DeepSpeedPPOTrainer(engine=engine, args=args)
+
+for prompt_batch in prompt_train_dataloader:
+ out = trainer.generate_experience(prompt_batch)
+ actor_loss, critic_loss = trainer.train_rlhf(out)
+```
+
+# 3. Full-fledged RLHF Training Pipeline
+
+To provide a seamless training experience, we follow InstructGPT and include a full-fledged end-to-end training pipeline in DeepSpeed-Chat as shown in Figure 1.
+
+
+
+
+
+*Figure 1: The illustration of DeepSpeed-Chat’s RLHF training pipeline with optional features.*
+
+
+
+
+
+*Figure 2. DeepSpeed Hybrid Engine design for accelerating the most time-consuming portion of a RLHF pipeline.*
+
+
+
+
+
+*Figure 3. Step 3 throughput comparison against two other system frameworks for accelerating RLHF \
+training on a single NVIDIA A100-40G commodity GPU. No icons represent OOM scenarios.*
+
+
+
+
+
+*Figure 4. End-to-end training throughput comparison for step 3 of the training pipeline (the most time \
+consuming portion) with different model sizes on a single DGX node equipped with 8 NVIDIA A100-40G GPUs.\
+No icons represent OOM scenarios.*
+
+
+
+
+
+*Figure 5. Superior generation phase acceleration from DeepSpeed Chat’s Hybrid Engine: A time/sequence breakdown for training OPT-1.3B actor model + OPT-350M reward model on a single DGX node with 8 A100-40G GPUs.*
+
+
+
+
+
+*Figure 6. RLHF Generation, training, and effective throughput with DeepSpeed-HE for different model sizes, at the GPU count that maximizes efficiency.*
+
+
+
+
+
+*Figure 7. Scalability for training 13B (left) and 66B (right) actor model+350M reward model on an increasing number of DGX nodes with 8 A100-40/80G GPUs*
+
+
+
+# DeepSpeed Chat: 一键式RLHF训练，让你的类ChatGPT千亿大模型提速省钱15倍
+
+
+
+
+
+
+
+
+
+| GPUs | OPT-6.7B | OPT-13B | OPT-30B | OPT-66B |
+|-------------|:--------:|:--------------:|:-------------:|:-----------:|
+| 8x A100-40GB | 5.7 hours | 10.8 hours | 1.85 days | NA |
+| 8x A100-80GB | 4.1 hours ($132) | 9 hours ($290) | 18 hours ($580) | 2.1 days ($1620) |
+
+*表 1. 单节点 8x A100：训练时长及预估的 Azure 费用。*
+
+
+
+***卓越的扩展性***：DeepSpeed-HE 能够支持训练拥有数千亿参数的模型，并在多节点多 GPU 系统上展现出卓越的扩展性。因此，即使是一个拥有 130 亿参数的模型，也只需 1.25 小时就能完成训练。而对于庞大的 拥有1750 亿参数的模型，使用 DeepSpeed-HE 进行训练也只需不到一天的时间。
+
+
+
+| GPUs | OPT-13B | OPT-30B | OPT-66B | OPT-175B |
+|---------------|:-----------------:|:---------------:|:-------------:|:-------------:|
+| 64x A100-80G | 1.25 hours ($320) | 4 hours ($1024) | 7.5 hours ($1920) | 20 hours ($5120)|
+
+*表 2. 多节点 64x A100-80GB：训练时长及预估的 Azure 费用。*
+
+
+> ***非常重要的细节***: 上述两个表格（即表一和表二）中的数据均针对 RLHF 训练的第 3 步，基于实际数据集和 DeepSpeed-RLHF 训练吞吐量的测试。该训练在总共 1.35 亿（135M）个字符（token）上进行一个时期（epoch）的训练。我们总共有 6750 万个查询（query）字符（131.9k 个 query，每个序列长度为 256）和 6750 万个生成/回答字符（131.9k 个答案，每个序列长度为 256），每步的最大全局字符批量大小约为 500 万个字符（1024 个查询-答案对）。在与 DeepSpeed-RLHF 进行任何成本和端到端时间比较之前，我们建议读者注意这些设定。想要了解更多详细信息，请参阅我们的页面 [benchmark setting](https://github.com/microsoft/DeepSpeedExamples-internal/blob/staging-deepspeed-chat-v2/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/BenckmarkSetting.md)。
+
+***实现 RLHF 训练的普及化***：仅凭单个 GPU，DeepSpeed-HE 就能支持训练超过 130 亿参数的模型。这使得那些无法使用多 GPU 系统的数据科学家和研究者不仅能够轻松创建轻量级的 RLHF 模型，还能创建大型且功能强大的模型，以应对不同的使用场景。
+
+
+
+| | V100 32G | A6000 48G | A100 40G | A100 80G |
+|------------|:---------:|:----------:|:--------:|:---------:|
+| Model Size | OPT-2.7B | OPT-6.7B | OPT-6.7B | OPT-13B |
+
+*表 3. DeepSpeed-HE 在不同的 GPU 单卡上支持的最大模型。*
+
+
+
+接下来，我们将深入了解上面介绍的 DeepSpeed-Chat 的三个功能。
+
+# 2. 简洁高效且经济的 ChatGPT 训练与推理体验
+
+接下来，我们向你展示如何使用一行命令即可利用 DeepSpeed-RLHF 系统训练 OPT-13B 和 OPT-66B 模型。即使时间有限，你也可以在大约两小时内在单个消费级 GPU 上训练一个 OPT-1.3B 模型。此外，我们还演示了如何利用我们的 DeepSpeed-chat RLHF API 来开发你自己的模型：
+
+## 使用 DeepSpeed-Chat 的 RLHF 示例轻松训练你的第一个 类ChatGPT 模型
+
+*a) 仅需一个脚本，完成 RLHF 训练的全部三个阶段，生成你的第一个 ChatGPT 模型！*
+
+以下示例展示了如何使用一个脚本，以预训练的 OPT-13B 作为 actor 模型，OPT-350M 作为 reward 模型，生成一个最终的 130 亿参数的 ChatGPT 类型的模型：
+```
+pip install deepspeed>=0.9.0
+
+git clone https://github.com/microsoft/DeepSpeedExamples.git
+cd DeepSpeedExamples/applications/DeepSpeed-Chat/
+pip install -r requirements.txt
+
+python train.py --actor-model facebook/opt-13b --reward-model facebook/opt-350m --deployment-type single_node
+```
+
+
+大约半天时间，你的 拥有130 亿参数的模型即可完成训练。下表展示了整个过程在三个步骤中所需的时间：
+
+
+
+| Model Sizes | Step 1 | Step 2 | Step 3 | Total |
+|--------------------------------- |:------:|:------:|:------:|:------:|
+| Actor: OPT-13B, Reward: OPT-350M | 2.5hr | 0.25hr | 10.8hr | 13.6hr |
+
+*表 4. 在单个DGX节点上（8x A100-40G），针对不同的RLHF步骤， 使用DeepSpeed-Chat训练OPT-13b所需的时间。*
+
+
+
+*b) 使用 DeepSpeed-Chat 的推理 API 测试训练好的模型*
+
+将训练好的模型导入提供的简易 API，你不仅可以与模型进行问答互动，还可以进行对话式交流。
+
+```
+Human: Do you know Microsoft?
+Assistant: Yes, I’m a big fan of Microsoft. What about them?
+Human: Can you explian it to a 6-year old child? I wonder how I should describe it
+Assistant: Sure, I can try. Microsoft is a company that makes computers, and they make a program called “Windows” which
+ is the operating system that runs on the computer. It’s like the “brain” of the computer. It’s where all the
+ programs and files are stored. It’s also where you’ll find the “start” button, which is where you’ll start to
+ use the computer.
+```
+
+## 想尝试不同的模型大小和配置？轻松实现！
+我们了解到用户通常喜欢尝试不同的模型大小和配置，以满足他们不同的训练时间、资源和质量的需求。借助 DeepSpeed-Chat，你可以轻松实现这些目标。例如，如果你想在 GPU 集群上训练一个更大、更高质量的模型，用于你的研究或业务，你可以使用相同的脚本，只需输入你期望的模型大小（例如 660 亿参数）和 GPU 数量（例如 64 个 GPU）：
+
+```python
+python train.py --actor-model facebook/opt-66b --reward-model facebook/opt-350m --deployment-type multi_node
+```
+
+在 接下来的9 小时内，你将拥有一个 660 亿参数的 ChatGPT 模型，并可以在你喜欢的前端 GUI 中使用：
+
+
+
+| Model Sizes | Step 1 | Step 2 | Step 3 | Total |
+|---------------------------------- |:-------:|:------:|:------:|:------:|
+| Actor: OPT-66B, Reward: OPT-350M | 82 mins | 5 mins | 7.5hr | 9hr |
+
+*表 5. 在8个DGX节点上（64x A100-80G），针对不同的RLHF步骤， 使用DeepSpeed-Chat训练OPT-66b所需的时间。*
+
+
+
+如果你只有大约 1-2 小时的咖啡或午餐休息时间，你也可以尝试使用 DeepSpeed-Chat 训练一个小型模型。例如，我们为单个数据集准备了一个 13 亿参数模型的训练示例，以便在你的消费级 GPU 上测试我们的框架。当你从午餐休息回来时，你的模型已经准备好供你使用！
+
+```python
+python train.py --actor-model facebook/opt-1.3b --reward-model facebook/opt-350m --deployment-type single_gpu
+```
+
+
+
+| Model Sizes | Step 1 | Step 2 | Step 3 | Total |
+|--------------------------------- |:---------:|:--------:|:------:|:------:|
+| Actor: OPT-1.3B, Reward: OPT-350M | 2900 secs | 670 secs | 1.2hr | 2.2hr |
+
+*表 6. 在单个消费级A6000-48G上，针对不同的RLHF步骤， 使用DeepSpeed-Chat训练OPT-1.3b所需的时间。*
+
+
+
+## 利用 DeepSpeed-Chat 的 RLHF API 自定义你自己的 RLHF 训练流程
+DeepSpeed-Chat 允许用户使用我们灵活的 API（如下所示）构建自己的 RLHF 训练流程，用户可以使用这些 API 重建自己的 RLHF 训练策略。我们希望这些功能可以为研究探索中创建各种 RLHF 算法提供通用接口和后端。
+```python
+
+engine = DeepSpeedRLHFEngine(
+ actor_model_name_or_path=args.actor_model_name_or_path,
+ critic_model_name_or_path=args.critic_model_name_or_path,
+ tokenizer=tokenizer,
+ num_total_iters=num_total_iters,
+ args=args)
+
+trainer = DeepSpeedPPOTrainer(engine=engine, args=args)
+
+for prompt_batch in prompt_train_dataloader:
+ out = trainer.generate_experience(prompt_batch)
+ actor_loss, critic_loss = trainer.train_rlhf(out)
+```
+
+# 3. 完整的 RLHF 训练流程概述
+
+为了实现无缝的训练体验，我们遵循 InstructGPT 论文的方法，并在 DeepSpeed-Chat 中整合了一个端到端的训练流程，如图 1 所示。
+
+
+
+
+
+*图 1: DeepSpeed-Chat 的 RLHF 训练流程图示，包含了一些可选择的功能。*
+
+
+
+
+
+*Figure 2. 设计图解：DeepSpeed Hybrid Engine，用于加速 RLHF 流程中最耗时的部分。*
+
+
+
+
+
+*图 3. 在单个 NVIDIA A100-40G GPU 上，将 RLHF 训练的吞吐量与另外两个系统框架在步骤 3 进行比较。没有图标表示 OOM（内存不足）的情况*
+
+
+
+
+
+*图 4. 在单个 DGX 节点上，使用 8 个 NVIDIA A100-40G GPU，对训练流程第 3 步（耗时最长的部分）的不同模型大小进行端到端训练吞吐量比较。没有图标表示 OOM（内存不足）的情况。*
+
+
+
+
+
+*图 5. DeepSpeed Chat 的混合引擎在生成阶段的优越加速：在单个 DGX 节点上使用 8 个 A100-40G GPU 训练 OPT-1.3B actor 模型 + OPT-350M reward 模型的时间/序列分解。*
+
+
+
+
+
+*图 6. 在最大效率的情况下，DeepSpeed-HE 针对不同模型大小的RLHF生成、训练和有效吞吐量。*
+
+
+
+
+
+*图 7. 在不同数量的DGX (A100-40/80G GPU) 节点上，进行13B（左）和66B（右）actor 模型 和 350M reward 模型的可扩展性训练。*
+
+
+
+# DeepSpeed Chat: ChatGPTライクなモデルを簡単・高速・低コストに、あらゆるスケールで学習
+
+
+
+
+
+
+
+
+
+| GPUs | OPT-6.7B | OPT-13B | OPT-30B | OPT-66B |
+| ------- | :----------------------------------------------------------: | :------------------------------: | :-----: | :-----: |
+| 8x A100-40GB | 5.7 時間 | 10.8 時間 | 1.85 日 | NA |
+| 8x A100-80GB | 4.1 時間 ($132) | 9 時間 ($290) | 18 時間 ($580) | 2.1 日（$1620） |
+
+*表1. ノード1台（8x A100）を用いた場合の訓練時間とAzureでの概算実行コスト*
+
+
+
+
+***高スケーラビリティ***: DeepSpeed-HEは、数千億のパラメータを持つモデルをサポートし、複数ノード・複数GPUのシステムで、優れたスケーラビリティを実現することができます。その結果、13Bのモデルであれば1.25時間で学習でき、175Bの巨大モデルでも、1日以内に学習できます。
+
+
+
+| GPUs | OPT-13B | OPT-30B | OPT-66B | OPT-like-175B |
+| ------------ | :-------------------------------: | :---------------------------------: | :-----: | :-----------: |
+| 64x A100-80G | 1.25 時間 ($320) | 4 時間 ($1024) | 7.5 時間 ($1920) | 20 時間 ($5120) | | |
+
+*表2. 複数ノード（64x A100-80GB）を用いた場合の訓練時間とAzureでの概算実行コスト*
+
+
+> ***注意事項***: 上記の2つの表の数値は、訓練のステージ3のものです。DeepSpeed-RLHFが用いるデータセットと訓練の設定において、合計1.35億トークンを1エポックで訓練した際のスループットの実測値に基づいています。合計6750万のクエリートークン（配列長256の13万件のクエリー）と6750万の生成トークン（配列長256の13万件の回答）があり、ステップごとの最大グローバルバッチサイズは 50万 トークン（クエリーと回答それぞれ1024件）です。DeepSpeedRLHFを用いた場合のコストおよび実行時間の比較にあたっては、これらの詳細をよくご確認ください。さらに詳細な情報は[ベンチマーク設定](https://github.com/microsoft/DeepSpeedExamples/blob/staging-deepspeed-chat-v2/applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/BenckmarkSetting.md)を参照ください。
+
+***RLHFを誰もが利用できるように***: DeepSpeed-HEは、1台のGPUのみで130億以上のパラメーターを持つモデルの訓練を実行できます。複数のGPUを備えた高価な計算設備を持たないデータサイエンティストも、小規模なトイモデルではなく、実際のシナリオで使用できる大規模で強力なRLHFモデルを作成できます。
+
+
+
+| | V100 32G | A6000 48G | A100 40G | A100 80G |
+| ---------- | :------: | :-------: | :------: | :------: |
+| モデルサイズ| OPT-2.7B | OPT-6.7B | OPT-6.7B | OPT-13B |
+
+表3. DeepSpeed-HEを用いて1台のGPUで訓練できるモデルサイズ
+
+
+
+以降では、上記で紹介したDeepSpeed-Chatの3つの機能を深く掘り下げて紹介していきます。
+
+# 2. ChatGPTの訓練と推論を手軽に実行
+
+初めに、DeepSpeed-RLHFを用いた訓練の容易さを示す例として、OPT-13BとOPT-66Bのモデルを訓練できることを示します。訓練を実行できる時間が短い場合でも、コンシューマーグレードのGPU1つで、OPT-1.3Bモデルをわずか2時間で訓練することも可能です。DeepSpeed-ChatのAPIを用いて、カスタムされたパイプラインをどのように作るかも紹介していきます。
+
+## DeepSpeed-Chatの例を用いたChatGPTスタイルのモデルの訓練
+
+*a) スクリプトを一つ実行するだけで、RLHF訓練の全3ステージが実行され、ChatGPTライクなモデルを生成されるのを見てみましょう！*
+
+ここではアクターモデルとして事前学習済みOPT-13B、報酬モデルとしてOPT-350MをRLHFで訓練する例を示します。
+以下のスクリプトを実行すると、最終的に130億パラメータのChatGPTライクなモデルが生成されます。
+
+```python
+pip install deepspeed>=0.9.0
+
+git clone https://github.com/microsoft/DeepSpeedExamples.git
+cd DeepSpeedExamples/applications/DeepSpeed-Chat/
+pip install -r requirements.txt
+
+python train.py --actor-model facebook/opt-13b --reward-model facebook/opt-350m --deployment-type single_node
+```
+
+8台のA100を備えたDGXサーバを使う場合、130億パラメータのモデルを半日で訓練できます。以下は各ステップに要する時間の内訳です。
+
+
+
+| モデルサイズ| Step 1 | Step 2 | Step 3 | Total |
+| -------------------------------- | :----: | :----: | :----: | :----: |
+| アクターモデル: OPT-13B, 報酬モデル: OPT-350M | 2.5時間 | 0.25時間 | 10.8時間 | 13.6時間 |
+
+表4. 8台のA100を備えたDGXサーバ1台でのend-to-end処理時間の内訳
+
+
+
+*b) 学習したモデルをDeepSpeed Chatの推論APIで試してみましょう！*
+
+学習させたモデルを組み込んで、試してみましょう。質問応答だけでなく、会話形式のやりとりも楽しめます。
+
+```
+Human: Do you know Microsoft?
+Assistant: Yes, I’m a big fan of Microsoft. What about them?
+Human: Can you explain it to a 6-year old child? I wonder how I should describe it
+Assistant: Sure, I can try. Microsoft is a company that makes computers, and they make a program called “Windows” which
+ is the operating system that runs on the computer. It’s like the “brain” of the computer. It’s where all the
+ programs and files are stored. It’s also where you’ll find the “start” button, which is where you’ll start to
+ use the computer.
+```
+
+## モデルサイズ・設定の変更
+
+訓練の時間、計算機資源、および品質の要件を満たすために、さまざまなモデルのサイズや構成を試す必要があることがあります。DeepSpeed-Chatを使用すれば、簡単にそれが可能です。例えば、研究やビジネスのために、GPUクラスタでより大規模で高品質なモデルを訓練したい場合、希望するモデルサイズとGPU数を指定するだけです。以下は、アクターモデルのパラメータ数を66Bに、GPU数を64に指定する例です。
+
+```python
+python train.py --actor-model facebook/opt-66b --reward-model facebook/opt-350m --deployment-type multi_node
+```
+
+64台のA100 (80GBメモリ) GPUを使用する場合、9時間で660億パラメータのChatGPTモデルを訓練できます。
+
+
+
+| モデルサイズ | Step 1 | Step 2 | Step 3 | Total |
+| -------------------------------- | :-----: | :----: | :----: | :---: |
+|アクターモデル: OPT-66B, 報酬モデル: OPT-350M | 82分 | 5分 | 7.5時間 | 9時間 |
+
+表5. 8台のA100(80GBメモリ)を備えたDGXサーバ8台で660億パラメータのモデルを訓練する場合のend-to-end処理時間の内訳
+
+
+
+1～2時間のコーヒータイムや昼休みに、DeepSpeed-Chatで小規模なトイモデルをトレーニングしてみるのも良いでしょう。例えば、コンシューマグレードのGPUでの訓練を動かしてみるため、1つのデータセットで1.3Bのモデルを訓練する例を用意しました。これなら、昼休みから戻ったときに、できあがったモデルのチェックポイントを試してみることができます。
+
+```python
+python train.py --actor-model facebook/opt-1.3b --reward-model facebook/opt-350m --deployment-type single_gpu
+```
+
+
+
+| モデルサイズ | Step 1 | Step 2 | Step 3 | Total |
+| --------------------------------- | :-------: | :------: | :----: | :---: |
+| Actor: OPT-1.3B, Reward: OPT-350M | 2900 秒| 670 秒 | 1.2時間 | 2.2時間 |
+
+*表6. コモディティグレードのGPU（A6000）1台でのend-to-end処理時間の内訳*
+
+
+
+## DeepSpeed-ChatのAPIを用いたRLHFパイプラインのカスタマイズ
+
+DeepSpeed-Chatでは、設定を柔軟に指定できるAPIを提供しており、ユーザーが独自のRLHF訓練パイプラインを構築することができます。このAPIを通じて、探索的な研究のための幅広いRLHFアルゴリズムを作成するための、汎用的なインターフェースとバックエンドを利用できます。
+
+```python
+
+engine = DeepSpeedRLHFEngine(
+ actor_model_name_or_path=args.actor_model_name_or_path,
+ critic_model_name_or_path=args.critic_model_name_or_path,
+ tokenizer=tokenizer,
+ num_total_iters=num_total_iters,
+ args=args)
+
+trainer = DeepSpeedPPOTrainer(engine=engine, args=args)
+
+for prompt_batch in prompt_train_dataloader:
+ out = trainer.generate_experience(prompt_batch)
+ actor_loss, critic_loss = trainer.train_rlhf(out)
+```
+
+# 3. 完全なRLHF訓練パイプライン
+
+シームレスに訓練を実行するため、DeepSpeed-Chatには、InstructGPTと同様の完全なend-to-endの訓練パイプラインが組み込まれています（図1）。
+
+
+
+
+
+*図1: DeepSpeed-Chatの訓練パイプラインの概要*
+
+
+
+
+
+*図2. RLHFで最も処理時間がかかる部分を高速化するハイブリッドエンジン（DeepSpeed-HE）*
+
+
+
+
+
+*図3. 他フレームワークとのStep 3のスループット比較（1台のA100-40Gを使用。バツ印はメモリ不足で実行できないことを示す）*
+
+
+
+
+
+*図4. ステップ3（全3ステップ処理時間の大半を占める）のEnd-to-endの訓練スループット比較 (8台のA100-40Gを備えた1台のDGXノードを使用。バツ印はメモリ不足で実行できないことを示す）*
+
+
+
+
+
+*図5. DeepSpeed-HEを用いた生成フェーズの高速化（OPT-1.3Bベースのアクターモデル + OPT-350Mベースの報酬モデル、8台のA100-40Gを備えた1台のDGXノードを使用）*
+
+
+
+
+
+*図6. DeepSpeed-HEを用いたRLHFにおける生成、訓練、および実効スループット（GPU数は最善の効率を得られるように設定）*
+
+
+
+
+
+*図7. DGXノード（ノードあたり8台のA100-40/80G）の数を増加させた場合のスケーラビリティ（13Bおよび66Bのアクターモデルと350Mの報酬モデルを使用）*
+
+
| [Transformers with DeepSpeed](https://huggingface.co/docs/transformers/main/main_classes/deepspeed) |
| 
| [Accelerate with DeepSpeed](https://huggingface.co/docs/accelerate/usage_guides/deepspeed) |
-| 
| [Lightning with DeepSpeed](https://pytorch-lightning.readthedocs.io/en/stable/api/pytorch_lightning.strategies.DeepSpeedStrategy.html) |
-| 
| [MosaicML with DeepSpeed](https://docs.mosaicml.com/en/latest/trainer/using_the_trainer.html?highlight=deepspeed#deepspeed-integration) |
+| 
| [Determined with DeepSpeed](https://docs.determined.ai/latest/training/apis-howto/deepspeed/overview.html) |
---
@@ -104,11 +107,12 @@ DeepSpeed has been integrated with several different popular open-source DL fram
| Description | Status |
| ----------- | ------ |
-| NVIDIA | [](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-torch12-p40.yml) [](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-torch18-v100.yml) [](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-torch-latest-v100.yml) [](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-inference.yml) [](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-nightly.yml) |
-| AMD | [](https://github.com/microsoft/DeepSpeed/actions/workflows/amd.yml) |
-| PyTorch Nightly | [](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-torch-nightly-v100.yml) |
-| Integrations | [](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-transformers-v100.yml) [](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-lightning-v100.yml) [](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-accelerate-v100.yml) |
-| Misc | [](https://github.com/microsoft/DeepSpeed/actions/workflows/formatting.yml) [](https://github.com/microsoft/DeepSpeed/actions/workflows/pages/pages-build-deployment) [](https://deepspeed.readthedocs.io/en/latest/?badge=latest)|
+| NVIDIA | [](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-torch19-p40.yml) [](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-torch19-v100.yml) [](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-torch-latest-v100.yml) [](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-inference.yml) [](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-nightly.yml) |
+| AMD | [](https://github.com/microsoft/DeepSpeed/actions/workflows/amd-mi100.yml) [](https://github.com/microsoft/DeepSpeed/actions/workflows/amd-mi200.yml) |
+| CPU | [](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-torch-latest-cpu.yml) |
+| PyTorch Nightly | [](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-torch-nightly-v100.yml) |
+| Integrations | [](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-transformers-v100.yml) [](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-lightning-v100.yml) [](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-accelerate-v100.yml)[](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-megatron.yml)[](https://github.com/microsoft/DeepSpeed/actions/workflows/nv-mii.yml) |
+| Misc | [](https://github.com/microsoft/DeepSpeed/actions/workflows/formatting.yml) [](https://github.com/microsoft/DeepSpeed/actions/workflows/pages/pages-build-deployment) [](https://deepspeed.readthedocs.io/en/latest/?badge=latest)[](https://github.com/microsoft/DeepSpeed/actions/workflows/python.yml) |
# Installation
@@ -122,7 +126,7 @@ dynamically link them at runtime.
## Requirements
* [PyTorch](https://pytorch.org/) must be installed _before_ installing DeepSpeed.
-* For full feature support we recommend a version of PyTorch that is >= 1.8 and ideally the latest PyTorch stable release.
+* For full feature support we recommend a version of PyTorch that is >= 1.9 and ideally the latest PyTorch stable release.
* A CUDA or ROCm compiler such as [nvcc](https://docs.nvidia.com/cuda/cuda-compiler-driver-nvcc/#introduction) or [hipcc](https://github.com/ROCm-Developer-Tools/HIPCC) used to compile C++/CUDA/HIP extensions.
* Specific GPUs we develop and test against are listed below, this doesn't mean your GPU will not work if it doesn't fall into this category it's just DeepSpeed is most well tested on the following:
* NVIDIA: Pascal, Volta, Ampere, and Hopper architectures
@@ -174,7 +178,12 @@ All DeepSpeed documentation, tutorials, and blogs can be found on our website: [
# Contributing
DeepSpeed welcomes your contributions! Please see our
[contributing](CONTRIBUTING.md) guide for more details on formatting, testing,
-etc.
+etc.