polish

examples/language/gpt/experiments/pipeline_parallel/README.md

-# Auto-Parallelism with GPT2
+# Pipeline Parallelism Demo with GPT2
 
 ## Requirements
 
@@ -33,5 +33,6 @@ For simplicity, the input data is randonly generated here.
 
 ```bash
 #Run the Pipeline Parallel on GPT with default setting and a dummy dataset.
+#You can change the GPU number or microbatch number in the run.sh .
 bash run.sh
 ```

examples/language/gpt/experiments/pipeline_parallel/utils.py

-import torch
-
-
-# Randomly Generated Data
-def get_data(batch_size, seq_len, vocab_size):
-    input_ids = torch.randint(0, vocab_size, (batch_size, seq_len), device=torch.cuda.current_device())
-    attention_mask = torch.ones_like(input_ids)
-    return input_ids, attention_mask
-
-
-def get_tflops(model_numel, batch_size, seq_len, step_time):
-    return model_numel * batch_size * seq_len * 8 / 1e12 / (step_time + 1e-12)