Skip to content

GitLab

Unverified Commit bdfefbad authored by Sourab Mangrulkar's avatar Sourab Mangrulkar Committed by GitHub
Browse files

Update ds_config_zero3.json (#30829)

parent 92544cb8
Hide whitespace changes
Inline Side-by-side
Showing with 3 additions and 8 deletions
tests/deepspeed/ds_config_zero3.json
View file @ bdfefbad
...@@ -7,11 +7,9 @@ ...@@ -7,11 +7,9 @@
"hysteresis": 2, "hysteresis": 2,
"min_loss_scale": 1 "min_loss_scale": 1
}, },
"bf16": { "bf16": {
"enabled": "auto" "enabled": "auto"
}, },
"optimizer": { "optimizer": {
"type": "AdamW", "type": "AdamW",
"params": { "params": {
...@@ -21,7 +19,6 @@ ...@@ -21,7 +19,6 @@
"weight_decay": "auto" "weight_decay": "auto"
} }
}, },
"scheduler": { "scheduler": {
"type": "WarmupLR", "type": "WarmupLR",
"params": { "params": {
...@@ -30,15 +27,14 @@ ...@@ -30,15 +27,14 @@
"warmup_num_steps": "auto" "warmup_num_steps": "auto"
} }
}, },
"zero_optimization": { "zero_optimization": {
"stage": 3, "stage": 3,
"offload_optimizer": { "offload_optimizer": {
"device": "cpu", "device": "none",
"pin_memory": true "pin_memory": true
}, },
"offload_param": { "offload_param": {
"device": "cpu", "device": "none",
"pin_memory": true "pin_memory": true
}, },
"overlap_comm": true, "overlap_comm": true,
...@@ -51,11 +47,10 @@ ...@@ -51,11 +47,10 @@
"stage3_max_reuse_distance": 1e9, "stage3_max_reuse_distance": 1e9,
"stage3_gather_16bit_weights_on_model_save": true "stage3_gather_16bit_weights_on_model_save": true
}, },
"gradient_accumulation_steps": "auto", "gradient_accumulation_steps": "auto",
"gradient_clipping": "auto", "gradient_clipping": "auto",
"steps_per_print": 2000, "steps_per_print": 2000,
"train_batch_size": "auto", "train_batch_size": "auto",
"train_micro_batch_size_per_gpu": "auto", "train_micro_batch_size_per_gpu": "auto",
"wall_clock_breakdown": false "wall_clock_breakdown": false
} }
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment