Add sample DeepSpeed config

8bbe9e5a · Gustaf Ahdritz · 29315bdc · 8bbe9e5a
Commit 8bbe9e5a authored Oct 12, 2021 by Gustaf Ahdritz
Hide whitespace changes
Inline Side-by-side

Showing with 34 additions and 0 deletions

deepspeed_config.json deepspeed_config.json +34 -0

No files found.
--- a/deepspeed_config.json
+++ b/deepspeed_config.json
+{
+  "optimizer": {
+    "type": "Adam",
+    "params": {
+      "lr": 0.001,
+      "eps": 1e-05
+    }
+  },
+  "scheduler": {
+      "type": "WarmupLR",
+      "params": {
+        "warmup_min_lr": 0,
+        "warmup_max_lr": 0.001,
+        "warmup_num_steps": 128000
+      }
+  },
+  "fp16": {
+    "enabled": true,
+    "min_loss_scale": 1
+  },
+  "amp": {
+    "enabled": false,
+    "opt_level": "O2"
+  },
+  "zero_optimization": {
+    "stage": 1
+  },
+  "activation_checkpointing": {
+    "partition_activations": true,
+    "cpu_checkpointing": false,
+    "profile": false
+  },
+  "gradient_clipping": 0.1
+}