"sgl-router/vscode:/vscode.git/clone" did not exist on "53c2934dcef8794449b7f270ad162f3fdabc01e3"
ds_config_zero2.json 1.08 KB
Newer Older
1
2
{
    "fp16": {
3
        "enabled": "auto",
4
5
        "loss_scale": 0,
        "loss_scale_window": 1000,
6
        "initial_scale_power": 16,
7
8
9
10
        "hysteresis": 2,
        "min_loss_scale": 1
    },

11
12
13
    "optimizer": {
        "type": "AdamW",
        "params": {
14
15
16
17
            "lr": "auto",
            "betas": "auto",
            "eps": "auto",
            "weight_decay": "auto"
18
19
        }
    },
20

21
22
23
    "scheduler": {
        "type": "WarmupLR",
        "params": {
24
25
26
            "warmup_min_lr": "auto",
            "warmup_max_lr": "auto",
            "warmup_num_steps": "auto"
27
28
        }
    },
29

30
31
32
33
34
35
36
37
38
39
40
41
42
    "zero_optimization": {
        "stage": 2,
        "allgather_partitions": true,
        "allgather_bucket_size": 2e8,
        "overlap_comm": true,
        "reduce_scatter": true,
        "reduce_bucket_size": 2e8,
        "contiguous_gradients": true,
        "cpu_offload": true
    },

    "gradient_accumulation_steps": "auto",
    "gradient_clipping": "auto",
43
    "steps_per_print": 2000,
44
45
    "train_batch_size": "auto",
    "train_micro_batch_size_per_gpu": "auto",
46
47
    "wall_clock_breakdown": false
}