Skip to content

GitLab

Switch branch/tag
Find file Normal viewHistoryPermalink
megatron_critic.yaml 3.46 KB
Newer Older
jerrrrry's avatar
Initial commit  
jerrrrry committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129 
# defaults specify the default config from each component
defaults:

  # dp actor config, inheriting from trainer/config/critic/critic.yaml
  - critic

  # load the reference default config, then apply the fields in the current yaml
  - _self_

# Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs
_target_: verl.workers.config.McoreCriticConfig

strategy: megatron

# seconds, default is 10 minutes for torch, you can set it to a larger value if you have long-running operations like 32B or 72B model using megatron
nccl_timeout: 600

# optimizer configs
optim:

  # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs
  _target_: verl.workers.config.McoreOptimizerConfig

  # select optimizer, default is Adam
  optimizer: adam

  # Clip gradients norm
  clip_grad: 1.0

  # initial learning rate for warmup, default to 0.0
  lr_warmup_init: 0.0

  lr_decay_steps: null

  # select from constant/linear/cosine/inverse_square_root
  lr_decay_style: linear

  # minimum learning rate, default to 0.0
  min_lr: 0.0

  # select from constant/linear/cosine
  weight_decay_incr_style: constant

  # select from constant/exponential/cosine
  lr_wsd_decay_style: exponential

  # number of steps for weight std decay
  lr_wsd_decay_steps: null

  # use checkpoint optimizer parameter scheduler
  use_checkpoint_opt_param_scheduler: False

# model config for the critic
model:

  # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs
  _target_: verl.trainer.config.BaseModelConfig

  # override default empty mapping
  override_config:

    model_config: {}

    moe_config:

      freeze_moe_router: False

# megatron-specific parallelism settings
megatron:

  # Required when using verl.utils.omega_conf_to_dataclass to instantiate dataclass configs
  _target_: verl.workers.config.McoreEngineConfig

  # Whether to offload model parameters to CPU
  param_offload: False

  # Whether to offload gradients to CPU
  grad_offload: False

  # Whether to offload optimizer state to CPU
  optimizer_offload: False

  # size of tensor model parallel group
  tensor_model_parallel_size: 1

  # size of expert model parallel group
  expert_model_parallel_size: 1

  # size of expert tensor parallel group
  expert_tensor_parallel_size: null

  # size of pipeline model parallel group
  pipeline_model_parallel_size: 1

  # size of virtual pipeline model parallel group
  virtual_pipeline_model_parallel_size: null

  # size of context parallel group
  context_parallel_size: 1

  # Whether to use sequence parallelism
  sequence_parallel: True

  # Whether to use distributed optimizer
  use_distributed_optimizer: True

  # Whether to use distributed checkpointing
  use_dist_checkpointing: False

  # Path for distributed checkpointing
  dist_checkpointing_path: null

  # Random seed for Megatron
  seed: ${oc.select:actor_rollout_ref.actor.megatron.seed,42}

  # Allow to override Distributed Data Parallel (DDP) config
  override_ddp_config: ${oc.select:actor_rollout_ref.actor.megatron.override_ddp_config,{}}

  # Transformer config overrides for Megatron
  override_transformer_config: ${oc.select:actor_rollout_ref.actor.megatron.override_transformer_config,{}}

  # Whether to use mBridge communications
  use_mbridge: ${oc.select:actor_rollout_ref.actor.megatron.use_mbridge,False}

# Whether to load initial weights
load_weight: True

# seed for data loader
data_loader_seed: ${oc.select:actor_rollout_ref.actor.data_loader_seed,null}