import torch
import torch.nn as nn
import torch.nn.functional as F



if __name__ == "__main__":
    # 模拟配置
    config = {
        "hidden_size": 4096,
        "mlp_activation": "silu",
        "model_parallel_size:q": 1,
        "evo2_style_activations": False,
    }