Skip to content

GitLab

Switch branch/tag
Find file Normal viewHistoryPermalink
test_swiglu_ffn.py 1.69 KB
Newer Older
limm's avatar
add tests part  
limm committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53 
# Copyright (c) OpenMMLab. All rights reserved.
from unittest import TestCase

import torch
import torch.nn as nn

from mmpretrain.models.utils import LayerScale, SwiGLUFFN, SwiGLUFFNFused


class TestSwiGLUFFN(TestCase):

    def test_init(self):
        swiglu = SwiGLUFFN(embed_dims=4)
        assert swiglu.w12.weight.shape == torch.ones((8, 4)).shape
        assert swiglu.w3.weight.shape == torch.ones((4, 4)).shape
        assert isinstance(swiglu.gamma2, nn.Identity)

        swiglu = SwiGLUFFN(embed_dims=4, layer_scale_init_value=0.1)
        assert isinstance(swiglu.gamma2, LayerScale)

    def test_forward(self):
        swiglu = SwiGLUFFN(embed_dims=4)
        x = torch.randn((1, 8, 4))
        out = swiglu(x)
        self.assertEqual(out.size(), x.size())

        swiglu = SwiGLUFFN(embed_dims=4, out_dims=12)
        x = torch.randn((1, 8, 4))
        out = swiglu(x)
        self.assertEqual(tuple(out.size()), (1, 8, 12))


class TestSwiGLUFFNFused(TestCase):

    def test_init(self):
        swiglu = SwiGLUFFNFused(embed_dims=4)
        assert swiglu.w12.weight.shape == torch.ones((16, 4)).shape
        assert swiglu.w3.weight.shape == torch.ones((4, 8)).shape
        assert isinstance(swiglu.gamma2, nn.Identity)

        swiglu = SwiGLUFFNFused(embed_dims=4, layer_scale_init_value=0.1)
        assert isinstance(swiglu.gamma2, LayerScale)

    def test_forward(self):
        swiglu = SwiGLUFFNFused(embed_dims=4)
        x = torch.randn((1, 8, 4))
        out = swiglu(x)
        self.assertEqual(out.size(), x.size())

        swiglu = SwiGLUFFNFused(embed_dims=4, out_dims=12)
        x = torch.randn((1, 8, 4))
        out = swiglu(x)
        self.assertEqual(tuple(out.size()), (1, 8, 12))