beit.py

import torch
from timm.models.beit import Beit

from colossalai.utils.cuda import get_current_device

from .registry import non_distributed_component_funcs
from .utils.dummy_data_generator import DummyDataGenerator


class DummyDataLoader(DummyDataGenerator):
    img_size = 64
    num_channel = 3
    num_class = 10
    batch_size = 4

    def generate(self):
        data = torch.randn((DummyDataLoader.batch_size, DummyDataLoader.num_channel, DummyDataLoader.img_size,
                            DummyDataLoader.img_size),
                           device=get_current_device())
        label = torch.randint(low=0,
                              high=DummyDataLoader.num_class,
                              size=(DummyDataLoader.batch_size,),
                              device=get_current_device())
        return data, label


@non_distributed_component_funcs.register(name='beit')
def get_training_components():

    def model_builder(checkpoint=False):
        model = Beit(img_size=DummyDataLoader.img_size,
                     num_classes=DummyDataLoader.num_class,
                     embed_dim=32,
                     depth=2,
                     num_heads=4)
        return model

    trainloader = DummyDataLoader()
    testloader = DummyDataLoader()

    criterion = torch.nn.CrossEntropyLoss()
    return model_builder, trainloader, testloader, torch.optim.Adam, criterion