from omegaconf import OmegaConf import oneflow as flow import oneflow.nn as nn from libai.data.build import build_image_train_loader, build_image_test_loader from libai.config import LazyCall, get_config from libai.optim import get_default_optimizer_params from libai.scheduler.lr_scheduler import WarmupCosineAnnealingLR, WarmupMultiStepLR from projects.NeRF.datasets import BlenderDataset, LLFFDataset from projects.NeRF.optimizers import Ranger, RAdam from projects.NeRF.evaluation.nerf_evaluator import NerfEvaluator from projects.NeRF.configs.config_model import model def get_nerf_dataset(dataset_type="Blender"): """ Args: dataset_type: Blender or LLFF """ assert dataset_type in ["Blender", "LLFF"], "The Nerf dataset must be one of Blender and LLFF" if dataset_type == "Blender": return BlenderDataset else: return LLFFDataset graph = get_config("common/models/graph.py").graph graph.enabled = False train = get_config("common/train.py").train # Refine train cfg for Nerf System train.train_micro_batch_size = 1024 # Verification by ray train.test_micro_batch_size = 1 # Verification by picture train.dataset_type = "Blender" # Blender or LLFF train.blender_dataset_path = "/path/to/blender" train.llff_dataset_path = "/path/to/llff" train.train_epoch = 16 if train.dataset_type == "Blender" else 30 train.warmup_ratio = int(1 / train.train_epoch) train.evaluation.eval_period = 1000 train.log_period = 50 train.optim_type = "adam" train.lr_scheduler_type = "cosine" # Redefining model config model.cfg.dataset_type = train.dataset_type model.cfg.loss_func = nn.MSELoss() model.cfg.noise_std = 0.0 if train.dataset_type == "Blender" else 1.0 # Redefining evaluator train.evaluation = dict( enabled=True, # evaluator for calculating psnr evaluator=LazyCall(NerfEvaluator)( img_wh=(400, 400) if train.dataset_type == "Blender" else (504, 378) ), eval_period=train.evaluation.eval_period, eval_iter=1e5, # running steps for validation/test # Metrics to be used for best model checkpoint. eval_metric="psnr", eval_mode="max", ) # Refine optimizer cfg for Nerf System # NOTE: In theory, both datasets used by Nerf are optimized using the Adam optimizer, but # since the borrowed code base also implements three other optimizer configurations, libai # also implements the corresponding optimizer. if train.optim_type == "adam": optimizer = flow.optim.Adam lr = 5e-4 elif train.optim_type == "sgd": optimizer = flow.optim.SGD lr = 5e-2 elif train.optim_type == "radam": optimizer = RAdam lr = 5e-4 elif train.optim_type == "ranger": optimizer = Ranger lr = 5e-4 else: raise NotImplementedError("Nerf does not support this type of optimizer!") optim = LazyCall(optimizer)( params=LazyCall(get_default_optimizer_params)( # params.model is meant to be set to the model object, # before instantiating the optimizer. clip_grad_max_norm=None, clip_grad_norm_type=None, weight_decay_norm=None, weight_decay_bias=None, ), lr=lr, weight_decay=0, ) if train.optim_type == "sgd": optim.momentum = 0.9 if train.lr_scheduler_type == "steplr": scheduler = WarmupMultiStepLR elif train.lr_scheduler_type == "cosine": scheduler = WarmupCosineAnnealingLR else: raise NotImplementedError("Nerf does not support this type of scheduler!") train.scheduler = LazyCall(scheduler)( warmup_factor=0.001, warmup_method="linear", ) if train.lr_scheduler_type == "steplr": if train.dataset_type == "Blender": milestones = [2 / 16, 4 / 16, 8 / 16] else: milestones = [10 / 30, 20 / 30] train.scheduler.milestones = milestones train.scheduler.gamma = 0.5 elif train.lr_scheduler_type == "cosine": train.scheduler.eta_min = 1e-8 train.warmup_ratio = ( train.warmup_ratio if train.warmup_ratio > 0 and train.optim_type not in ["radam", "ranger"] else 0.0 ) # Set fp16 ON train.amp.enabled = True dataset = LazyCall(get_nerf_dataset)(dataset_type=train.dataset_type) dataloader = OmegaConf.create() dataloader.train = LazyCall(build_image_train_loader)( dataset=[ LazyCall(dataset)( split="train", img_wh=(400, 400) if dataset.dataset_type == "Blender" else (504, 378), root_dir=train.blender_dataset_path if dataset.dataset_type == "Blender" else train.llff_dataset_path, spheric_poses=None if dataset.dataset_type == "Blender" else False, val_num=None if dataset.dataset_type == "Blender" else 1, # Number of your GPUs batchsize=train.train_micro_batch_size, ) ], num_workers=4, train_batch_size=1, test_batch_size=train.test_micro_batch_size, ) dataloader.test = [ LazyCall(build_image_test_loader)( dataset=LazyCall(dataset)( split="val", img_wh=(400, 400) if dataset.dataset_type == "Blender" else (504, 378), root_dir=train.blender_dataset_path if dataset.dataset_type == "Blender" else train.llff_dataset_path, spheric_poses=None if dataset.dataset_type == "Blender" else False, val_num=None if dataset.dataset_type == "Blender" else 1, # Number of your GPUs ), num_workers=0, test_batch_size=train.test_micro_batch_size, ) ] # Distributed Settings depth = None train.train_micro_batch_size = 1 train.dist.pipeline_num_layers = depth train.dist.data_parallel_size = 1 train.dist.tensor_parallel_size = 1 train.dist.pipeline_parallel_size = 1