"git@developer.sourcefind.cn:chenpangpang/open-webui.git" did not exist on "a901031896aa7085d03064ba6ee046d695f03fcb"
Commit 9b87ce51 authored by dongchy920's avatar dongchy920
Browse files

arcface

parents
from easydict import EasyDict as edict
# make training faster
# our RAM is 256G
# mount -t tmpfs -o size=140G tmpfs /train_tmp
config = edict()
config.margin_list = (1.0, 0.0, 0.4)
config.network = "r200"
config.resume = False
config.output = None
config.embedding_size = 512
config.sample_rate = 0.3
config.fp16 = True
config.momentum = 0.9
config.weight_decay = 5e-4
config.batch_size = 128
config.lr = 0.4
config.verbose = 2000
config.dali = False
config.rec = "/train_tmp/WebFace42M"
config.num_classes = 2059906
config.num_image = 42474557
config.num_epoch = 20
config.warmup_epoch = config.num_epoch // 10
config.val_targets = ["lfw", "cfp_fp", "agedb_30"]
from easydict import EasyDict as edict
# make training faster
# our RAM is 256G
# mount -t tmpfs -o size=140G tmpfs /train_tmp
config = edict()
config.margin_list = (1.0, 0.0, 0.4)
config.network = "r50"
config.resume = False
config.output = None
config.embedding_size = 512
config.sample_rate = 0.3
config.fp16 = True
config.momentum = 0.9
config.weight_decay = 5e-4
config.batch_size = 128
config.lr = 0.4
config.verbose = 2000
config.dali = False
config.rec = "/train_tmp/WebFace42M"
config.num_classes = 2059906
config.num_image = 42474557
config.num_epoch = 20
config.warmup_epoch = config.num_epoch // 10
config.val_targets = ["lfw", "cfp_fp", "agedb_30"]
from easydict import EasyDict as edict
# make training faster
# our RAM is 256G
# mount -t tmpfs -o size=140G tmpfs /train_tmp
config = edict()
config.margin_list = (1.0, 0.0, 0.4)
config.network = "vit_b_dp005_mask_005"
config.resume = False
config.output = None
config.embedding_size = 512
config.sample_rate = 0.3
config.fp16 = True
config.weight_decay = 0.1
config.batch_size = 384
config.optimizer = "adamw"
config.lr = 0.001
config.verbose = 2000
config.dali = False
config.rec = "/train_tmp/WebFace42M"
config.num_classes = 2059906
config.num_image = 42474557
config.num_epoch = 40
config.warmup_epoch = config.num_epoch // 10
config.val_targets = []
from easydict import EasyDict as edict
# make training faster
# our RAM is 256G
# mount -t tmpfs -o size=140G tmpfs /train_tmp
config = edict()
config.margin_list = (1.0, 0.0, 0.4)
config.network = "vit_l_dp005_mask_005"
config.resume = False
config.output = None
config.embedding_size = 512
config.sample_rate = 0.3
config.fp16 = True
config.weight_decay = 0.1
config.batch_size = 384
config.optimizer = "adamw"
config.lr = 0.001
config.verbose = 2000
config.dali = False
config.rec = "/train_tmp/WebFace42M"
config.num_classes = 2059906
config.num_image = 42474557
config.num_epoch = 40
config.warmup_epoch = config.num_epoch // 10
config.val_targets = []
from easydict import EasyDict as edict
# make training faster
# our RAM is 256G
# mount -t tmpfs -o size=140G tmpfs /train_tmp
config = edict()
config.margin_list = (1.0, 0.0, 0.4)
config.network = "vit_s_dp005_mask_0"
config.resume = False
config.output = None
config.embedding_size = 512
config.sample_rate = 0.3
config.fp16 = True
config.weight_decay = 0.1
config.batch_size = 384
config.optimizer = "adamw"
config.lr = 0.001
config.verbose = 2000
config.dali = False
config.rec = "/train_tmp/WebFace42M"
config.num_classes = 2059906
config.num_image = 42474557
config.num_epoch = 40
config.warmup_epoch = config.num_epoch // 10
config.val_targets = []
from easydict import EasyDict as edict
# make training faster
# our RAM is 256G
# mount -t tmpfs -o size=140G tmpfs /train_tmp
config = edict()
config.margin_list = (1.0, 0.0, 0.4)
config.network = "vit_t_dp005_mask0"
config.resume = False
config.output = None
config.embedding_size = 512
config.sample_rate = 0.3
config.fp16 = True
config.weight_decay = 0.1
config.batch_size = 384
config.optimizer = "adamw"
config.lr = 0.001
config.verbose = 2000
config.dali = False
config.rec = "/train_tmp/WebFace42M"
config.num_classes = 2059906
config.num_image = 42474557
config.num_epoch = 40
config.warmup_epoch = config.num_epoch // 10
config.val_targets = []
from easydict import EasyDict as edict
# make training faster
# our RAM is 256G
# mount -t tmpfs -o size=140G tmpfs /train_tmp
config = edict()
config.margin_list = (1.0, 0.0, 0.4)
config.network = "vit_b_dp005_mask_005"
config.resume = False
config.output = None
config.embedding_size = 512
config.sample_rate = 0.3
config.fp16 = True
config.weight_decay = 0.1
config.batch_size = 256
config.gradient_acc = 12 # total batchsize is 256 * 12
config.optimizer = "adamw"
config.lr = 0.001
config.verbose = 2000
config.dali = False
config.rec = "/train_tmp/WebFace42M"
config.num_classes = 2059906
config.num_image = 42474557
config.num_epoch = 40
config.warmup_epoch = config.num_epoch // 10
config.val_targets = []
from easydict import EasyDict as edict
# make training faster
# our RAM is 256G
# mount -t tmpfs -o size=140G tmpfs /train_tmp
config = edict()
config.margin_list = (1.0, 0.0, 0.4)
config.network = "vit_t_dp005_mask0"
config.resume = False
config.output = None
config.embedding_size = 512
config.sample_rate = 0.3
config.fp16 = True
config.weight_decay = 0.1
config.batch_size = 512
config.optimizer = "adamw"
config.lr = 0.001
config.verbose = 2000
config.dali = False
config.rec = "/train_tmp/WebFace42M"
config.num_classes = 2059906
config.num_image = 42474557
config.num_epoch = 40
config.warmup_epoch = config.num_epoch // 10
config.val_targets = []
from easydict import EasyDict as edict
# make training faster
# our RAM is 256G
# mount -t tmpfs -o size=140G tmpfs /train_tmp
config = edict()
config.margin_list = (1.0, 0.0, 0.4)
config.network = "mbf"
config.resume = False
config.output = None
config.embedding_size = 512
config.sample_rate = 1.0
config.fp16 = True
config.momentum = 0.9
config.weight_decay = 1e-4
config.batch_size = 128
config.lr = 0.1
config.verbose = 2000
config.dali = False
config.rec = "/train_tmp/WebFace4M"
config.num_classes = 205990
config.num_image = 4235242
config.num_epoch = 20
config.warmup_epoch = 0
config.val_targets = ['lfw', 'cfp_fp', "agedb_30"]
from easydict import EasyDict as edict
# make training faster
# our RAM is 256G
# mount -t tmpfs -o size=140G tmpfs /train_tmp
config = edict()
config.margin_list = (1.0, 0.0, 0.4)
config.network = "r100"
config.resume = False
config.output = None
config.embedding_size = 512
config.sample_rate = 1.0
config.fp16 = True
config.momentum = 0.9
config.weight_decay = 5e-4
config.batch_size = 128
config.lr = 0.1
config.verbose = 2000
config.dali = False
config.rec = "/train_tmp/WebFace4M"
config.num_classes = 205990
config.num_image = 4235242
config.num_epoch = 20
config.warmup_epoch = 0
config.val_targets = ['lfw', 'cfp_fp', "agedb_30"]
from easydict import EasyDict as edict
# make training faster
# our RAM is 256G
# mount -t tmpfs -o size=140G tmpfs /train_tmp
config = edict()
config.margin_list = (1.0, 0.0, 0.4)
config.network = "r50"
config.resume = False
config.output = None
config.embedding_size = 512
config.sample_rate = 1.0
config.fp16 = True
config.momentum = 0.9
config.weight_decay = 5e-4
config.batch_size = 128
config.lr = 0.1
config.verbose = 2000
config.dali = False
config.rec = "/train_tmp/WebFace4M"
config.num_classes = 205990
config.num_image = 4235242
config.num_epoch = 20
config.warmup_epoch = 0
config.val_targets = ['lfw', 'cfp_fp', "agedb_30"]
import numbers
import os
import queue as Queue
import threading
from typing import Iterable
import mxnet as mx
import numpy as np
import torch
from functools import partial
from torch import distributed
from torch.utils.data import DataLoader, Dataset
from torchvision import transforms
from torchvision.datasets import ImageFolder
from utils.utils_distributed_sampler import DistributedSampler
from utils.utils_distributed_sampler import get_dist_info, worker_init_fn
def get_dataloader(
root_dir,
local_rank,
batch_size,
dali = False,
dali_aug = False,
seed = 2048,
num_workers = 2,
) -> Iterable:
rec = os.path.join(root_dir, 'train.rec')
idx = os.path.join(root_dir, 'train.idx')
train_set = None
# Synthetic
if root_dir == "synthetic":
train_set = SyntheticDataset()
dali = False
# Mxnet RecordIO
elif os.path.exists(rec) and os.path.exists(idx):
train_set = MXFaceDataset(root_dir=root_dir, local_rank=local_rank)
# Image Folder
else:
transform = transforms.Compose([
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]),
])
train_set = ImageFolder(root_dir, transform)
# DALI
if dali:
return dali_data_iter(
batch_size=batch_size, rec_file=rec, idx_file=idx,
num_threads=2, local_rank=local_rank, dali_aug=dali_aug)
rank, world_size = get_dist_info()
train_sampler = DistributedSampler(
train_set, num_replicas=world_size, rank=rank, shuffle=True, seed=seed)
if seed is None:
init_fn = None
else:
init_fn = partial(worker_init_fn, num_workers=num_workers, rank=rank, seed=seed)
train_loader = DataLoaderX(
local_rank=local_rank,
dataset=train_set,
batch_size=batch_size,
sampler=train_sampler,
num_workers=num_workers,
pin_memory=True,
drop_last=True,
worker_init_fn=init_fn,
)
return train_loader
class BackgroundGenerator(threading.Thread):
def __init__(self, generator, local_rank, max_prefetch=6):
super(BackgroundGenerator, self).__init__()
self.queue = Queue.Queue(max_prefetch)
self.generator = generator
self.local_rank = local_rank
self.daemon = True
self.start()
def run(self):
torch.cuda.set_device(self.local_rank)
for item in self.generator:
self.queue.put(item)
self.queue.put(None)
def next(self):
next_item = self.queue.get()
if next_item is None:
raise StopIteration
return next_item
def __next__(self):
return self.next()
def __iter__(self):
return self
class DataLoaderX(DataLoader):
def __init__(self, local_rank, **kwargs):
super(DataLoaderX, self).__init__(**kwargs)
self.stream = torch.cuda.Stream(local_rank)
self.local_rank = local_rank
def __iter__(self):
self.iter = super(DataLoaderX, self).__iter__()
self.iter = BackgroundGenerator(self.iter, self.local_rank)
self.preload()
return self
def preload(self):
self.batch = next(self.iter, None)
if self.batch is None:
return None
with torch.cuda.stream(self.stream):
for k in range(len(self.batch)):
self.batch[k] = self.batch[k].to(device=self.local_rank, non_blocking=True)
def __next__(self):
torch.cuda.current_stream().wait_stream(self.stream)
batch = self.batch
if batch is None:
raise StopIteration
self.preload()
return batch
class MXFaceDataset(Dataset):
def __init__(self, root_dir, local_rank):
super(MXFaceDataset, self).__init__()
self.transform = transforms.Compose(
[transforms.ToPILImage(),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]),
])
self.root_dir = root_dir
self.local_rank = local_rank
path_imgrec = os.path.join(root_dir, 'train.rec')
path_imgidx = os.path.join(root_dir, 'train.idx')
self.imgrec = mx.recordio.MXIndexedRecordIO(path_imgidx, path_imgrec, 'r')
s = self.imgrec.read_idx(0)
header, _ = mx.recordio.unpack(s)
if header.flag > 0:
self.header0 = (int(header.label[0]), int(header.label[1]))
self.imgidx = np.array(range(1, int(header.label[0])))
else:
self.imgidx = np.array(list(self.imgrec.keys))
def __getitem__(self, index):
idx = self.imgidx[index]
s = self.imgrec.read_idx(idx)
header, img = mx.recordio.unpack(s)
label = header.label
if not isinstance(label, numbers.Number):
label = label[0]
label = torch.tensor(label, dtype=torch.long)
sample = mx.image.imdecode(img).asnumpy()
if self.transform is not None:
sample = self.transform(sample)
return sample, label
def __len__(self):
return len(self.imgidx)
class SyntheticDataset(Dataset):
def __init__(self):
super(SyntheticDataset, self).__init__()
img = np.random.randint(0, 255, size=(112, 112, 3), dtype=np.int32)
img = np.transpose(img, (2, 0, 1))
img = torch.from_numpy(img).squeeze(0).float()
img = ((img / 255) - 0.5) / 0.5
self.img = img
self.label = 1
def __getitem__(self, index):
return self.img, self.label
def __len__(self):
return 1000000
def dali_data_iter(
batch_size: int, rec_file: str, idx_file: str, num_threads: int,
initial_fill=32768, random_shuffle=True,
prefetch_queue_depth=1, local_rank=0, name="reader",
mean=(127.5, 127.5, 127.5),
std=(127.5, 127.5, 127.5),
dali_aug=False
):
"""
Parameters:
----------
initial_fill: int
Size of the buffer that is used for shuffling. If random_shuffle is False, this parameter is ignored.
"""
rank: int = distributed.get_rank()
world_size: int = distributed.get_world_size()
import nvidia.dali.fn as fn
import nvidia.dali.types as types
from nvidia.dali.pipeline import Pipeline
from nvidia.dali.plugin.pytorch import DALIClassificationIterator
def dali_random_resize(img, resize_size, image_size=112):
img = fn.resize(img, resize_x=resize_size, resize_y=resize_size)
img = fn.resize(img, size=(image_size, image_size))
return img
def dali_random_gaussian_blur(img, window_size):
img = fn.gaussian_blur(img, window_size=window_size * 2 + 1)
return img
def dali_random_gray(img, prob_gray):
saturate = fn.random.coin_flip(probability=1 - prob_gray)
saturate = fn.cast(saturate, dtype=types.FLOAT)
img = fn.hsv(img, saturation=saturate)
return img
def dali_random_hsv(img, hue, saturation):
img = fn.hsv(img, hue=hue, saturation=saturation)
return img
def multiplexing(condition, true_case, false_case):
neg_condition = condition ^ True
return condition * true_case + neg_condition * false_case
condition_resize = fn.random.coin_flip(probability=0.1)
size_resize = fn.random.uniform(range=(int(112 * 0.5), int(112 * 0.8)), dtype=types.FLOAT)
condition_blur = fn.random.coin_flip(probability=0.2)
window_size_blur = fn.random.uniform(range=(1, 2), dtype=types.INT32)
condition_flip = fn.random.coin_flip(probability=0.5)
condition_hsv = fn.random.coin_flip(probability=0.2)
hsv_hue = fn.random.uniform(range=(0., 20.), dtype=types.FLOAT)
hsv_saturation = fn.random.uniform(range=(1., 1.2), dtype=types.FLOAT)
pipe = Pipeline(
batch_size=batch_size, num_threads=num_threads,
device_id=local_rank, prefetch_queue_depth=prefetch_queue_depth, )
condition_flip = fn.random.coin_flip(probability=0.5)
with pipe:
jpegs, labels = fn.readers.mxnet(
path=rec_file, index_path=idx_file, initial_fill=initial_fill,
num_shards=world_size, shard_id=rank,
random_shuffle=random_shuffle, pad_last_batch=False, name=name)
images = fn.decoders.image(jpegs, device="mixed", output_type=types.RGB)
if dali_aug:
images = fn.cast(images, dtype=types.UINT8)
images = multiplexing(condition_resize, dali_random_resize(images, size_resize, image_size=112), images)
images = multiplexing(condition_blur, dali_random_gaussian_blur(images, window_size_blur), images)
images = multiplexing(condition_hsv, dali_random_hsv(images, hsv_hue, hsv_saturation), images)
images = dali_random_gray(images, 0.1)
images = fn.crop_mirror_normalize(
images, dtype=types.FLOAT, mean=mean, std=std, mirror=condition_flip)
pipe.set_outputs(images, labels)
pipe.build()
return DALIWarper(DALIClassificationIterator(pipelines=[pipe], reader_name=name, ))
@torch.no_grad()
class DALIWarper(object):
def __init__(self, dali_iter):
self.iter = dali_iter
def __next__(self):
data_dict = self.iter.__next__()[0]
tensor_data = data_dict['data'].cuda()
tensor_label: torch.Tensor = data_dict['label'].cuda().long()
tensor_label.squeeze_()
return tensor_data, tensor_label
def __iter__(self):
return self
def reset(self):
self.iter.reset()
ip_list=("ip1" "ip2" "ip3" "ip4" "ip5")
config=wf42m_pfc02_vit_h.py
for((node_rank=0;node_rank<${#ip_list[*]};node_rank++));
do
ssh root@${ip_list[node_rank]} "cd `pwd`;PATH=$PATH \
CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
torchrun \
--nproc_per_node=8 \
--nnodes=${#ip_list[*]} \
--node_rank=$node_rank \
--master_addr=${ip_list[0]} \
--master_port=22345 train_v2.py configs/$config" &
done
FROM image.sourcefind.cn:5000/dcu/admin/base/pytorch:1.13.1-centos7.6-dtk23.10-py310
RUN source /opt/dtk/env.sh
## Eval on ICCV2021-MFR
coming soon.
## Eval IJBC
You can eval ijbc with pytorch or onnx.
1. Eval IJBC With Onnx
```shell
CUDA_VISIBLE_DEVICES=0 python onnx_ijbc.py --model-root ms1mv3_arcface_r50 --image-path IJB_release/IJBC --result-dir ms1mv3_arcface_r50
```
2. Eval IJBC With Pytorch
```shell
CUDA_VISIBLE_DEVICES=0,1 python eval_ijbc.py \
--model-prefix ms1mv3_arcface_r50/backbone.pth \
--image-path IJB_release/IJBC \
--result-dir ms1mv3_arcface_r50 \
--batch-size 128 \
--job ms1mv3_arcface_r50 \
--target IJBC \
--network iresnet50
```
## Inference
```shell
python inference.py --weight ms1mv3_arcface_r50/backbone.pth --network r50
```
## Result
| Datasets | Backbone | **MFR-ALL** | IJB-C(1E-4) | IJB-C(1E-5) |
|:---------------|:--------------------|:------------|:------------|:------------|
| WF12M-PFC-0.05 | r100 | 94.05 | 97.51 | 95.75 |
| WF12M-PFC-0.1 | r100 | 94.49 | 97.56 | 95.92 |
| WF12M-PFC-0.2 | r100 | 94.75 | 97.60 | 95.90 |
| WF12M-PFC-0.3 | r100 | 94.71 | 97.64 | 96.01 |
| WF12M | r100 | 94.69 | 97.59 | 95.97 |
\ No newline at end of file
# Installation
### [Torch v1.11.0](https://pytorch.org/get-started/previous-versions/#v1110)
#### Linux and Windows
- CUDA 11.3
```shell
pip install torch==1.11.0+cu113 torchvision==0.12.0+cu113 torchaudio==0.11.0 --extra-index-url https://download.pytorch.org/whl/cu113
```
- CUDA 10.2
```shell
pip install torch==1.11.0+cu102 torchvision==0.12.0+cu102 torchaudio==0.11.0 --extra-index-url https://download.pytorch.org/whl/cu102
```
### [Torch v1.9.0](https://pytorch.org/get-started/previous-versions/#v190)
#### Linux and Windows
- CUDA 11.1
```shell
pip install torch==1.9.0+cu111 torchvision==0.10.0+cu111 torchaudio==0.9.0 -f https://download.pytorch.org/whl/torch_stable.html
```
- CUDA 10.2
```shell
pip install torch==1.9.0+cu102 torchvision==0.10.0+cu102 torchaudio==0.9.0 -f https://download.pytorch.org/whl/torch_stable.html
```
# Installation
## Prerequisites
1. Linux x64.
2. NVIDIA Driver supporting CUDA 10.0 or later (i.e., 410.48 or later driver releases).
3. (Optional) One or more of the following deep learning frameworks:
* [MXNet 1.3](http://mxnet.incubator.apache.org/) `mxnet-cu100` or later.
* [PyTorch 0.4](https://pytorch.org/) or later.
* [TensorFlow 1.7](https://www.tensorflow.org/) or later.
## DALI in NGC Containers
DALI is preinstalled in the TensorFlow, PyTorch, and MXNet containers in versions 18.07 and later on NVIDIA GPU Cloud.
## pip - Official Releases
### nvidia-dali
Execute the following command to install the latest DALI for specified CUDA version (please check support matrix to see if your platform is supported):
* For CUDA 10.2:
```bash
pip install --extra-index-url https://developer.download.nvidia.com/compute/redist --upgrade nvidia-dali-cuda102
```
* For CUDA 11.0:
```bash
pip install --extra-index-url https://developer.download.nvidia.com/compute/redist --upgrade nvidia-dali-cuda110
```
> Note: CUDA 11.0 build uses CUDA toolkit enhanced compatibility. It is built with the latest CUDA 11.x toolkit while it can run on the latest, stable CUDA 11.0 capable drivers (450.80 or later). Using the latest driver may enable additional functionality. More details can be found in [enhanced CUDA compatibility guide](https://docs.nvidia.com/deploy/cuda-compatibility/index.html#enhanced-compat-minor-releases).
> Note: Please always use the latest version of pip available (at least >= 19.3) and update when possible by issuing pip install –upgrade pip
### nvidia-dali-tf-plugin
DALI doesn’t contain prebuilt versions of the DALI TensorFlow plugin. It needs to be installed as a separate package which will be built against the currently installed version of TensorFlow:
* For CUDA 10.2:
```bash
pip install --extra-index-url https://developer.download.nvidia.com/compute/redist --upgrade nvidia-dali-tf-plugin-cuda102
```
* For CUDA 11.0:
```bash
pip install --extra-index-url https://developer.download.nvidia.com/compute/redist --upgrade nvidia-dali-tf-plugin-cuda110
```
Installing this package will install `nvidia-dali-cudaXXX` and its dependencies, if they are not already installed. The package `tensorflow-gpu` must be installed before attempting to install `nvidia-dali-tf-plugin-cudaXXX`.
> Note: The packages `nvidia-dali-tf-plugin-cudaXXX` and `nvidia-dali-cudaXXX` should be in exactly the same version. Therefore, installing the latest `nvidia-dali-tf-plugin-cudaXXX`, will replace any older `nvidia-dali-cudaXXX` version already installed. To work with older versions of DALI, provide the version explicitly to the `pip install` command.
### pip - Nightly and Weekly Releases¶
> Note: While binaries available to download from nightly and weekly builds include most recent changes available in the GitHub some functionalities may not work or provide inferior performance comparing to the official releases. Those builds are meant for the early adopters seeking for the most recent version available and being ready to boldly go where no man has gone before.
> Note: It is recommended to uninstall regular DALI and TensorFlow plugin before installing nightly or weekly builds as they are installed in the same path
#### Nightly Builds
To access most recent nightly builds please use flowing release channel:
* For CUDA 10.2:
```bash
pip install --extra-index-url https://developer.download.nvidia.com/compute/redist/nightly --upgrade nvidia-dali-nightly-cuda102
```
```
pip install --extra-index-url https://developer.download.nvidia.com/compute/redist/nightly --upgrade nvidia-dali-tf-plugin-nightly-cuda102
```
* For CUDA 11.0:
```bash
pip install --extra-index-url https://developer.download.nvidia.com/compute/redist/nightly --upgrade nvidia-dali-nightly-cuda110
```
```bash
pip install --extra-index-url https://developer.download.nvidia.com/compute/redist/nightly --upgrade nvidia-dali-tf-plugin-nightly-cuda110
```
#### Weekly Builds
Also, there is a weekly release channel with more thorough testing. To access most recent weekly builds please use the following release channel (available only for CUDA 11):
```bash
pip install --extra-index-url https://developer.download.nvidia.com/compute/redist/weekly --upgrade nvidia-dali-weekly-cuda110
```
```bash
pip install --extra-index-url https://developer.download.nvidia.com/compute/redist/weekly --upgrade nvidia-dali-tf-plugin-week
```
---
### For more information about Dali and installation, please refer to [DALI documentation](https://docs.nvidia.com/deeplearning/dali/user-guide/docs/installation.html).
Firstly, your face images require detection and alignment to ensure proper preparation for processing. Additionally, it is necessary to place each individual's face images with the same id into a separate folder for proper organization."
```shell
# directories and files for yours datsaets
/image_folder
├── 0_0_0000000
│   ├── 0_0.jpg
│   ├── 0_1.jpg
│   ├── 0_2.jpg
│   ├── 0_3.jpg
│   └── 0_4.jpg
├── 0_0_0000001
│   ├── 0_5.jpg
│   ├── 0_6.jpg
│   ├── 0_7.jpg
│   ├── 0_8.jpg
│   └── 0_9.jpg
├── 0_0_0000002
│   ├── 0_10.jpg
│   ├── 0_11.jpg
│   ├── 0_12.jpg
│   ├── 0_13.jpg
│   ├── 0_14.jpg
│   ├── 0_15.jpg
│   ├── 0_16.jpg
│   └── 0_17.jpg
├── 0_0_0000003
│   ├── 0_18.jpg
│   ├── 0_19.jpg
│   └── 0_20.jpg
├── 0_0_0000004
# 0) Dependencies installation
pip install opencv-python
apt-get update
apt-get install ffmpeg libsm6 libxext6 -y
# 1) create train.lst using follow command
python -m mxnet.tools.im2rec --list --recursive train image_folder
# 2) create train.rec and train.idx using train.lst using following command
python -m mxnet.tools.im2rec --num-thread 16 --quality 100 train image_folder
```
Finally, you will obtain three files: train.lst, train.rec, and train.idx, where train.idx and train.rec are utilized for training.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment