update

fastreid/__init__.py

+# encoding: utf-8
+"""
+@author:  liaoxingyu
+@contact: sherlockliao01@gmail.com
+"""
+
+
+__version__ = "1.3"

fastreid/config/__init__.py

+# encoding: utf-8
+"""
+@author:  l1aoxingyu
+@contact: sherlockliao01@gmail.com
+"""
+
+from .config import CfgNode, get_cfg, global_cfg, set_global_cfg, configurable
+
+__all__ = [
+    'CfgNode',
+    'get_cfg',
+    'global_cfg',
+    'set_global_cfg',
+    'configurable'
+]

fastreid/config/config.py

+# encoding: utf-8
+"""
+@author:  l1aoxingyu
+@contact: sherlockliao01@gmail.com
+"""
+
+import functools
+import inspect
+import logging
+import os
+from typing import Any
+
+import yaml
+from yacs.config import CfgNode as _CfgNode
+
+from ..utils.file_io import PathManager
+
+BASE_KEY = "_BASE_"
+
+
+class CfgNode(_CfgNode):
+    """
+    Our own extended version of :class:`yacs.config.CfgNode`.
+    It contains the following extra features:
+    1. The :meth:`merge_from_file` method supports the "_BASE_" key,
+       which allows the new CfgNode to inherit all the attributes from the
+       base configuration file.
+    2. Keys that start with "COMPUTED_" are treated as insertion-only
+       "computed" attributes. They can be inserted regardless of whether
+       the CfgNode is frozen or not.
+    3. With "allow_unsafe=True", it supports pyyaml tags that evaluate
+       expressions in config. See examples in
+       https://pyyaml.org/wiki/PyYAMLDocumentation#yaml-tags-and-python-types
+       Note that this may lead to arbitrary code execution: you must not
+       load a config file from untrusted sources before manually inspecting
+       the content of the file.
+    """
+
+    @staticmethod
+    def load_yaml_with_base(filename: str, allow_unsafe: bool = False):
+        """
+        Just like `yaml.load(open(filename))`, but inherit attributes from its
+            `_BASE_`.
+        Args:
+            filename (str): the file name of the current config. Will be used to
+                find the base config file.
+            allow_unsafe (bool): whether to allow loading the config file with
+                `yaml.unsafe_load`.
+        Returns:
+            (dict): the loaded yaml
+        """
+        with PathManager.open(filename, "r") as f:
+            try:
+                cfg = yaml.safe_load(f)
+            except yaml.constructor.ConstructorError:
+                if not allow_unsafe:
+                    raise
+                logger = logging.getLogger(__name__)
+                logger.warning(
+                    "Loading config {} with yaml.unsafe_load. Your machine may "
+                    "be at risk if the file contains malicious content.".format(
+                        filename
+                    )
+                )
+                f.close()
+                with open(filename, "r") as f:
+                    cfg = yaml.unsafe_load(f)
+
+        def merge_a_into_b(a, b):
+            # merge dict a into dict b. values in a will overwrite b.
+            for k, v in a.items():
+                if isinstance(v, dict) and k in b:
+                    assert isinstance(
+                        b[k], dict
+                    ), "Cannot inherit key '{}' from base!".format(k)
+                    merge_a_into_b(v, b[k])
+                else:
+                    b[k] = v
+
+        if BASE_KEY in cfg:
+            base_cfg_file = cfg[BASE_KEY]
+            if base_cfg_file.startswith("~"):
+                base_cfg_file = os.path.expanduser(base_cfg_file)
+            if not any(
+                    map(base_cfg_file.startswith, ["/", "https://", "http://"])
+            ):
+                # the path to base cfg is relative to the config file itself.
+                base_cfg_file = os.path.join(
+                    os.path.dirname(filename), base_cfg_file
+                )
+            base_cfg = CfgNode.load_yaml_with_base(
+                base_cfg_file, allow_unsafe=allow_unsafe
+            )
+            del cfg[BASE_KEY]
+
+            merge_a_into_b(cfg, base_cfg)
+            return base_cfg
+        return cfg
+
+    def merge_from_file(self, cfg_filename: str, allow_unsafe: bool = False):
+        """
+        Merge configs from a given yaml file.
+        Args:
+            cfg_filename: the file name of the yaml config.
+            allow_unsafe: whether to allow loading the config file with
+                `yaml.unsafe_load`.
+        """
+        loaded_cfg = CfgNode.load_yaml_with_base(
+            cfg_filename, allow_unsafe=allow_unsafe
+        )
+        loaded_cfg = type(self)(loaded_cfg)
+        self.merge_from_other_cfg(loaded_cfg)
+
+    # Forward the following calls to base, but with a check on the BASE_KEY.
+    def merge_from_other_cfg(self, cfg_other):
+        """
+        Args:
+            cfg_other (CfgNode): configs to merge from.
+        """
+        assert (
+                BASE_KEY not in cfg_other
+        ), "The reserved key '{}' can only be used in files!".format(BASE_KEY)
+        return super().merge_from_other_cfg(cfg_other)
+
+    def merge_from_list(self, cfg_list: list):
+        """
+        Args:
+            cfg_list (list): list of configs to merge from.
+        """
+        keys = set(cfg_list[0::2])
+        assert (
+                BASE_KEY not in keys
+        ), "The reserved key '{}' can only be used in files!".format(BASE_KEY)
+        return super().merge_from_list(cfg_list)
+
+    def __setattr__(self, name: str, val: Any):
+        if name.startswith("COMPUTED_"):
+            if name in self:
+                old_val = self[name]
+                if old_val == val:
+                    return
+                raise KeyError(
+                    "Computed attributed '{}' already exists "
+                    "with a different value! old={}, new={}.".format(
+                        name, old_val, val
+                    )
+                )
+            self[name] = val
+        else:
+            super().__setattr__(name, val)
+
+
+global_cfg = CfgNode()
+
+
+def get_cfg() -> CfgNode:
+    """
+    Get a copy of the default config.
+    Returns:
+        a fastreid CfgNode instance.
+    """
+    from .defaults import _C
+
+    return _C.clone()
+
+
+def set_global_cfg(cfg: CfgNode) -> None:
+    """
+    Let the global config point to the given cfg.
+    Assume that the given "cfg" has the key "KEY", after calling
+    `set_global_cfg(cfg)`, the key can be accessed by:
+    ::
+        from detectron2.config import global_cfg
+        print(global_cfg.KEY)
+    By using a hacky global config, you can access these configs anywhere,
+    without having to pass the config object or the values deep into the code.
+    This is a hacky feature introduced for quick prototyping / research exploration.
+    """
+    global global_cfg
+    global_cfg.clear()
+    global_cfg.update(cfg)
+
+
+def configurable(init_func=None, *, from_config=None):
+    """
+    Decorate a function or a class's __init__ method so that it can be called
+    with a :class:`CfgNode` object using a :func:`from_config` function that translates
+    :class:`CfgNode` to arguments.
+    Examples:
+    ::
+        # Usage 1: Decorator on __init__:
+        class A:
+            @configurable
+            def __init__(self, a, b=2, c=3):
+                pass
+            @classmethod
+            def from_config(cls, cfg):   # 'cfg' must be the first argument
+                # Returns kwargs to be passed to __init__
+                return {"a": cfg.A, "b": cfg.B}
+        a1 = A(a=1, b=2)  # regular construction
+        a2 = A(cfg)       # construct with a cfg
+        a3 = A(cfg, b=3, c=4)  # construct with extra overwrite
+        # Usage 2: Decorator on any function. Needs an extra from_config argument:
+        @configurable(from_config=lambda cfg: {"a: cfg.A, "b": cfg.B})
+        def a_func(a, b=2, c=3):
+            pass
+        a1 = a_func(a=1, b=2)  # regular call
+        a2 = a_func(cfg)       # call with a cfg
+        a3 = a_func(cfg, b=3, c=4)  # call with extra overwrite
+    Args:
+        init_func (callable): a class's ``__init__`` method in usage 1. The
+            class must have a ``from_config`` classmethod which takes `cfg` as
+            the first argument.
+        from_config (callable): the from_config function in usage 2. It must take `cfg`
+            as its first argument.
+    """
+
+    def check_docstring(func):
+        if func.__module__.startswith("fastreid."):
+            assert (
+                    func.__doc__ is not None and "experimental" in func.__doc__.lower()
+            ), f"configurable {func} should be marked experimental"
+
+    if init_func is not None:
+        assert (
+                inspect.isfunction(init_func)
+                and from_config is None
+                and init_func.__name__ == "__init__"
+        ), "Incorrect use of @configurable. Check API documentation for examples."
+        check_docstring(init_func)
+
+        @functools.wraps(init_func)
+        def wrapped(self, *args, **kwargs):
+            try:
+                from_config_func = type(self).from_config
+            except AttributeError as e:
+                raise AttributeError(
+                    "Class with @configurable must have a 'from_config' classmethod."
+                ) from e
+            if not inspect.ismethod(from_config_func):
+                raise TypeError("Class with @configurable must have a 'from_config' classmethod.")
+
+            if _called_with_cfg(*args, **kwargs):
+                explicit_args = _get_args_from_config(from_config_func, *args, **kwargs)
+                init_func(self, **explicit_args)
+            else:
+                init_func(self, *args, **kwargs)
+
+        return wrapped
+
+    else:
+        if from_config is None:
+            return configurable  # @configurable() is made equivalent to @configurable
+        assert inspect.isfunction(
+            from_config
+        ), "from_config argument of configurable must be a function!"
+
+        def wrapper(orig_func):
+            check_docstring(orig_func)
+
+            @functools.wraps(orig_func)
+            def wrapped(*args, **kwargs):
+                if _called_with_cfg(*args, **kwargs):
+                    explicit_args = _get_args_from_config(from_config, *args, **kwargs)
+                    return orig_func(**explicit_args)
+                else:
+                    return orig_func(*args, **kwargs)
+
+            return wrapped
+
+        return wrapper
+
+
+def _get_args_from_config(from_config_func, *args, **kwargs):
+    """
+    Use `from_config` to obtain explicit arguments.
+    Returns:
+        dict: arguments to be used for cls.__init__
+    """
+    signature = inspect.signature(from_config_func)
+    if list(signature.parameters.keys())[0] != "cfg":
+        if inspect.isfunction(from_config_func):
+            name = from_config_func.__name__
+        else:
+            name = f"{from_config_func.__self__}.from_config"
+        raise TypeError(f"{name} must take 'cfg' as the first argument!")
+    support_var_arg = any(
+        param.kind in [param.VAR_POSITIONAL, param.VAR_KEYWORD]
+        for param in signature.parameters.values()
+    )
+    if support_var_arg:  # forward all arguments to from_config, if from_config accepts them
+        ret = from_config_func(*args, **kwargs)
+    else:
+        # forward supported arguments to from_config
+        supported_arg_names = set(signature.parameters.keys())
+        extra_kwargs = {}
+        for name in list(kwargs.keys()):
+            if name not in supported_arg_names:
+                extra_kwargs[name] = kwargs.pop(name)
+        ret = from_config_func(*args, **kwargs)
+        # forward the other arguments to __init__
+        ret.update(extra_kwargs)
+    return ret
+
+
+def _called_with_cfg(*args, **kwargs):
+    """
+    Returns:
+        bool: whether the arguments contain CfgNode and should be considered
+            forwarded to from_config.
+    """
+
+    if len(args) and isinstance(args[0], _CfgNode):
+        return True
+    if isinstance(kwargs.pop("cfg", None), _CfgNode):
+        return True
+    # `from_config`'s first argument is forced to be "cfg".
+    # So the above check covers all cases.
+    return False

fastreid/config/defaults.py

+from .config import CfgNode as CN
+
+# -----------------------------------------------------------------------------
+# Convention about Training / Test specific parameters
+# -----------------------------------------------------------------------------
+# Whenever an argument can be either used for training or for testing, the
+# corresponding name will be post-fixed by a _TRAIN for a training parameter,
+# or _TEST for a test-specific parameter.
+# For example, the number of images during training will be
+# IMAGES_PER_BATCH_TRAIN, while the number of images for testing will be
+# IMAGES_PER_BATCH_TEST
+
+# -----------------------------------------------------------------------------
+# Config definition
+# -----------------------------------------------------------------------------
+
+_C = CN()
+
+# -----------------------------------------------------------------------------
+# MODEL
+# -----------------------------------------------------------------------------
+_C.MODEL = CN()
+_C.MODEL.DEVICE = "cuda"
+_C.MODEL.META_ARCHITECTURE = "Baseline"
+
+_C.MODEL.FREEZE_LAYERS = []
+
+# MoCo memory size
+_C.MODEL.QUEUE_SIZE = 8192
+
+# ---------------------------------------------------------------------------- #
+# Backbone options
+# ---------------------------------------------------------------------------- #
+_C.MODEL.BACKBONE = CN()
+
+_C.MODEL.BACKBONE.NAME = "build_resnet_backbone"
+_C.MODEL.BACKBONE.DEPTH = "50x"
+_C.MODEL.BACKBONE.LAST_STRIDE = 1
+# Backbone feature dimension
+_C.MODEL.BACKBONE.FEAT_DIM = 2048
+# Normalization method for the convolution layers.
+_C.MODEL.BACKBONE.NORM = "BN"
+# If use IBN block in backbone
+_C.MODEL.BACKBONE.WITH_IBN = False
+# If use SE block in backbone
+_C.MODEL.BACKBONE.WITH_SE = False
+# If use Non-local block in backbone
+_C.MODEL.BACKBONE.WITH_NL = False
+# Vision Transformer options
+_C.MODEL.BACKBONE.SIE_COE = 3.0
+_C.MODEL.BACKBONE.STRIDE_SIZE = (16, 16)
+_C.MODEL.BACKBONE.DROP_PATH_RATIO = 0.1
+_C.MODEL.BACKBONE.DROP_RATIO = 0.0
+_C.MODEL.BACKBONE.ATT_DROP_RATE = 0.0
+# If use ImageNet pretrain model
+_C.MODEL.BACKBONE.PRETRAIN = False
+# Pretrain model path
+_C.MODEL.BACKBONE.PRETRAIN_PATH = ''
+
+# ---------------------------------------------------------------------------- #
+# REID HEADS options
+# ---------------------------------------------------------------------------- #
+_C.MODEL.HEADS = CN()
+_C.MODEL.HEADS.NAME = "EmbeddingHead"
+# Normalization method for the convolution layers.
+_C.MODEL.HEADS.NORM = "BN"
+# Number of identity
+_C.MODEL.HEADS.NUM_CLASSES = 0
+# Embedding dimension in head
+_C.MODEL.HEADS.EMBEDDING_DIM = 0
+# If use BNneck in embedding
+_C.MODEL.HEADS.WITH_BNNECK = False
+# Triplet feature using feature before(after) bnneck
+_C.MODEL.HEADS.NECK_FEAT = "before"  # options: before, after
+# Pooling layer type
+_C.MODEL.HEADS.POOL_LAYER = "GlobalAvgPool"
+
+# Classification layer type
+_C.MODEL.HEADS.CLS_LAYER = "Linear"  # ArcSoftmax" or "CircleSoftmax"
+
+# Margin and Scale for margin-based classification layer
+_C.MODEL.HEADS.MARGIN = 0.
+_C.MODEL.HEADS.SCALE = 1
+
+# ---------------------------------------------------------------------------- #
+# REID LOSSES options
+# ---------------------------------------------------------------------------- #
+_C.MODEL.LOSSES = CN()
+_C.MODEL.LOSSES.NAME = ("CrossEntropyLoss",)
+
+# Cross Entropy Loss options
+_C.MODEL.LOSSES.CE = CN()
+# if epsilon == 0, it means no label smooth regularization,
+# if epsilon == -1, it means adaptive label smooth regularization
+_C.MODEL.LOSSES.CE.EPSILON = 0.0
+_C.MODEL.LOSSES.CE.ALPHA = 0.2
+_C.MODEL.LOSSES.CE.SCALE = 1.0
+
+# Focal Loss options
+_C.MODEL.LOSSES.FL = CN()
+_C.MODEL.LOSSES.FL.ALPHA = 0.25
+_C.MODEL.LOSSES.FL.GAMMA = 2
+_C.MODEL.LOSSES.FL.SCALE = 1.0
+
+# Triplet Loss options
+_C.MODEL.LOSSES.TRI = CN()
+_C.MODEL.LOSSES.TRI.MARGIN = 0.3
+_C.MODEL.LOSSES.TRI.NORM_FEAT = False
+_C.MODEL.LOSSES.TRI.HARD_MINING = False
+_C.MODEL.LOSSES.TRI.SCALE = 1.0
+
+# Circle Loss options
+_C.MODEL.LOSSES.CIRCLE = CN()
+_C.MODEL.LOSSES.CIRCLE.MARGIN = 0.25
+_C.MODEL.LOSSES.CIRCLE.GAMMA = 128
+_C.MODEL.LOSSES.CIRCLE.SCALE = 1.0
+
+# Cosface Loss options
+_C.MODEL.LOSSES.COSFACE = CN()
+_C.MODEL.LOSSES.COSFACE.MARGIN = 0.25
+_C.MODEL.LOSSES.COSFACE.GAMMA = 128
+_C.MODEL.LOSSES.COSFACE.SCALE = 1.0
+
+# Path to a checkpoint file to be loaded to the model. You can find available models in the model zoo.
+_C.MODEL.WEIGHTS = ""
+
+# Values to be used for image normalization
+_C.MODEL.PIXEL_MEAN = [0.485*255, 0.456*255, 0.406*255]
+# Values to be used for image normalization
+_C.MODEL.PIXEL_STD = [0.229*255, 0.224*255, 0.225*255]
+
+# -----------------------------------------------------------------------------
+# KNOWLEDGE DISTILLATION
+# -----------------------------------------------------------------------------
+
+_C.KD = CN()
+_C.KD.MODEL_CONFIG = []
+_C.KD.MODEL_WEIGHTS = []
+_C.KD.EMA = CN({"ENABLED": False})
+_C.KD.EMA.MOMENTUM = 0.999
+
+# -----------------------------------------------------------------------------
+# INPUT
+# -----------------------------------------------------------------------------
+_C.INPUT = CN()
+# Size of the image during training
+_C.INPUT.SIZE_TRAIN = [256, 128]
+# Size of the image during test
+_C.INPUT.SIZE_TEST = [256, 128]
+
+# `True` if cropping is used for data augmentation during training
+_C.INPUT.CROP = CN({"ENABLED": False})
+# Size of the image cropped
+_C.INPUT.CROP.SIZE = [224, 224]
+# Size of the origin size cropped
+_C.INPUT.CROP.SCALE = [0.16, 1]
+# Aspect ratio of the origin aspect ratio cropped
+_C.INPUT.CROP.RATIO = [3./4., 4./3.]
+
+# Random probability for image horizontal flip
+_C.INPUT.FLIP = CN({"ENABLED": False})
+_C.INPUT.FLIP.PROB = 0.5
+
+# Value of padding size
+_C.INPUT.PADDING = CN({"ENABLED": False})
+_C.INPUT.PADDING.MODE = 'constant'
+_C.INPUT.PADDING.SIZE = 10
+
+# Random color jitter
+_C.INPUT.CJ = CN({"ENABLED": False})
+_C.INPUT.CJ.PROB = 0.5
+_C.INPUT.CJ.BRIGHTNESS = 0.15
+_C.INPUT.CJ.CONTRAST = 0.15
+_C.INPUT.CJ.SATURATION = 0.1
+_C.INPUT.CJ.HUE = 0.1
+
+# Random Affine
+_C.INPUT.AFFINE = CN({"ENABLED": False})
+
+# Auto augmentation
+_C.INPUT.AUTOAUG = CN({"ENABLED": False})
+_C.INPUT.AUTOAUG.PROB = 0.0
+
+# Augmix augmentation
+_C.INPUT.AUGMIX = CN({"ENABLED": False})
+_C.INPUT.AUGMIX.PROB = 0.0
+
+# Random Erasing
+_C.INPUT.REA = CN({"ENABLED": False})
+_C.INPUT.REA.PROB = 0.5
+_C.INPUT.REA.VALUE = [0.485*255, 0.456*255, 0.406*255]
+# Random Patch
+_C.INPUT.RPT = CN({"ENABLED": False})
+_C.INPUT.RPT.PROB = 0.5
+
+# -----------------------------------------------------------------------------
+# Dataset
+# -----------------------------------------------------------------------------
+_C.DATASETS = CN()
+# List of the dataset names for training
+_C.DATASETS.NAMES = ("Market1501",)
+# List of the dataset names for testing
+_C.DATASETS.TESTS = ("Market1501",)
+# Combine trainset and testset joint training
+_C.DATASETS.COMBINEALL = False
+
+# -----------------------------------------------------------------------------
+# DataLoader
+# -----------------------------------------------------------------------------
+_C.DATALOADER = CN()
+# Options: TrainingSampler, NaiveIdentitySampler, BalancedIdentitySampler
+_C.DATALOADER.SAMPLER_TRAIN = "TrainingSampler"
+# Number of instance for each person
+_C.DATALOADER.NUM_INSTANCE = 4
+_C.DATALOADER.NUM_WORKERS = 8
+
+# For set re-weight
+_C.DATALOADER.SET_WEIGHT = []
+
+# ---------------------------------------------------------------------------- #
+# Solver
+# ---------------------------------------------------------------------------- #
+_C.SOLVER = CN()
+
+# AUTOMATIC MIXED PRECISION
+_C.SOLVER.AMP = CN({"ENABLED": False})
+
+# Optimizer
+_C.SOLVER.OPT = "Adam"
+
+_C.SOLVER.MAX_EPOCH = 120
+
+_C.SOLVER.BASE_LR = 3e-4
+
+# This LR is applied to the last classification layer if
+# you want to 10x higher than BASE_LR.
+_C.SOLVER.HEADS_LR_FACTOR = 1.
+
+_C.SOLVER.MOMENTUM = 0.9
+_C.SOLVER.NESTEROV = False
+
+_C.SOLVER.WEIGHT_DECAY = 0.0005
+# The weight decay that's applied to parameters of normalization layers
+# (typically the affine transformation)
+_C.SOLVER.WEIGHT_DECAY_NORM = 0.0005
+
+# The previous detection code used a 2x higher LR and 0 WD for bias.
+# This is not useful (at least for recent models). You should avoid
+# changing these and they exists only to reproduce previous model
+# training if desired.
+_C.SOLVER.BIAS_LR_FACTOR = 1.0
+_C.SOLVER.WEIGHT_DECAY_BIAS = _C.SOLVER.WEIGHT_DECAY
+
+# Multi-step learning rate options
+_C.SOLVER.SCHED = "MultiStepLR"
+
+_C.SOLVER.DELAY_EPOCHS = 0
+
+_C.SOLVER.GAMMA = 0.1
+_C.SOLVER.STEPS = [30, 55]
+
+# Cosine annealing learning rate options
+_C.SOLVER.ETA_MIN_LR = 1e-7
+
+# Warmup options
+_C.SOLVER.WARMUP_FACTOR = 0.1
+_C.SOLVER.WARMUP_ITERS = 1000
+_C.SOLVER.WARMUP_METHOD = "linear"
+
+# Backbone freeze iters
+_C.SOLVER.FREEZE_ITERS = 0
+
+_C.SOLVER.CHECKPOINT_PERIOD = 20
+
+# Number of images per batch across all machines.
+# This is global, so if we have 8 GPUs and IMS_PER_BATCH = 256, each GPU will
+# see 32 images per batch
+_C.SOLVER.IMS_PER_BATCH = 64
+
+# Gradient clipping
+_C.SOLVER.CLIP_GRADIENTS = CN({"ENABLED": False})
+# Type of gradient clipping, currently 2 values are supported:
+# - "value": the absolute values of elements of each gradients are clipped
+# - "norm": the norm of the gradient for each parameter is clipped thus
+#   affecting all elements in the parameter
+_C.SOLVER.CLIP_GRADIENTS.CLIP_TYPE = "norm"
+# Maximum absolute value used for clipping gradients
+_C.SOLVER.CLIP_GRADIENTS.CLIP_VALUE = 5.0
+# Floating point number p for L-p norm to be used with the "norm"
+# gradient clipping type; for L-inf, please specify .inf
+_C.SOLVER.CLIP_GRADIENTS.NORM_TYPE = 2.0
+
+_C.TEST = CN()
+
+_C.TEST.EVAL_PERIOD = 20
+
+# Number of images per batch across all machines.
+_C.TEST.IMS_PER_BATCH = 64
+_C.TEST.METRIC = "cosine"
+_C.TEST.ROC = CN({"ENABLED": False})
+_C.TEST.FLIP = CN({"ENABLED": False})
+
+# Average query expansion
+_C.TEST.AQE = CN({"ENABLED": False})
+_C.TEST.AQE.ALPHA = 3.0
+_C.TEST.AQE.QE_TIME = 1
+_C.TEST.AQE.QE_K = 5
+
+# Re-rank
+_C.TEST.RERANK = CN({"ENABLED": False})
+_C.TEST.RERANK.K1 = 20
+_C.TEST.RERANK.K2 = 6
+_C.TEST.RERANK.LAMBDA = 0.3
+
+# Precise batchnorm
+_C.TEST.PRECISE_BN = CN({"ENABLED": False})
+_C.TEST.PRECISE_BN.DATASET = 'Market1501'
+_C.TEST.PRECISE_BN.NUM_ITER = 300
+
+# ---------------------------------------------------------------------------- #
+# Misc options
+# ---------------------------------------------------------------------------- #
+_C.OUTPUT_DIR = "logs/"
+
+# Benchmark different cudnn algorithms.
+# If input images have very different sizes, this option will have large overhead
+# for about 10k iterations. It usually hurts total time, but can benefit for certain models.
+# If input images have the same or similar sizes, benchmark is often helpful.
+_C.CUDNN_BENCHMARK = False

fastreid/data/__init__.py

+# encoding: utf-8
+"""
+@author:  sherlock
+@contact: sherlockliao01@gmail.com
+"""
+
+from . import transforms  # isort:skip
+from .build import (
+    build_reid_train_loader,
+    build_reid_test_loader
+)
+from .common import CommDataset
+
+# ensure the builtin datasets are registered
+from . import datasets, samplers  # isort:skip
+
+__all__ = [k for k in globals().keys() if not k.startswith("_")]

fastreid/data/build.py

+# encoding: utf-8
+"""
+@author:  l1aoxingyu
+@contact: sherlockliao01@gmail.com
+"""
+
+import logging
+import os
+
+import torch
+TORCH_MAJOR = int(torch.__version__.split('.')[0])
+TORCH_MINOR = int(torch.__version__.split('.')[1])
+
+if TORCH_MAJOR == 1 and TORCH_MINOR < 8:
+    from torch._six import string_classes
+else:
+    string_classes = str
+
+from collections import Mapping
+
+from fastreid.config import configurable
+from fastreid.utils import comm
+from . import samplers
+from .common import CommDataset
+from .data_utils import DataLoaderX
+from .datasets import DATASET_REGISTRY
+from .transforms import build_transforms
+
+__all__ = [
+    "build_reid_train_loader",
+    "build_reid_test_loader"
+]
+
+_root = os.getenv("FASTREID_DATASETS", "datasets")
+
+
+def _train_loader_from_config(cfg, *, train_set=None, transforms=None, sampler=None, **kwargs):
+    if transforms is None:
+        transforms = build_transforms(cfg, is_train=True)
+
+    if train_set is None:
+        train_items = list()
+        for d in cfg.DATASETS.NAMES:
+            data = DATASET_REGISTRY.get(d)(root=_root, **kwargs)
+            if comm.is_main_process():
+                data.show_train()
+            train_items.extend(data.train)
+
+        train_set = CommDataset(train_items, transforms, relabel=True)
+
+    if sampler is None:
+        sampler_name = cfg.DATALOADER.SAMPLER_TRAIN
+        num_instance = cfg.DATALOADER.NUM_INSTANCE
+        mini_batch_size = cfg.SOLVER.IMS_PER_BATCH // comm.get_world_size()
+
+        logger = logging.getLogger(__name__)
+        logger.info("Using training sampler {}".format(sampler_name))
+        if sampler_name == "TrainingSampler":
+            sampler = samplers.TrainingSampler(len(train_set))
+        elif sampler_name == "NaiveIdentitySampler":
+            sampler = samplers.NaiveIdentitySampler(train_set.img_items, mini_batch_size, num_instance)
+        elif sampler_name == "BalancedIdentitySampler":
+            sampler = samplers.BalancedIdentitySampler(train_set.img_items, mini_batch_size, num_instance)
+        elif sampler_name == "SetReWeightSampler":
+            set_weight = cfg.DATALOADER.SET_WEIGHT
+            sampler = samplers.SetReWeightSampler(train_set.img_items, mini_batch_size, num_instance, set_weight)
+        elif sampler_name == "ImbalancedDatasetSampler":
+            sampler = samplers.ImbalancedDatasetSampler(train_set.img_items)
+        else:
+            raise ValueError("Unknown training sampler: {}".format(sampler_name))
+
+    return {
+        "train_set": train_set,
+        "sampler": sampler,
+        "total_batch_size": cfg.SOLVER.IMS_PER_BATCH,
+        "num_workers": cfg.DATALOADER.NUM_WORKERS,
+    }
+
+
+@configurable(from_config=_train_loader_from_config)
+def build_reid_train_loader(
+        train_set, *, sampler=None, total_batch_size, num_workers=0,
+):
+    """
+    Build a dataloader for object re-identification with some default features.
+    This interface is experimental.
+
+    Returns:
+        torch.utils.data.DataLoader: a dataloader.
+    """
+
+    mini_batch_size = total_batch_size // comm.get_world_size()
+
+    batch_sampler = torch.utils.data.sampler.BatchSampler(sampler, mini_batch_size, True)
+
+    train_loader = DataLoaderX(
+        comm.get_local_rank(),
+        dataset=train_set,
+        num_workers=num_workers,
+        batch_sampler=batch_sampler,
+        collate_fn=fast_batch_collator,
+        pin_memory=True,
+    )
+
+    return train_loader
+
+
+def _test_loader_from_config(cfg, *, dataset_name=None, test_set=None, num_query=0, transforms=None, **kwargs):
+    if transforms is None:
+        transforms = build_transforms(cfg, is_train=False)
+
+    if test_set is None:
+        assert dataset_name is not None, "dataset_name must be explicitly passed in when test_set is not provided"
+        data = DATASET_REGISTRY.get(dataset_name)(root=_root, **kwargs)
+        if comm.is_main_process():
+            data.show_test()
+        test_items = data.query + data.gallery
+        test_set = CommDataset(test_items, transforms, relabel=False)
+
+        # Update query number
+        num_query = len(data.query)
+
+    return {
+        "test_set": test_set,
+        "test_batch_size": cfg.TEST.IMS_PER_BATCH,
+        "num_query": num_query,
+    }
+
+
+@configurable(from_config=_test_loader_from_config)
+def build_reid_test_loader(test_set, test_batch_size, num_query, num_workers=4):
+    """
+    Similar to `build_reid_train_loader`. This sampler coordinates all workers to produce
+    the exact set of all samples
+    This interface is experimental.
+
+    Args:
+        test_set:
+        test_batch_size:
+        num_query:
+        num_workers:
+
+    Returns:
+        DataLoader: a torch DataLoader, that loads the given reid dataset, with
+        the test-time transformation.
+
+    Examples:
+    ::
+        data_loader = build_reid_test_loader(test_set, test_batch_size, num_query)
+        # or, instantiate with a CfgNode:
+        data_loader = build_reid_test_loader(cfg, "my_test")
+    """
+
+    mini_batch_size = test_batch_size // comm.get_world_size()
+    data_sampler = samplers.InferenceSampler(len(test_set))
+    batch_sampler = torch.utils.data.BatchSampler(data_sampler, mini_batch_size, False)
+    test_loader = DataLoaderX(
+        comm.get_local_rank(),
+        dataset=test_set,
+        batch_sampler=batch_sampler,
+        num_workers=num_workers,  # save some memory
+        collate_fn=fast_batch_collator,
+        pin_memory=True,
+    )
+    return test_loader, num_query
+
+
+def trivial_batch_collator(batch):
+    """
+    A batch collator that does nothing.
+    """
+    return batch
+
+
+def fast_batch_collator(batched_inputs):
+    """
+    A simple batch collator for most common reid tasks
+    """
+    elem = batched_inputs[0]
+    if isinstance(elem, torch.Tensor):
+        out = torch.zeros((len(batched_inputs), *elem.size()), dtype=elem.dtype)
+        for i, tensor in enumerate(batched_inputs):
+            out[i] += tensor
+        return out
+
+    elif isinstance(elem, Mapping):
+        return {key: fast_batch_collator([d[key] for d in batched_inputs]) for key in elem}
+
+    elif isinstance(elem, float):
+        return torch.tensor(batched_inputs, dtype=torch.float64)
+    elif isinstance(elem, int):
+        return torch.tensor(batched_inputs)
+    elif isinstance(elem, string_classes):
+        return batched_inputs

fastreid/data/common.py

+# encoding: utf-8
+"""
+@author:  liaoxingyu
+@contact: sherlockliao01@gmail.com
+"""
+
+from torch.utils.data import Dataset
+
+from .data_utils import read_image
+
+
+class CommDataset(Dataset):
+    """Image Person ReID Dataset"""
+
+    def __init__(self, img_items, transform=None, relabel=True):
+        self.img_items = img_items
+        self.transform = transform
+        self.relabel = relabel
+
+        pid_set = set()
+        cam_set = set()
+        for i in img_items:
+            pid_set.add(i[1])
+            cam_set.add(i[2])
+
+        self.pids = sorted(list(pid_set))
+        self.cams = sorted(list(cam_set))
+        if relabel:
+            self.pid_dict = dict([(p, i) for i, p in enumerate(self.pids)])
+            self.cam_dict = dict([(p, i) for i, p in enumerate(self.cams)])
+
+    def __len__(self):
+        return len(self.img_items)
+
+    def __getitem__(self, index):
+        img_item = self.img_items[index]
+        img_path = img_item[0]
+        pid = img_item[1]
+        camid = img_item[2]
+        img = read_image(img_path)
+        if self.transform is not None: img = self.transform(img)
+        if self.relabel:
+            pid = self.pid_dict[pid]
+            camid = self.cam_dict[camid]
+        return {
+            "images": img,
+            "targets": pid,
+            "camids": camid,
+            "img_paths": img_path,
+        }
+
+    @property
+    def num_classes(self):
+        return len(self.pids)
+
+    @property
+    def num_cameras(self):
+        return len(self.cams)

fastreid/data/data_utils.py

+# encoding: utf-8
+"""
+@author:  liaoxingyu
+@contact: sherlockliao01@gmail.com
+"""
+import torch
+import numpy as np
+from PIL import Image, ImageOps
+import threading
+
+import queue
+from torch.utils.data import DataLoader
+
+from fastreid.utils.file_io import PathManager
+
+
+def read_image(file_name, format=None):
+    """
+    Read an image into the given format.
+    Will apply rotation and flipping if the image has such exif information.
+
+    Args:
+        file_name (str): image file path
+        format (str): one of the supported image modes in PIL, or "BGR"
+    Returns:
+        image (np.ndarray): an HWC image
+    """
+    with PathManager.open(file_name, "rb") as f:
+        image = Image.open(f)
+
+        # work around this bug: https://github.com/python-pillow/Pillow/issues/3973
+        try:
+            image = ImageOps.exif_transpose(image)
+        except Exception:
+            pass
+
+        if format is not None:
+            # PIL only supports RGB, so convert to RGB and flip channels over below
+            conversion_format = format
+            if format == "BGR":
+                conversion_format = "RGB"
+            image = image.convert(conversion_format)
+        image = np.asarray(image)
+
+        # PIL squeezes out the channel dimension for "L", so make it HWC
+        if format == "L":
+            image = np.expand_dims(image, -1)
+
+        # handle formats not supported by PIL
+        elif format == "BGR":
+            # flip channels if needed
+            image = image[:, :, ::-1]
+
+        # handle grayscale mixed in RGB images
+        elif len(image.shape) == 2:
+            image = np.repeat(image[..., np.newaxis], 3, axis=-1)
+
+        image = Image.fromarray(image)
+
+        return image
+
+
+"""
+#based on http://stackoverflow.com/questions/7323664/python-generator-pre-fetch
+This is a single-function package that transforms arbitrary generator into a background-thead generator that 
+prefetches several batches of data in a parallel background thead.
+
+This is useful if you have a computationally heavy process (CPU or GPU) that 
+iteratively processes minibatches from the generator while the generator 
+consumes some other resource (disk IO / loading from database / more CPU if you have unused cores). 
+
+By default these two processes will constantly wait for one another to finish. If you make generator work in 
+prefetch mode (see examples below), they will work in parallel, potentially saving you your GPU time.
+We personally use the prefetch generator when iterating minibatches of data for deep learning with PyTorch etc.
+
+Quick usage example (ipython notebook) - https://github.com/justheuristic/prefetch_generator/blob/master/example.ipynb
+This package contains this object
+ - BackgroundGenerator(any_other_generator[,max_prefetch = something])
+"""
+
+
+class BackgroundGenerator(threading.Thread):
+    """
+    the usage is below
+    >> for batch in BackgroundGenerator(my_minibatch_iterator):
+    >>    doit()
+    More details are written in the BackgroundGenerator doc
+    >> help(BackgroundGenerator)
+    """
+
+    def __init__(self, generator, local_rank, max_prefetch=10):
+        """
+        This function transforms generator into a background-thead generator.
+        :param generator: generator or genexp or any
+        It can be used with any minibatch generator.
+
+        It is quite lightweight, but not entirely weightless.
+        Using global variables inside generator is not recommended (may raise GIL and zero-out the
+        benefit of having a background thread.)
+        The ideal use case is when everything it requires is store inside it and everything it
+        outputs is passed through queue.
+
+        There's no restriction on doing weird stuff, reading/writing files, retrieving
+        URLs [or whatever] wlilst iterating.
+
+        :param max_prefetch: defines, how many iterations (at most) can background generator keep
+        stored at any moment of time.
+        Whenever there's already max_prefetch batches stored in queue, the background process will halt until
+        one of these batches is dequeued.
+
+        !Default max_prefetch=1 is okay unless you deal with some weird file IO in your generator!
+
+        Setting max_prefetch to -1 lets it store as many batches as it can, which will work
+        slightly (if any) faster, but will require storing
+        all batches in memory. If you use infinite generator with max_prefetch=-1, it will exceed the RAM size
+        unless dequeued quickly enough.
+        """
+        super().__init__()
+        self.queue = queue.Queue(max_prefetch)
+        self.generator = generator
+        self.local_rank = local_rank
+        self.daemon = True
+        self.exit_event = threading.Event()
+        self.start()
+
+    def run(self):
+        torch.cuda.set_device(self.local_rank)
+        for item in self.generator:
+            if self.exit_event.is_set():
+                break
+            self.queue.put(item)
+        self.queue.put(None)
+
+    def next(self):
+        next_item = self.queue.get()
+        if next_item is None:
+            raise StopIteration
+        return next_item
+
+    # Python 3 compatibility
+    def __next__(self):
+        return self.next()
+
+    def __iter__(self):
+        return self
+
+
+class DataLoaderX(DataLoader):
+    def __init__(self, local_rank, **kwargs):
+        super().__init__(**kwargs)
+        self.stream = torch.cuda.Stream(
+            local_rank
+        )  # create a new cuda stream in each process
+        self.local_rank = local_rank
+
+    def __iter__(self):
+        self.iter = super().__iter__()
+        self.iter = BackgroundGenerator(self.iter, self.local_rank)
+        self.preload()
+        return self
+
+    def _shutdown_background_thread(self):
+        if not self.iter.is_alive():
+            # avoid re-entrance or ill-conditioned thread state
+            return
+
+        # Set exit event to True for background threading stopping
+        self.iter.exit_event.set()
+
+        # Exhaust all remaining elements, so that the queue becomes empty,
+        # and the thread should quit
+        for _ in self.iter:
+            pass
+
+        # Waiting for background thread to quit
+        self.iter.join()
+
+    def preload(self):
+        self.batch = next(self.iter, None)
+        if self.batch is None:
+            return None
+        with torch.cuda.stream(self.stream):
+            for k in self.batch:
+                if isinstance(self.batch[k], torch.Tensor):
+                    self.batch[k] = self.batch[k].to(
+                        device=self.local_rank, non_blocking=True
+                    )
+
+    def __next__(self):
+        torch.cuda.current_stream().wait_stream(
+            self.stream
+        )  # wait tensor to put on GPU
+        batch = self.batch
+        if batch is None:
+            raise StopIteration
+        self.preload()
+        return batch
+
+    # Signal for shutting down background thread
+    def shutdown(self):
+        # If the dataloader is to be freed, shutdown its BackgroundGenerator
+        self._shutdown_background_thread()

fastreid/data/datasets/AirportALERT.py

+# encoding: utf-8
+"""
+@author:  xingyu liao
+@contact: sherlockliao01@gmail.com
+"""
+
+import os
+
+from fastreid.data.datasets import DATASET_REGISTRY
+from fastreid.data.datasets.bases import ImageDataset
+
+__all__ = ['AirportALERT', ]
+
+
+@DATASET_REGISTRY.register()
+class AirportALERT(ImageDataset):
+    """Airport 
+
+    """
+    dataset_dir = "AirportALERT"
+    dataset_name = "airport"
+
+    def __init__(self, root='datasets', **kwargs):
+        self.root = root
+        self.train_path = os.path.join(self.root, self.dataset_dir)
+        self.train_file = os.path.join(self.root, self.dataset_dir, 'filepath.txt')
+
+        required_files = [self.train_file, self.train_path]
+        self.check_before_run(required_files)
+
+        train = self.process_train(self.train_path, self.train_file)
+
+        super().__init__(train, [], [], **kwargs)
+
+    def process_train(self, dir_path, train_file):
+        data = []
+        with open(train_file, "r") as f:
+            img_paths = [line.strip('\n') for line in f.readlines()]
+
+        for path in img_paths:
+            split_path = path.split('\\')
+            img_path = '/'.join(split_path)
+            camid = self.dataset_name + "_" + split_path[0]
+            pid = self.dataset_name + "_" + split_path[1]
+            img_path = os.path.join(dir_path, img_path)
+            # if 11001 <= int(split_path[1]) <= 401999:
+            if 11001 <= int(split_path[1]):
+                data.append([img_path, pid, camid])
+
+        return data

fastreid/data/datasets/__init__.py

+# encoding: utf-8
+"""
+@author:  liaoxingyu
+@contact: sherlockliao01@gmail.com
+"""
+
+from ...utils.registry import Registry
+
+DATASET_REGISTRY = Registry("DATASET")
+DATASET_REGISTRY.__doc__ = """
+Registry for datasets
+It must returns an instance of :class:`Backbone`.
+"""
+
+# Person re-id datasets
+from .cuhk03 import CUHK03
+from .dukemtmcreid import DukeMTMC
+from .market1501 import Market1501
+from .msmt17 import MSMT17
+from .AirportALERT import AirportALERT
+from .iLIDS import iLIDS
+from .pku import PKU
+from .prai import PRAI
+from .prid import PRID
+from .grid import GRID
+from .saivt import SAIVT
+from .sensereid import SenseReID
+from .sysu_mm import SYSU_mm
+from .thermalworld import Thermalworld
+from .pes3d import PeS3D
+from .caviara import CAVIARa
+from .viper import VIPeR
+from .lpw import LPW
+from .shinpuhkan import Shinpuhkan
+from .wildtracker import WildTrackCrop
+from .cuhk_sysu import cuhkSYSU
+
+# Vehicle re-id datasets
+from .veri import VeRi
+from .vehicleid import VehicleID, SmallVehicleID, MediumVehicleID, LargeVehicleID
+from .veriwild import VeRiWild, SmallVeRiWild, MediumVeRiWild, LargeVeRiWild
+
+__all__ = [k for k in globals().keys() if "builtin" not in k and not k.startswith("_")]

fastreid/data/datasets/bases.py

+# encoding: utf-8
+"""
+@author:  sherlock
+@contact: sherlockliao01@gmail.com
+"""
+
+import copy
+import logging
+import os
+
+from tabulate import tabulate
+from termcolor import colored
+
+logger = logging.getLogger(__name__)
+
+
+class Dataset(object):
+    """An abstract class representing a Dataset.
+    This is the base class for ``ImageDataset`` and ``VideoDataset``.
+
+    Args:
+        train (list or Callable): contains tuples of (img_path(s), pid, camid).
+        query (list or Callable): contains tuples of (img_path(s), pid, camid).
+        gallery (list or Callable): contains tuples of (img_path(s), pid, camid).
+        transform: transform function.
+        mode (str): 'train', 'query' or 'gallery'.
+        combineall (bool): combines train, query and gallery in a
+            dataset for training.
+        verbose (bool): show information.
+    """
+    _junk_pids = []  # contains useless person IDs, e.g. background, false detections
+
+    def __init__(self, train, query, gallery, transform=None, mode='train',
+                 combineall=False, verbose=True, **kwargs):
+        self._train = train
+        self._query = query
+        self._gallery = gallery
+        self.transform = transform
+        self.mode = mode
+        self.combineall = combineall
+        self.verbose = verbose
+
+        if self.combineall:
+            self.combine_all()
+
+        if self.mode == 'train':
+            self.data = self.train
+        elif self.mode == 'query':
+            self.data = self.query
+        elif self.mode == 'gallery':
+            self.data = self.gallery
+        else:
+            raise ValueError('Invalid mode. Got {}, but expected to be '
+                             'one of [train | query | gallery]'.format(self.mode))
+
+    @property
+    def train(self):
+        if callable(self._train):
+            self._train = self._train()
+        return self._train
+
+    @property
+    def query(self):
+        if callable(self._query):
+            self._query = self._query()
+        return self._query
+
+    @property
+    def gallery(self):
+        if callable(self._gallery):
+            self._gallery = self._gallery()
+        return self._gallery
+
+    def __getitem__(self, index):
+        raise NotImplementedError
+
+    def __len__(self):
+        return len(self.data)
+
+    def __radd__(self, other):
+        """Supports sum([dataset1, dataset2, dataset3])."""
+        if other == 0:
+            return self
+        else:
+            return self.__add__(other)
+
+    def parse_data(self, data):
+        """Parses data list and returns the number of person IDs
+        and the number of camera views.
+        Args:
+            data (list): contains tuples of (img_path(s), pid, camid)
+        """
+        pids = set()
+        cams = set()
+        for info in data:
+            pids.add(info[1])
+            cams.add(info[2])
+        return len(pids), len(cams)
+
+    def get_num_pids(self, data):
+        """Returns the number of training person identities."""
+        return self.parse_data(data)[0]
+
+    def get_num_cams(self, data):
+        """Returns the number of training cameras."""
+        return self.parse_data(data)[1]
+
+    def show_summary(self):
+        """Shows dataset statistics."""
+        pass
+
+    def combine_all(self):
+        """Combines train, query and gallery in a dataset for training."""
+        combined = copy.deepcopy(self.train)
+
+        def _combine_data(data):
+            for img_path, pid, camid in data:
+                if pid in self._junk_pids:
+                    continue
+                pid = getattr(self, "dataset_name", "Unknown") + "_test_" + str(pid)
+                camid = getattr(self, "dataset_name", "Unknown") + "_test_" + str(camid)
+                combined.append((img_path, pid, camid))
+
+        _combine_data(self.query)
+        _combine_data(self.gallery)
+
+        self._train = combined
+
+    def check_before_run(self, required_files):
+        """Checks if required files exist before going deeper.
+        Args:
+            required_files (str or list): string file name(s).
+        """
+        if isinstance(required_files, str):
+            required_files = [required_files]
+
+        for fpath in required_files:
+            if not os.path.exists(fpath):
+                raise RuntimeError('"{}" is not found'.format(fpath))
+
+
+class ImageDataset(Dataset):
+    """A base class representing ImageDataset.
+    All other image datasets should subclass it.
+    ``__getitem__`` returns an image given index.
+    It will return ``img``, ``pid``, ``camid`` and ``img_path``
+    where ``img`` has shape (channel, height, width). As a result,
+    data in each batch has shape (batch_size, channel, height, width).
+    """
+
+    def show_train(self):
+        num_train_pids, num_train_cams = self.parse_data(self.train)
+
+        headers = ['subset', '# ids', '# images', '# cameras']
+        csv_results = [['train', num_train_pids, len(self.train), num_train_cams]]
+
+        # tabulate it
+        table = tabulate(
+            csv_results,
+            tablefmt="pipe",
+            headers=headers,
+            numalign="left",
+        )
+        logger.info(f"=> Loaded {self.__class__.__name__} in csv format: \n" + colored(table, "cyan"))
+
+    def show_test(self):
+        num_query_pids, num_query_cams = self.parse_data(self.query)
+        num_gallery_pids, num_gallery_cams = self.parse_data(self.gallery)
+
+        headers = ['subset', '# ids', '# images', '# cameras']
+        csv_results = [
+            ['query', num_query_pids, len(self.query), num_query_cams],
+            ['gallery', num_gallery_pids, len(self.gallery), num_gallery_cams],
+        ]
+
+        # tabulate it
+        table = tabulate(
+            csv_results,
+            tablefmt="pipe",
+            headers=headers,
+            numalign="left",
+        )
+        logger.info(f"=> Loaded {self.__class__.__name__} in csv format: \n" + colored(table, "cyan"))

fastreid/data/datasets/caviara.py

+# encoding: utf-8
+"""
+@author:  xingyu liao
+@contact: sherlockliao01@gmail.com
+"""
+
+import os
+from glob import glob
+
+from fastreid.data.datasets import DATASET_REGISTRY
+from fastreid.data.datasets.bases import ImageDataset
+
+__all__ = ['CAVIARa', ]
+
+
+@DATASET_REGISTRY.register()
+class CAVIARa(ImageDataset):
+    """CAVIARa
+    """
+    dataset_dir = "CAVIARa"
+    dataset_name = "caviara"
+
+    def __init__(self, root='datasets', **kwargs):
+        self.root = root
+        self.train_path = os.path.join(self.root, self.dataset_dir)
+
+        required_files = [self.train_path]
+        self.check_before_run(required_files)
+
+        train = self.process_train(self.train_path)
+
+        super().__init__(train, [], [], **kwargs)
+
+    def process_train(self, train_path):
+        data = []
+
+        img_list = glob(os.path.join(train_path, "*.jpg"))
+        for img_path in img_list:
+            img_name = img_path.split('/')[-1]
+            pid = self.dataset_name + "_" + img_name[:4]
+            camid = self.dataset_name + "_cam0"
+            data.append([img_path, pid, camid])
+
+        return data

fastreid/data/datasets/cuhk03.py

+# encoding: utf-8
+"""
+@author:  liaoxingyu
+@contact: liaoxingyu2@jd.com
+"""
+
+import json
+import os.path as osp
+
+from fastreid.data.datasets import DATASET_REGISTRY
+from fastreid.utils.file_io import PathManager
+from .bases import ImageDataset
+
+
+@DATASET_REGISTRY.register()
+class CUHK03(ImageDataset):
+    """CUHK03.
+
+    Reference:
+        Li et al. DeepReID: Deep Filter Pairing Neural Network for Person Re-identification. CVPR 2014.
+
+    URL: `<http://www.ee.cuhk.edu.hk/~xgwang/CUHK_identification.html#!>`_
+
+    Dataset statistics:
+        - identities: 1360.
+        - images: 13164.
+        - cameras: 6.
+        - splits: 20 (classic).
+    """
+    dataset_dir = 'cuhk03'
+    dataset_url = None
+    dataset_name = "cuhk03"
+
+    def __init__(self, root='datasets', split_id=0, cuhk03_labeled=True, cuhk03_classic_split=False, **kwargs):
+        self.root = root
+        self.dataset_dir = osp.join(self.root, self.dataset_dir)
+
+        self.data_dir = osp.join(self.dataset_dir, 'cuhk03_release')
+        self.raw_mat_path = osp.join(self.data_dir, 'cuhk-03.mat')
+
+        self.imgs_detected_dir = osp.join(self.dataset_dir, 'images_detected')
+        self.imgs_labeled_dir = osp.join(self.dataset_dir, 'images_labeled')
+
+        self.split_classic_det_json_path = osp.join(self.dataset_dir, 'splits_classic_detected.json')
+        self.split_classic_lab_json_path = osp.join(self.dataset_dir, 'splits_classic_labeled.json')
+
+        self.split_new_det_json_path = osp.join(self.dataset_dir, 'splits_new_detected.json')
+        self.split_new_lab_json_path = osp.join(self.dataset_dir, 'splits_new_labeled.json')
+
+        self.split_new_det_mat_path = osp.join(self.dataset_dir, 'cuhk03_new_protocol_config_detected.mat')
+        self.split_new_lab_mat_path = osp.join(self.dataset_dir, 'cuhk03_new_protocol_config_labeled.mat')
+
+        required_files = [
+            self.dataset_dir,
+            self.data_dir,
+            self.raw_mat_path,
+            self.split_new_det_mat_path,
+            self.split_new_lab_mat_path
+        ]
+        self.check_before_run(required_files)
+
+        self.preprocess_split()
+
+        if cuhk03_labeled:
+            split_path = self.split_classic_lab_json_path if cuhk03_classic_split else self.split_new_lab_json_path
+        else:
+            split_path = self.split_classic_det_json_path if cuhk03_classic_split else self.split_new_det_json_path
+
+        with PathManager.open(split_path) as f:
+            splits = json.load(f)
+        assert split_id < len(splits), 'Condition split_id ({}) < len(splits) ({}) is false'.format(split_id,
+                                                                                                    len(splits))
+        split = splits[split_id]
+
+        train = split['train']
+        tmp_train = []
+        for img_path, pid, camid in train:
+            new_pid = self.dataset_name + "_" + str(pid)
+            new_camid = self.dataset_name + "_" + str(camid)
+            tmp_train.append((img_path, new_pid, new_camid))
+        train = tmp_train
+        del tmp_train
+        query = split['query']
+        gallery = split['gallery']
+
+        super(CUHK03, self).__init__(train, query, gallery, **kwargs)
+
+    def preprocess_split(self):
+        # This function is a bit complex and ugly, what it does is
+        # 1. extract data from cuhk-03.mat and save as png images
+        # 2. create 20 classic splits (Li et al. CVPR'14)
+        # 3. create new split (Zhong et al. CVPR'17)
+        if osp.exists(self.imgs_labeled_dir) \
+                and osp.exists(self.imgs_detected_dir) \
+                and osp.exists(self.split_classic_det_json_path) \
+                and osp.exists(self.split_classic_lab_json_path) \
+                and osp.exists(self.split_new_det_json_path) \
+                and osp.exists(self.split_new_lab_json_path):
+            return
+
+        import h5py
+        from imageio import imwrite
+        from scipy import io
+
+        PathManager.mkdirs(self.imgs_detected_dir)
+        PathManager.mkdirs(self.imgs_labeled_dir)
+
+        print('Extract image data from "{}" and save as png'.format(self.raw_mat_path))
+        mat = h5py.File(self.raw_mat_path, 'r')
+
+        def _deref(ref):
+            return mat[ref][:].T
+
+        def _process_images(img_refs, campid, pid, save_dir):
+            img_paths = []  # Note: some persons only have images for one view
+            for imgid, img_ref in enumerate(img_refs):
+                img = _deref(img_ref)
+                if img.size == 0 or img.ndim < 3:
+                    continue  # skip empty cell
+                # images are saved with the following format, index-1 (ensure uniqueness)
+                # campid: index of camera pair (1-5)
+                # pid: index of person in 'campid'-th camera pair
+                # viewid: index of view, {1, 2}
+                # imgid: index of image, (1-10)
+                viewid = 1 if imgid < 5 else 2
+                img_name = '{:01d}_{:03d}_{:01d}_{:02d}.png'.format(campid + 1, pid + 1, viewid, imgid + 1)
+                img_path = osp.join(save_dir, img_name)
+                if not osp.isfile(img_path):
+                    imwrite(img_path, img)
+                img_paths.append(img_path)
+            return img_paths
+
+        def _extract_img(image_type):
+            print('Processing {} images ...'.format(image_type))
+            meta_data = []
+            imgs_dir = self.imgs_detected_dir if image_type == 'detected' else self.imgs_labeled_dir
+            for campid, camp_ref in enumerate(mat[image_type][0]):
+                camp = _deref(camp_ref)
+                num_pids = camp.shape[0]
+                for pid in range(num_pids):
+                    img_paths = _process_images(camp[pid, :], campid, pid, imgs_dir)
+                    assert len(img_paths) > 0, 'campid{}-pid{} has no images'.format(campid, pid)
+                    meta_data.append((campid + 1, pid + 1, img_paths))
+                print('- done camera pair {} with {} identities'.format(campid + 1, num_pids))
+            return meta_data
+
+        meta_detected = _extract_img('detected')
+        meta_labeled = _extract_img('labeled')
+
+        def _extract_classic_split(meta_data, test_split):
+            train, test = [], []
+            num_train_pids, num_test_pids = 0, 0
+            num_train_imgs, num_test_imgs = 0, 0
+            for i, (campid, pid, img_paths) in enumerate(meta_data):
+
+                if [campid, pid] in test_split:
+                    for img_path in img_paths:
+                        camid = int(osp.basename(img_path).split('_')[2]) - 1  # make it 0-based
+                        test.append((img_path, num_test_pids, camid))
+                    num_test_pids += 1
+                    num_test_imgs += len(img_paths)
+                else:
+                    for img_path in img_paths:
+                        camid = int(osp.basename(img_path).split('_')[2]) - 1  # make it 0-based
+                        train.append((img_path, num_train_pids, camid))
+                    num_train_pids += 1
+                    num_train_imgs += len(img_paths)
+            return train, num_train_pids, num_train_imgs, test, num_test_pids, num_test_imgs
+
+        print('Creating classic splits (# = 20) ...')
+        splits_classic_det, splits_classic_lab = [], []
+        for split_ref in mat['testsets'][0]:
+            test_split = _deref(split_ref).tolist()
+
+            # create split for detected images
+            train, num_train_pids, num_train_imgs, test, num_test_pids, num_test_imgs = \
+                _extract_classic_split(meta_detected, test_split)
+            splits_classic_det.append({
+                'train': train,
+                'query': test,
+                'gallery': test,
+                'num_train_pids': num_train_pids,
+                'num_train_imgs': num_train_imgs,
+                'num_query_pids': num_test_pids,
+                'num_query_imgs': num_test_imgs,
+                'num_gallery_pids': num_test_pids,
+                'num_gallery_imgs': num_test_imgs
+            })
+
+            # create split for labeled images
+            train, num_train_pids, num_train_imgs, test, num_test_pids, num_test_imgs = \
+                _extract_classic_split(meta_labeled, test_split)
+            splits_classic_lab.append({
+                'train': train,
+                'query': test,
+                'gallery': test,
+                'num_train_pids': num_train_pids,
+                'num_train_imgs': num_train_imgs,
+                'num_query_pids': num_test_pids,
+                'num_query_imgs': num_test_imgs,
+                'num_gallery_pids': num_test_pids,
+                'num_gallery_imgs': num_test_imgs
+            })
+
+        with PathManager.open(self.split_classic_det_json_path, 'w') as f:
+            json.dump(splits_classic_det, f, indent=4, separators=(',', ': '))
+        with PathManager.open(self.split_classic_lab_json_path, 'w') as f:
+            json.dump(splits_classic_lab, f, indent=4, separators=(',', ': '))
+
+        def _extract_set(filelist, pids, pid2label, idxs, img_dir, relabel):
+            tmp_set = []
+            unique_pids = set()
+            for idx in idxs:
+                img_name = filelist[idx][0]
+                camid = int(img_name.split('_')[2]) - 1  # make it 0-based
+                pid = pids[idx]
+                if relabel:
+                    pid = pid2label[pid]
+                img_path = osp.join(img_dir, img_name)
+                tmp_set.append((img_path, int(pid), camid))
+                unique_pids.add(pid)
+            return tmp_set, len(unique_pids), len(idxs)
+
+        def _extract_new_split(split_dict, img_dir):
+            train_idxs = split_dict['train_idx'].flatten() - 1  # index-0
+            pids = split_dict['labels'].flatten()
+            train_pids = set(pids[train_idxs])
+            pid2label = {pid: label for label, pid in enumerate(train_pids)}
+            query_idxs = split_dict['query_idx'].flatten() - 1
+            gallery_idxs = split_dict['gallery_idx'].flatten() - 1
+            filelist = split_dict['filelist'].flatten()
+            train_info = _extract_set(filelist, pids, pid2label, train_idxs, img_dir, relabel=True)
+            query_info = _extract_set(filelist, pids, pid2label, query_idxs, img_dir, relabel=False)
+            gallery_info = _extract_set(filelist, pids, pid2label, gallery_idxs, img_dir, relabel=False)
+            return train_info, query_info, gallery_info
+
+        print('Creating new split for detected images (767/700) ...')
+        train_info, query_info, gallery_info = _extract_new_split(
+            io.loadmat(self.split_new_det_mat_path),
+            self.imgs_detected_dir
+        )
+        split = [{
+            'train': train_info[0],
+            'query': query_info[0],
+            'gallery': gallery_info[0],
+            'num_train_pids': train_info[1],
+            'num_train_imgs': train_info[2],
+            'num_query_pids': query_info[1],
+            'num_query_imgs': query_info[2],
+            'num_gallery_pids': gallery_info[1],
+            'num_gallery_imgs': gallery_info[2]
+        }]
+
+        with PathManager.open(self.split_new_det_json_path, 'w') as f:
+            json.dump(split, f, indent=4, separators=(',', ': '))
+
+        print('Creating new split for labeled images (767/700) ...')
+        train_info, query_info, gallery_info = _extract_new_split(
+            io.loadmat(self.split_new_lab_mat_path),
+            self.imgs_labeled_dir
+        )
+        split = [{
+            'train': train_info[0],
+            'query': query_info[0],
+            'gallery': gallery_info[0],
+            'num_train_pids': train_info[1],
+            'num_train_imgs': train_info[2],
+            'num_query_pids': query_info[1],
+            'num_query_imgs': query_info[2],
+            'num_gallery_pids': gallery_info[1],
+            'num_gallery_imgs': gallery_info[2]
+        }]
+        with PathManager.open(self.split_new_lab_json_path, 'w') as f:
+            json.dump(split, f, indent=4, separators=(',', ': '))

fastreid/data/datasets/cuhk_sysu.py

+# encoding: utf-8
+"""
+@author:  xingyu liao
+@contact: sherlockliao01@gmail.com
+"""
+
+import glob
+import os.path as osp
+import re
+import warnings
+
+from .bases import ImageDataset
+from ..datasets import DATASET_REGISTRY
+
+
+@DATASET_REGISTRY.register()
+class cuhkSYSU(ImageDataset):
+    """CUHK SYSU datasets.
+
+    The dataset is collected from two sources: street snap and movie.
+    In street snap, 12,490 images and 6,057 query persons were collected
+    with movable cameras across hundreds of scenes while 5,694 images and
+    2,375 query persons were selected from movies and TV dramas.
+
+    Dataset statistics:
+        - identities: xxx.
+        - images: 12936 (train).
+    """
+    dataset_dir = 'cuhk_sysu'
+    dataset_name = "cuhksysu"
+
+    def __init__(self, root='datasets', **kwargs):
+        self.root = root
+        self.dataset_dir = osp.join(self.root, self.dataset_dir)
+
+        self.data_dir = osp.join(self.dataset_dir, "cropped_images")
+
+        required_files = [self.data_dir]
+        self.check_before_run(required_files)
+
+        train = self.process_dir(self.data_dir)
+        query = []
+        gallery = []
+
+        super(cuhkSYSU, self).__init__(train, query, gallery, **kwargs)
+
+    def process_dir(self, dir_path):
+        img_paths = glob.glob(osp.join(dir_path, '*.jpg'))
+        pattern = re.compile(r'p([-\d]+)_s(\d)')
+
+        data = []
+        for img_path in img_paths:
+            pid, _ = map(int, pattern.search(img_path).groups())
+            pid = self.dataset_name + "_" + str(pid)
+            camid = self.dataset_name + "_0"
+            data.append((img_path, pid, camid))
+
+        return data

fastreid/data/datasets/dukemtmcreid.py

+# encoding: utf-8
+"""
+@author:  liaoxingyu
+@contact: liaoxingyu2@jd.com
+"""
+
+import glob
+import os.path as osp
+import re
+
+from .bases import ImageDataset
+from ..datasets import DATASET_REGISTRY
+
+
+@DATASET_REGISTRY.register()
+class DukeMTMC(ImageDataset):
+    """DukeMTMC-reID.
+
+    Reference:
+        - Ristani et al. Performance Measures and a Data Set for Multi-Target, Multi-Camera Tracking. ECCVW 2016.
+        - Zheng et al. Unlabeled Samples Generated by GAN Improve the Person Re-identification Baseline in vitro. ICCV 2017.
+
+    URL: `<https://github.com/layumi/DukeMTMC-reID_evaluation>`_
+
+    Dataset statistics:
+        - identities: 1404 (train + query).
+        - images:16522 (train) + 2228 (query) + 17661 (gallery).
+        - cameras: 8.
+    """
+    dataset_dir = 'DukeMTMC-reID'
+    dataset_url = 'http://vision.cs.duke.edu/DukeMTMC/data/misc/DukeMTMC-reID.zip'
+    dataset_name = "dukemtmc"
+
+    def __init__(self, root='datasets', **kwargs):
+        # self.root = osp.abspath(osp.expanduser(root))
+        self.root = root
+        self.dataset_dir = osp.join(self.root, self.dataset_dir)
+        self.train_dir = osp.join(self.dataset_dir, 'bounding_box_train')
+        self.query_dir = osp.join(self.dataset_dir, 'query')
+        self.gallery_dir = osp.join(self.dataset_dir, 'bounding_box_test')
+
+        required_files = [
+            self.dataset_dir,
+            self.train_dir,
+            self.query_dir,
+            self.gallery_dir,
+        ]
+        self.check_before_run(required_files)
+
+        train = self.process_dir(self.train_dir)
+        query = self.process_dir(self.query_dir, is_train=False)
+        gallery = self.process_dir(self.gallery_dir, is_train=False)
+
+        super(DukeMTMC, self).__init__(train, query, gallery, **kwargs)
+
+    def process_dir(self, dir_path, is_train=True):
+        img_paths = glob.glob(osp.join(dir_path, '*.jpg'))
+        pattern = re.compile(r'([-\d]+)_c(\d)')
+
+        data = []
+        for img_path in img_paths:
+            pid, camid = map(int, pattern.search(img_path).groups())
+            assert 1 <= camid <= 8
+            camid -= 1  # index starts from 0
+            if is_train:
+                pid = self.dataset_name + "_" + str(pid)
+                camid = self.dataset_name + "_" + str(camid)
+            data.append((img_path, pid, camid))
+
+        return data

fastreid/data/datasets/grid.py

+# encoding: utf-8
+"""
+@author:  xingyu liao
+@contact: sherlockliao01@gmail.com
+"""
+
+import os
+from glob import glob
+
+from fastreid.data.datasets import DATASET_REGISTRY
+from fastreid.data.datasets.bases import ImageDataset
+
+__all__ = ['GRID', ]
+
+
+@DATASET_REGISTRY.register()
+class GRID(ImageDataset):
+    """GRID
+    """
+    dataset_dir = "underground_reid"
+    dataset_name = 'grid'
+
+    def __init__(self, root='datasets', **kwargs):
+        self.root = root
+        self.train_path = os.path.join(self.root, self.dataset_dir, 'images')
+
+        required_files = [self.train_path]
+        self.check_before_run(required_files)
+
+        train = self.process_train(self.train_path)
+
+        super().__init__(train, [], [], **kwargs)
+
+    def process_train(self, train_path):
+        data = []
+        img_paths = glob(os.path.join(train_path, "*.jpeg"))
+
+        for img_path in img_paths:
+            img_name = os.path.basename(img_path)
+            img_info = img_name.split('_')
+            pid = self.dataset_name + "_" + img_info[0]
+            camid = self.dataset_name + "_" + img_info[1]
+            data.append([img_path, pid, camid])
+        return data

fastreid/data/datasets/iLIDS.py

+# encoding: utf-8
+"""
+@author:  xingyu liao
+@contact: sherlockliao01@gmail.com
+"""
+
+import os
+from glob import glob
+
+from fastreid.data.datasets import DATASET_REGISTRY
+from fastreid.data.datasets.bases import ImageDataset
+
+__all__ = ['iLIDS', ]
+
+
+@DATASET_REGISTRY.register()
+class iLIDS(ImageDataset):
+    """iLIDS
+    """
+    dataset_dir = "iLIDS"
+    dataset_name = "ilids"
+
+    def __init__(self, root='datasets', **kwargs):
+        self.root = root
+        self.train_path = os.path.join(self.root, self.dataset_dir)
+
+        required_files = [self.train_path]
+        self.check_before_run(required_files)
+
+        train = self.process_train(self.train_path)
+
+        super().__init__(train, [], [], **kwargs)
+
+    def process_train(self, train_path):
+        data = []
+        file_path = os.listdir(train_path)
+        for pid_dir in file_path:
+            img_file = os.path.join(train_path, pid_dir)
+            img_paths = glob(os.path.join(img_file, "*.png"))
+            for img_path in img_paths:
+                split_path = img_path.split('/')
+                pid = self.dataset_name + "_" + split_path[-2]
+                camid = self.dataset_name + "_" + split_path[-1].split('_')[0]
+                data.append([img_path, pid, camid])
+        return data

fastreid/data/datasets/lpw.py

+# encoding: utf-8
+"""
+@author:  xingyu liao
+@contact: sherlockliao01@gmail.com
+"""
+
+import os
+from glob import glob
+
+from fastreid.data.datasets import DATASET_REGISTRY
+from fastreid.data.datasets.bases import ImageDataset
+
+__all__ = ['LPW', ]
+
+
+@DATASET_REGISTRY.register()
+class LPW(ImageDataset):
+    """LPW
+    """
+    dataset_dir = "pep_256x128/data_slim"
+    dataset_name = "lpw"
+
+    def __init__(self, root='datasets', **kwargs):
+        self.root = root
+        self.train_path = os.path.join(self.root, self.dataset_dir)
+
+        required_files = [self.train_path]
+        self.check_before_run(required_files)
+
+        train = self.process_train(self.train_path)
+
+        super().__init__(train, [], [], **kwargs)
+
+    def process_train(self, train_path):
+        data = []
+
+        file_path_list = ['scen1', 'scen2', 'scen3']
+
+        for scene in file_path_list:
+            cam_list = os.listdir(os.path.join(train_path, scene))
+            for cam in cam_list:
+                camid = self.dataset_name + "_" + cam
+                pid_list = os.listdir(os.path.join(train_path, scene, cam))
+                for pid_dir in pid_list:
+                    img_paths = glob(os.path.join(train_path, scene, cam, pid_dir, "*.jpg"))
+                    for img_path in img_paths:
+                        pid = self.dataset_name + "_" + scene + "-" + pid_dir
+                        data.append([img_path, pid, camid])
+        return data

fastreid/data/datasets/market1501.py

+# encoding: utf-8
+"""
+@author:  sherlock
+@contact: sherlockliao01@gmail.com
+"""
+
+import glob
+import os.path as osp
+import re
+import warnings
+
+from .bases import ImageDataset
+from ..datasets import DATASET_REGISTRY
+
+
+@DATASET_REGISTRY.register()
+class Market1501(ImageDataset):
+    """Market1501.
+
+    Reference:
+        Zheng et al. Scalable Person Re-identification: A Benchmark. ICCV 2015.
+
+    URL: `<http://www.liangzheng.org/Project/project_reid.html>`_
+
+    Dataset statistics:
+        - identities: 1501 (+1 for background).
+        - images: 12936 (train) + 3368 (query) + 15913 (gallery).
+    """
+    _junk_pids = [0, -1]
+    dataset_dir = ''
+    dataset_url = 'http://188.138.127.15:81/Datasets/Market-1501-v15.09.15.zip'
+    dataset_name = "market1501"
+
+    def __init__(self, root='datasets', market1501_500k=False, **kwargs):
+        # self.root = osp.abspath(osp.expanduser(root))
+        self.root = root
+        self.dataset_dir = osp.join(self.root, self.dataset_dir)
+
+        # allow alternative directory structure
+        self.data_dir = self.dataset_dir
+        data_dir = osp.join(self.data_dir, 'Market-1501-v15.09.15')
+        if osp.isdir(data_dir):
+            self.data_dir = data_dir
+        else:
+            warnings.warn('The current data structure is deprecated. Please '
+                          'put data folders such as "bounding_box_train" under '
+                          '"Market-1501-v15.09.15".')
+
+        self.train_dir = osp.join(self.data_dir, 'bounding_box_train')
+        self.query_dir = osp.join(self.data_dir, 'query')
+        self.gallery_dir = osp.join(self.data_dir, 'bounding_box_test')
+        self.extra_gallery_dir = osp.join(self.data_dir, 'images')
+        self.market1501_500k = market1501_500k
+
+        required_files = [
+            self.data_dir,
+            self.train_dir,
+            self.query_dir,
+            self.gallery_dir,
+        ]
+        if self.market1501_500k:
+            required_files.append(self.extra_gallery_dir)
+        self.check_before_run(required_files)
+
+        train = lambda: self.process_dir(self.train_dir)
+        query = lambda: self.process_dir(self.query_dir, is_train=False)
+        gallery = lambda: self.process_dir(self.gallery_dir, is_train=False) + \
+                          (self.process_dir(self.extra_gallery_dir, is_train=False) if self.market1501_500k else [])
+
+        super(Market1501, self).__init__(train, query, gallery, **kwargs)
+
+    def process_dir(self, dir_path, is_train=True):
+        img_paths = glob.glob(osp.join(dir_path, '*.jpg'))
+        pattern = re.compile(r'([-\d]+)_c(\d)')
+
+        data = []
+        for img_path in img_paths:
+            pid, camid = map(int, pattern.search(img_path).groups())
+            if pid == -1:
+                continue  # junk images are just ignored
+            assert 0 <= pid <= 1501  # pid == 0 means background
+            assert 1 <= camid <= 6
+            camid -= 1  # index starts from 0
+            if is_train:
+                pid = self.dataset_name + "_" + str(pid)
+                camid = self.dataset_name + "_" + str(camid)
+            data.append((img_path, pid, camid))
+
+        return data