update

configs/Market1501/AGW_S50.yml

+_BASE_: ../Base-AGW.yml
+
+MODEL:
+  BACKBONE:
+    NAME: build_resnest_backbone
+
+DATASETS:
+  NAMES: ("Market1501",)
+  TESTS: ("Market1501",)
+
+OUTPUT_DIR: logs/market1501/agw_S50

configs/Market1501/bagtricks_R101-ibn.yml

+_BASE_: ../Base-bagtricks.yml
+
+MODEL:
+  BACKBONE:
+    DEPTH: 101x
+    WITH_IBN: True
+
+DATASETS:
+  NAMES: ("Market1501",)
+  TESTS: ("Market1501",)
+
+OUTPUT_DIR: logs/market1501/bagtricks_R101-ibn

configs/Market1501/bagtricks_R50-ibn.yml

+_BASE_: ../Base-bagtricks.yml
+
+MODEL:
+  BACKBONE:
+    WITH_IBN: True
+
+DATASETS:
+  NAMES: ("Market1501",)
+  TESTS: ("Market1501",)
+
+OUTPUT_DIR: logs/market1501/bagtricks_R50-ibn

configs/Market1501/bagtricks_R50.yml

+_BASE_: ../Base-bagtricks.yml
+
+DATASETS:
+  NAMES: ("Market1501",)
+  TESTS: ("Market1501",)
+
+OUTPUT_DIR: logs/market1501/bagtricks_R50

configs/Market1501/bagtricks_S50.yml

+_BASE_: ../Base-bagtricks.yml
+
+MODEL:
+  BACKBONE:
+    NAME: build_resnest_backbone
+
+DATASETS:
+  NAMES: ("Market1501",)
+  TESTS: ("Market1501",)
+
+OUTPUT_DIR: logs/market1501/bagtricks_S50

configs/Market1501/bagtricks_vit.yml

+
+MODEL:
+  META_ARCHITECTURE: Baseline
+  PIXEL_MEAN: [127.5, 127.5, 127.5]
+  PIXEL_STD: [127.5, 127.5, 127.5]
+
+  BACKBONE:
+    NAME: build_vit_backbone
+    DEPTH: base
+    FEAT_DIM: 768
+    PRETRAIN: True
+    PRETRAIN_PATH: /export/home/lxy/.cache/torch/checkpoints/jx_vit_base_p16_224-80ecf9dd.pth
+    STRIDE_SIZE: (16, 16)
+    DROP_PATH_RATIO: 0.1
+    DROP_RATIO: 0.0
+    ATT_DROP_RATE: 0.0
+
+  HEADS:
+    NAME: EmbeddingHead
+    NORM: BN
+    WITH_BNNECK: True
+    POOL_LAYER: Identity
+    NECK_FEAT: before
+    CLS_LAYER: Linear
+
+  LOSSES:
+    NAME: ("CrossEntropyLoss", "TripletLoss",)
+
+    CE:
+      EPSILON: 0. # no smooth
+      SCALE: 1.
+
+    TRI:
+      MARGIN: 0.0
+      HARD_MINING: True
+      NORM_FEAT: False
+      SCALE: 1.
+
+INPUT:
+  SIZE_TRAIN: [ 256, 128 ]
+  SIZE_TEST: [ 256, 128 ]
+
+  REA:
+    ENABLED: True
+    PROB: 0.5
+
+  FLIP:
+    ENABLED: True
+
+  PADDING:
+    ENABLED: True
+
+DATALOADER:
+  SAMPLER_TRAIN: NaiveIdentitySampler
+  NUM_INSTANCE: 4
+  NUM_WORKERS: 8
+
+SOLVER:
+  AMP:
+    ENABLED: False
+  OPT: SGD
+  MAX_EPOCH: 120
+  BASE_LR: 0.008
+  WEIGHT_DECAY: 0.0001
+  IMS_PER_BATCH: 64
+
+  SCHED: CosineAnnealingLR
+  ETA_MIN_LR: 0.000016
+
+  WARMUP_FACTOR: 0.01
+  WARMUP_ITERS: 1000
+
+  CLIP_GRADIENTS:
+    ENABLED: True
+
+  CHECKPOINT_PERIOD: 30
+
+TEST:
+  EVAL_PERIOD: 5
+  IMS_PER_BATCH: 128
+
+CUDNN_BENCHMARK: True
+
+DATASETS:
+  NAMES: ("Market1501",)
+  TESTS: ("Market1501",)
+
+OUTPUT_DIR: logs/market1501/sbs_vit_base

configs/Market1501/mgn_R50-ibn.yml

+_BASE_: ../Base-MGN.yml
+
+MODEL:
+  BACKBONE:
+    WITH_IBN: True
+
+DATASETS:
+  NAMES: ("Market1501",)
+  TESTS: ("Market1501",)
+
+OUTPUT_DIR: logs/market1501/mgn_R50-ibn

configs/Market1501/sbs_R101-ibn.yml

+_BASE_: ../Base-SBS.yml
+
+MODEL:
+  BACKBONE:
+    DEPTH: 101x
+    WITH_IBN: True
+
+DATASETS:
+  NAMES: ("Market1501",)
+  TESTS: ("Market1501",)
+
+OUTPUT_DIR: logs/market1501/sbs_R101-ibn

configs/Market1501/sbs_R50-ibn.yml

+_BASE_: ../Base-SBS.yml
+
+MODEL:
+  BACKBONE:
+    WITH_IBN: True
+
+DATASETS:
+  NAMES: ("Market1501",)
+  TESTS: ("Market1501",)
+
+OUTPUT_DIR: logs/market1501/sbs_R50-ibn

configs/Market1501/sbs_R50.yml

+_BASE_: ../Base-SBS.yml
+
+DATASETS:
+  NAMES: ("Market1501",)
+  TESTS: ("Market1501",)
+
+OUTPUT_DIR: logs/market1501/sbs_R50

configs/Market1501/sbs_S50.yml

+_BASE_: ../Base-SBS.yml
+
+MODEL:
+  BACKBONE:
+    NAME: build_resnest_backbone
+
+DATASETS:
+  NAMES: ("Market1501",)
+  TESTS: ("Market1501",)
+
+OUTPUT_DIR: logs/market1501/sbs_S50

configs/VERIWild/bagtricks_R50-ibn.yml

+_BASE_: ../Base-bagtricks.yml
+
+INPUT:
+  SIZE_TRAIN: [256, 256]
+  SIZE_TEST: [256, 256]
+
+MODEL:
+  BACKBONE:
+    WITH_IBN: True
+
+  HEADS:
+    POOL_LAYER: GeneralizedMeanPooling
+
+  LOSSES:
+    TRI:
+      HARD_MINING: False
+      MARGIN: 0.0
+
+DATASETS:
+  NAMES: ("VeRiWild",)
+  TESTS: ("SmallVeRiWild", "MediumVeRiWild", "LargeVeRiWild",)
+
+SOLVER:
+  IMS_PER_BATCH: 512 # 512 For 4 GPUs
+  MAX_EPOCH: 120
+  STEPS: [30, 70, 90]
+  WARMUP_ITERS: 5000
+
+  CHECKPOINT_PERIOD: 20
+
+TEST:
+  EVAL_PERIOD: 10
+  IMS_PER_BATCH: 128
+
+OUTPUT_DIR: logs/veriwild/bagtricks_R50-ibn_4gpu

configs/VeRi/sbs_R50-ibn.yml

+_BASE_: ../Base-SBS.yml
+
+INPUT:
+  SIZE_TRAIN: [256, 256]
+  SIZE_TEST: [256, 256]
+
+MODEL:
+  BACKBONE:
+    WITH_IBN: True
+    WITH_NL: True
+
+SOLVER:
+  OPT: SGD
+  BASE_LR: 0.01
+  ETA_MIN_LR: 7.7e-5
+
+  IMS_PER_BATCH: 64
+  MAX_EPOCH: 60
+  WARMUP_ITERS: 3000
+  FREEZE_ITERS: 3000
+
+  CHECKPOINT_PERIOD: 10
+
+DATASETS:
+  NAMES: ("VeRi",)
+  TESTS: ("VeRi",)
+
+DATALOADER:
+  SAMPLER_TRAIN: BalancedIdentitySampler
+
+TEST:
+  EVAL_PERIOD: 10
+  IMS_PER_BATCH: 256
+
+OUTPUT_DIR: logs/veri/sbs_R50-ibn

configs/VehicleID/bagtricks_R50-ibn.yml

+_BASE_: ../Base-bagtricks.yml
+
+INPUT:
+  SIZE_TRAIN: [256, 256]
+  SIZE_TEST: [256, 256]
+
+MODEL:
+  BACKBONE:
+    WITH_IBN: True
+  HEADS:
+    POOL_LAYER: GeneralizedMeanPooling
+
+  LOSSES:
+    TRI:
+      HARD_MINING: False
+      MARGIN: 0.0
+
+DATASETS:
+  NAMES: ("VehicleID",)
+  TESTS: ("SmallVehicleID", "MediumVehicleID", "LargeVehicleID",)
+
+SOLVER:
+  BIAS_LR_FACTOR: 1.
+
+  IMS_PER_BATCH: 512
+  MAX_EPOCH: 60
+  STEPS: [30, 50]
+  WARMUP_ITERS: 2000
+
+  CHECKPOINT_PERIOD: 20
+
+TEST:
+  EVAL_PERIOD: 20
+  IMS_PER_BATCH: 128
+
+OUTPUT_DIR: logs/vehicleid/bagtricks_R50-ibn_4gpu

datasets/README.md

+# Setup Buildin Dataset
+
+Fastreid has buildin support for a few datasets. The datasets are assumed to exist in a directory specified by the environment variable `FASTREID_DATASETS`. Under this directory, fastreid expects to find datasets in the structure described below.
+
+You can set the location for builtin datasets by `export FASTREID_DATASETS=/path/to/datasets/`. If left unset, the default is `datasets/` relative to your current working directory.
+
+The [model zoo](https://github.com/JDAI-CV/fast-reid/blob/master/MODEL_ZOO.md) contains configs and models that use these buildin datasets.
+
+## Expected dataset structure for [Market1501](https://www.cv-foundation.org/openaccess/content_iccv_2015/papers/Zheng_Scalable_Person_Re-Identification_ICCV_2015_paper.pdf)
+
+1. Download dataset to `datasets/` from [baidu pan](https://pan.baidu.com/s/1ntIi2Op) or [google driver](https://drive.google.com/file/d/0B8-rUzbwVRk0c054eEozWG9COHM/view)
+2. Extract dataset. The dataset structure would like:
+
+```bash
+datasets/
+    Market-1501-v15.09.15/
+        bounding_box_test/
+        bounding_box_train/
+```
+
+## Expected dataset structure for [DukeMTMC-reID](https://openaccess.thecvf.com/content_ICCV_2017/papers/Zheng_Unlabeled_Samples_Generated_ICCV_2017_paper.pdf)
+
+1. Download datasets to `datasets/`
+2. Extract dataset. The dataset structure would like:
+
+```bash
+datasets/
+    DukeMTMC-reID/
+        bounding_box_train/
+        bounding_box_test/
+```
+
+## Expected dataset structure for [MSMT17](https://arxiv.org/abs/1711.08565)
+
+1. Download datasets to `datasets/`
+2. Extract dataset. The dataset structure would like:
+
+```bash
+datasets/
+    MSMT17_V2/
+        mask_train_v2/
+        mask_test_v2/
+```

demo/README.md

+# FastReID Demo
+
+We provide a command line tool to run a simple demo of builtin models.
+
+You can run this command to get cosine similarites between different images
+
+```bash
+python demo/visualize_result.py --config-file logs/dukemtmc/mgn_R50-ibn/config.yaml \
+--parallel --vis-label --dataset-name DukeMTMC --output logs/mgn_duke_vis \
+--opts MODEL.WEIGHTS logs/dukemtmc/mgn_R50-ibn/model_final.pth
+```

demo/demo.py

+# encoding: utf-8
+"""
+@author:  liaoxingyu
+@contact: sherlockliao01@gmail.com
+"""
+
+import argparse
+import glob
+import os
+import sys
+
+import torch.nn.functional as F
+import cv2
+import numpy as np
+import tqdm
+from torch.backends import cudnn
+
+sys.path.append('.')
+
+from fastreid.config import get_cfg
+from fastreid.utils.logger import setup_logger
+from fastreid.utils.file_io import PathManager
+
+from predictor import FeatureExtractionDemo
+
+# import some modules added in project like this below
+# sys.path.append("projects/PartialReID")
+# from partialreid import *
+
+cudnn.benchmark = True
+setup_logger(name="fastreid")
+
+
+def setup_cfg(args):
+    # load config from file and command-line arguments
+    cfg = get_cfg()
+    # add_partialreid_config(cfg)
+    cfg.merge_from_file(args.config_file)
+    cfg.merge_from_list(args.opts)
+    cfg.freeze()
+    return cfg
+
+
+def get_parser():
+    parser = argparse.ArgumentParser(description="Feature extraction with reid models")
+    parser.add_argument(
+        "--config-file",
+        metavar="FILE",
+        help="path to config file",
+    )
+    parser.add_argument(
+        "--parallel",
+        action='store_true',
+        help='If use multiprocess for feature extraction.'
+    )
+    parser.add_argument(
+        "--input",
+        nargs="+",
+        help="A list of space separated input images; "
+             "or a single glob pattern such as 'directory/*.jpg'",
+    )
+    parser.add_argument(
+        "--output",
+        default='demo_output',
+        help='path to save features'
+    )
+    parser.add_argument(
+        "--opts",
+        help="Modify config options using the command-line 'KEY VALUE' pairs",
+        default=[],
+        nargs=argparse.REMAINDER,
+    )
+    return parser
+
+
+def postprocess(features):
+    # Normalize feature to compute cosine distance
+    features = F.normalize(features)
+    features = features.cpu().data.numpy()
+    return features
+
+
+if __name__ == '__main__':
+    args = get_parser().parse_args()
+    cfg = setup_cfg(args)
+    demo = FeatureExtractionDemo(cfg, parallel=args.parallel)
+
+    PathManager.mkdirs(args.output)
+    if args.input:
+        if PathManager.isdir(args.input[0]):
+            args.input = glob.glob(os.path.expanduser(args.input[0]))
+            assert args.input, "The input path(s) was not found"
+        for path in tqdm.tqdm(args.input):
+            img = cv2.imread(path)
+            feat = demo.run_on_image(img)
+            feat = postprocess(feat)
+            np.save(os.path.join(args.output, os.path.basename(path).split('.')[0] + '.npy'), feat)

demo/plot_roc_with_pickle.py

+# encoding: utf-8
+"""
+@author:  xingyu liao
+@contact: sherlockliao01@gmail.com
+"""
+
+import matplotlib.pyplot as plt
+import sys
+
+sys.path.append('.')
+from fastreid.utils.visualizer import Visualizer
+
+if __name__ == "__main__":
+    baseline_res = Visualizer.load_roc_info("logs/duke_vis/roc_info.pickle")
+    mgn_res = Visualizer.load_roc_info("logs/mgn_duke_vis/roc_info.pickle")
+
+    fig = Visualizer.plot_roc_curve(baseline_res['fpr'], baseline_res['tpr'], name='baseline')
+    Visualizer.plot_roc_curve(mgn_res['fpr'], mgn_res['tpr'], name='mgn', fig=fig)
+    plt.savefig('roc.jpg')
+
+    fig = Visualizer.plot_distribution(baseline_res['pos'], baseline_res['neg'], name='baseline')
+    Visualizer.plot_distribution(mgn_res['pos'], mgn_res['neg'], name='mgn', fig=fig)
+    plt.savefig('dist.jpg')

demo/predictor.py

+# encoding: utf-8
+"""
+@author:  xingyu liao
+@contact: sherlockliao01@gmail.com
+"""
+
+import atexit
+import bisect
+from collections import deque
+
+import cv2
+import torch
+import torch.multiprocessing as mp
+
+from fastreid.engine import DefaultPredictor
+
+try:
+    mp.set_start_method('spawn')
+except RuntimeError:
+    pass
+
+
+class FeatureExtractionDemo(object):
+    def __init__(self, cfg, parallel=False):
+        """
+        Args:
+            cfg (CfgNode):
+            parallel (bool) whether to run the model in different processes from visualization.:
+                Useful since the visualization logic can be slow.
+        """
+        self.cfg = cfg
+        self.parallel = parallel
+
+        if parallel:
+            self.num_gpus = torch.cuda.device_count()
+            self.predictor = AsyncPredictor(cfg, self.num_gpus)
+        else:
+            self.predictor = DefaultPredictor(cfg)
+
+    def run_on_image(self, original_image):
+        """
+
+        Args:
+            original_image (np.ndarray): an image of shape (H, W, C) (in BGR order).
+                This is the format used by OpenCV.
+
+        Returns:
+            predictions (np.ndarray): normalized feature of the model.
+        """
+        # the model expects RGB inputs
+        original_image = original_image[:, :, ::-1]
+        # Apply pre-processing to image.
+        image = cv2.resize(original_image, tuple(self.cfg.INPUT.SIZE_TEST[::-1]), interpolation=cv2.INTER_CUBIC)
+        # Make shape with a new batch dimension which is adapted for
+        # network input
+        image = torch.as_tensor(image.astype("float32").transpose(2, 0, 1))[None]
+        predictions = self.predictor(image)
+        return predictions
+
+    def run_on_loader(self, data_loader):
+        if self.parallel:
+            buffer_size = self.predictor.default_buffer_size
+
+            batch_data = deque()
+
+            for cnt, batch in enumerate(data_loader):
+                batch_data.append(batch)
+                self.predictor.put(batch["images"])
+
+                if cnt >= buffer_size:
+                    batch = batch_data.popleft()
+                    predictions = self.predictor.get()
+                    yield predictions, batch["targets"].cpu().numpy(), batch["camids"].cpu().numpy()
+
+            while len(batch_data):
+                batch = batch_data.popleft()
+                predictions = self.predictor.get()
+                yield predictions, batch["targets"].cpu().numpy(), batch["camids"].cpu().numpy()
+        else:
+            for batch in data_loader:
+                predictions = self.predictor(batch["images"])
+                yield predictions, batch["targets"].cpu().numpy(), batch["camids"].cpu().numpy()
+
+
+class AsyncPredictor:
+    """
+    A predictor that runs the model asynchronously, possibly on >1 GPUs.
+    Because when the amount of data is large.
+    """
+
+    class _StopToken:
+        pass
+
+    class _PredictWorker(mp.Process):
+        def __init__(self, cfg, task_queue, result_queue):
+            self.cfg = cfg
+            self.task_queue = task_queue
+            self.result_queue = result_queue
+            super().__init__()
+
+        def run(self):
+            predictor = DefaultPredictor(self.cfg)
+
+            while True:
+                task = self.task_queue.get()
+                if isinstance(task, AsyncPredictor._StopToken):
+                    break
+                idx, data = task
+                result = predictor(data)
+                self.result_queue.put((idx, result))
+
+    def __init__(self, cfg, num_gpus: int = 1):
+        """
+
+        Args:
+            cfg (CfgNode):
+            num_gpus (int): if 0, will run on CPU
+        """
+        num_workers = max(num_gpus, 1)
+        self.task_queue = mp.Queue(maxsize=num_workers * 3)
+        self.result_queue = mp.Queue(maxsize=num_workers * 3)
+        self.procs = []
+        for gpuid in range(max(num_gpus, 1)):
+            cfg = cfg.clone()
+            cfg.defrost()
+            cfg.MODEL.DEVICE = "cuda:{}".format(gpuid) if num_gpus > 0 else "cpu"
+            self.procs.append(
+                AsyncPredictor._PredictWorker(cfg, self.task_queue, self.result_queue)
+            )
+
+        self.put_idx = 0
+        self.get_idx = 0
+        self.result_rank = []
+        self.result_data = []
+
+        for p in self.procs:
+            p.start()
+
+        atexit.register(self.shutdown)
+
+    def put(self, image):
+        self.put_idx += 1
+        self.task_queue.put((self.put_idx, image))
+
+    def get(self):
+        self.get_idx += 1
+        if len(self.result_rank) and self.result_rank[0] == self.get_idx:
+            res = self.result_data[0]
+            del self.result_data[0], self.result_rank[0]
+            return res
+
+        while True:
+            # Make sure the results are returned in the correct order
+            idx, res = self.result_queue.get()
+            if idx == self.get_idx:
+                return res
+            insert = bisect.bisect(self.result_rank, idx)
+            self.result_rank.insert(insert, idx)
+            self.result_data.insert(insert, res)
+
+    def __len__(self):
+        return self.put_idx - self.get_idx
+
+    def __call__(self, image):
+        self.put(image)
+        return self.get()
+
+    def shutdown(self):
+        for _ in self.procs:
+            self.task_queue.put(AsyncPredictor._StopToken())
+
+    @property
+    def default_buffer_size(self):
+        return len(self.procs) * 5

demo/visualize_result.py

+# encoding: utf-8
+"""
+@author:  xingyu liao
+@contact: sherlockliao01@gmail.com
+"""
+
+import argparse
+import logging
+import sys
+
+import numpy as np
+import torch
+import tqdm
+from torch.backends import cudnn
+
+sys.path.append('.')
+
+from fastreid.evaluation import evaluate_rank
+from fastreid.config import get_cfg
+from fastreid.utils.logger import setup_logger
+from fastreid.data import build_reid_test_loader
+from predictor import FeatureExtractionDemo
+from fastreid.utils.visualizer import Visualizer
+
+# import some modules added in project
+# for example, add partial reid like this below
+# sys.path.append("projects/PartialReID")
+# from partialreid import *
+
+cudnn.benchmark = True
+setup_logger(name="fastreid")
+
+logger = logging.getLogger('fastreid.visualize_result')
+
+
+def setup_cfg(args):
+    # load config from file and command-line arguments
+    cfg = get_cfg()
+    # add_partialreid_config(cfg)
+    cfg.merge_from_file(args.config_file)
+    cfg.merge_from_list(args.opts)
+    cfg.freeze()
+    return cfg
+
+
+def get_parser():
+    parser = argparse.ArgumentParser(description="Feature extraction with reid models")
+    parser.add_argument(
+        "--config-file",
+        metavar="FILE",
+        help="path to config file",
+    )
+    parser.add_argument(
+        '--parallel',
+        action='store_true',
+        help='if use multiprocess for feature extraction.'
+    )
+    parser.add_argument(
+        "--dataset-name",
+        help="a test dataset name for visualizing ranking list."
+    )
+    parser.add_argument(
+        "--output",
+        default="./vis_rank_list",
+        help="a file or directory to save rankling list result.",
+
+    )
+    parser.add_argument(
+        "--vis-label",
+        action='store_true',
+        help="if visualize label of query instance"
+    )
+    parser.add_argument(
+        "--num-vis",
+        default=100,
+        help="number of query images to be visualized",
+    )
+    parser.add_argument(
+        "--rank-sort",
+        default="ascending",
+        help="rank order of visualization images by AP metric",
+    )
+    parser.add_argument(
+        "--label-sort",
+        default="ascending",
+        help="label order of visualization images by cosine similarity metric",
+    )
+    parser.add_argument(
+        "--max-rank",
+        default=10,
+        help="maximum number of rank list to be visualized",
+    )
+    parser.add_argument(
+        "--opts",
+        help="Modify config options using the command-line 'KEY VALUE' pairs",
+        default=[],
+        nargs=argparse.REMAINDER,
+    )
+    return parser
+
+
+if __name__ == '__main__':
+    args = get_parser().parse_args()
+    cfg = setup_cfg(args)
+    test_loader, num_query = build_reid_test_loader(cfg, dataset_name=args.dataset_name)
+    demo = FeatureExtractionDemo(cfg, parallel=args.parallel)
+
+    logger.info("Start extracting image features")
+    feats = []
+    pids = []
+    camids = []
+    for (feat, pid, camid) in tqdm.tqdm(demo.run_on_loader(test_loader), total=len(test_loader)):
+        feats.append(feat)
+        pids.extend(pid)
+        camids.extend(camid)
+
+    feats = torch.cat(feats, dim=0)
+    q_feat = feats[:num_query]
+    g_feat = feats[num_query:]
+    q_pids = np.asarray(pids[:num_query])
+    g_pids = np.asarray(pids[num_query:])
+    q_camids = np.asarray(camids[:num_query])
+    g_camids = np.asarray(camids[num_query:])
+
+    # compute cosine distance
+    distmat = 1 - torch.mm(q_feat, g_feat.t())
+    distmat = distmat.numpy()
+
+    logger.info("Computing APs for all query images ...")
+    cmc, all_ap, all_inp = evaluate_rank(distmat, q_pids, g_pids, q_camids, g_camids)
+    logger.info("Finish computing APs for all query images!")
+
+    visualizer = Visualizer(test_loader.dataset)
+    visualizer.get_model_output(all_ap, distmat, q_pids, g_pids, q_camids, g_camids)
+
+    logger.info("Start saving ROC curve ...")
+    fpr, tpr, pos, neg = visualizer.vis_roc_curve(args.output)
+    visualizer.save_roc_info(args.output, fpr, tpr, pos, neg)
+    logger.info("Finish saving ROC curve!")
+
+    logger.info("Saving rank list result ...")
+    query_indices = visualizer.vis_rank_list(args.output, args.vis_label, args.num_vis,
+                                             args.rank_sort, args.label_sort, args.max_rank)
+    logger.info("Finish saving rank list results!")