Unverified Commit 4a3f90f6 authored by VVsssssk's avatar VVsssssk Committed by GitHub
Browse files

[FIx]Replace mmcv.fileclient to mmengine.fileclient (#1737)

* fix mmcv file client

* fix set_env

* fix mmcv file load and dump
parent 47bb07f7
......@@ -23,6 +23,7 @@ import subprocess
from os import path as osp
import mmcv
import mmengine
import torch
# build schedule look-up table to automatically find the final model
......@@ -222,7 +223,7 @@ def main():
models = dict(models=publish_model_infos)
print(f'Totally gathered {len(publish_model_infos)} models')
mmcv.dump(models, osp.join(models_out, 'model_info.json'))
mmengine.dump(models, osp.join(models_out, 'model_info.json'))
if __name__ == '__main__':
......
......@@ -4,7 +4,7 @@ from copy import deepcopy
from os import path as osp
from typing import Sequence, Union
import mmcv
import mmengine
import numpy as np
import torch
import torch.nn as nn
......@@ -187,7 +187,7 @@ def inference_multi_modality_detector(model: nn.Module,
box_type_3d, box_mode_3d = \
get_box_type(cfg.test_dataloader.dataset.box_type_3d)
data_list = mmcv.load(ann_file)['data_list']
data_list = mmengine.load(ann_file)['data_list']
assert len(imgs) == len(data_list)
data = []
......@@ -269,7 +269,7 @@ def inference_mono_3d_detector(model: nn.Module,
box_type_3d, box_mode_3d = \
get_box_type(cfg.test_dataloader.dataset.box_type_3d)
data_list = mmcv.load(ann_file)
data_list = mmengine.load(ann_file)
assert len(imgs) == len(data_list)
data = []
......
......@@ -3,7 +3,7 @@ import copy
from os import path as osp
from typing import Callable, List, Optional, Union
import mmcv
import mmengine
import numpy as np
from mmengine.dataset import BaseDataset
......@@ -75,7 +75,7 @@ class Det3DDataset(BaseDataset):
file_client_args: dict = dict(backend='disk'),
**kwargs):
# init file client
self.file_client = mmcv.FileClient(**file_client_args)
self.file_client = mmengine.FileClient(**file_client_args)
self.filter_empty_gt = filter_empty_gt
self.load_eval_anns = load_eval_anns
_default_modality_keys = ('use_lidar', 'use_camera')
......
# Copyright (c) OpenMMLab. All rights reserved.
import mmcv
import mmengine
import numpy as np
from mmdet3d.datasets import Det3DDataset
......@@ -87,7 +87,7 @@ class Kitti2DDataset(Det3DDataset):
Returns:
list[dict]: List of annotations.
"""
self.data_infos = mmcv.load(ann_file)
self.data_infos = mmengine.load(ann_file)
self.cat2label = {
cat_name: i
for i, cat_name in enumerate(self.CLASSES)
......
......@@ -2,7 +2,7 @@
from os import path as osp
from typing import Callable, Dict, List, Optional, Sequence, Union
import mmcv
import mmengine
import numpy as np
from mmengine.dataset import BaseDataset
......@@ -72,7 +72,7 @@ class Seg3DDataset(BaseDataset):
file_client_args: dict = dict(backend='disk'),
**kwargs) -> None:
# init file client
self.file_client = mmcv.FileClient(**file_client_args)
self.file_client = mmengine.FileClient(**file_client_args)
self.modality = modality
self.load_eval_anns = load_eval_anns
......
......@@ -3,7 +3,7 @@ import copy
import os
import warnings
import mmcv
import mmengine
import numpy as np
from mmdet3d.datasets.transforms import data_augment_utils
......@@ -115,20 +115,21 @@ class DataBaseSampler(object):
self.cat2label = {name: i for i, name in enumerate(classes)}
self.label2cat = {i: name for i, name in enumerate(classes)}
self.points_loader = TRANSFORMS.build(points_loader)
self.file_client = mmcv.FileClient(**file_client_args)
self.file_client = mmengine.FileClient(**file_client_args)
# load data base infos
if hasattr(self.file_client, 'get_local_path'):
with self.file_client.get_local_path(info_path) as local_path:
# loading data from a file-like object needs file format
db_infos = mmcv.load(open(local_path, 'rb'), file_format='pkl')
db_infos = mmengine.load(
open(local_path, 'rb'), file_format='pkl')
else:
warnings.warn(
'The used MMCV version does not have get_local_path. '
f'We treat the {info_path} as local paths and it '
'might cause errors if the path is not a local path. '
'Please use MMCV>= 1.3.16 if you meet errors.')
db_infos = mmcv.load(info_path)
db_infos = mmengine.load(info_path)
# filter database infos
from mmengine.logging import MMLogger
......
......@@ -2,6 +2,7 @@
from typing import List
import mmcv
import mmengine
import numpy as np
from mmcv.transforms import LoadImageFromFile
from mmcv.transforms.base import BaseTransform
......@@ -176,7 +177,7 @@ class LoadPointsFromMultiSweeps(BaseTransform):
np.ndarray: An array containing point clouds data.
"""
if self.file_client is None:
self.file_client = mmcv.FileClient(**self.file_client_args)
self.file_client = mmengine.FileClient(**self.file_client_args)
try:
pts_bytes = self.file_client.get(pts_filename)
points = np.frombuffer(pts_bytes, dtype=np.float32)
......@@ -434,7 +435,7 @@ class LoadPointsFromFile(BaseTransform):
np.ndarray: An array containing point clouds data.
"""
if self.file_client is None:
self.file_client = mmcv.FileClient(**self.file_client_args)
self.file_client = mmengine.FileClient(**self.file_client_args)
try:
pts_bytes = self.file_client.get(pts_filename)
points = np.frombuffer(pts_bytes, dtype=np.float32)
......@@ -690,7 +691,7 @@ class LoadAnnotations3D(LoadAnnotations):
pts_instance_mask_path = results['pts_instance_mask_path']
if self.file_client is None:
self.file_client = mmcv.FileClient(**self.file_client_args)
self.file_client = mmengine.FileClient(**self.file_client_args)
try:
mask_bytes = self.file_client.get(pts_instance_mask_path)
pts_instance_mask = np.frombuffer(mask_bytes, dtype=np.int64)
......@@ -717,7 +718,7 @@ class LoadAnnotations3D(LoadAnnotations):
pts_semantic_mask_path = results['pts_semantic_mask_path']
if self.file_client is None:
self.file_client = mmcv.FileClient(**self.file_client_args)
self.file_client = mmengine.FileClient(**self.file_client_args)
try:
mask_bytes = self.file_client.get(pts_semantic_mask_path)
# add .copy() to fix read-only bug
......
......@@ -2,6 +2,7 @@
from os import path as osp
import mmcv
import mmengine
import numpy as np
from lyft_dataset_sdk.eval.detection.mAP_evaluation import (Box3D, get_ap,
get_class_names,
......@@ -80,7 +81,7 @@ def load_lyft_predictions(res_path):
Returns:
list[dict]: List of prediction dictionaries.
"""
predictions = mmcv.load(res_path)
predictions = mmengine.load(res_path)
predictions = predictions['results']
all_preds = []
for sample_token in predictions.keys():
......@@ -136,7 +137,7 @@ def lyft_eval(lyft, data_root, res_path, eval_set, output_dir, logger=None):
print_log(APs_table.table, logger=logger)
res_path = osp.join(output_dir, 'lyft_metrics.json')
mmcv.dump(metrics, res_path)
mmengine.dump(metrics, res_path)
return metrics
......
......@@ -4,6 +4,7 @@ from os import path as osp
from typing import Dict, List, Optional, Sequence, Union
import mmcv
import mmengine
import numpy as np
import torch
from mmcv.utils import print_log
......@@ -426,7 +427,7 @@ class KittiMetric(BaseMetric):
out = f'{pklfile_prefix}.pkl'
else:
out = pklfile_prefix
mmcv.dump(det_annos, out)
mmengine.dump(det_annos, out)
print(f'Result is saved to {out}.')
return det_annos
......@@ -517,7 +518,7 @@ class KittiMetric(BaseMetric):
out = f'{pklfile_prefix}.pkl'
else:
out = pklfile_prefix
mmcv.dump(det_annos, out)
mmengine.dump(det_annos, out)
print(f'Result is saved to {out}.')
if submission_prefix is not None:
......
......@@ -6,6 +6,7 @@ from os import path as osp
from typing import Dict, List, Optional, Sequence, Tuple, Union
import mmcv
import mmengine
import numpy as np
import pandas as pd
from lyft_dataset_sdk.lyftdataset import LyftDataset as Lyft
......@@ -192,7 +193,7 @@ class LyftMetric(BaseMetric):
json_path (str): Path of the result json file.
csv_savepath (str): Path to save the csv file.
"""
results = mmcv.load(json_path)['results']
results = mmengine.load(json_path)['results']
sample_list_path = osp.join(self.data_root, 'sample_submission.csv')
data = pd.read_csv(sample_list_path)
Id_list = list(data['Id'])
......@@ -269,7 +270,7 @@ class LyftMetric(BaseMetric):
mmcv.mkdir_or_exist(jsonfile_prefix)
res_path = osp.join(jsonfile_prefix, 'results_lyft.json')
print('Results writes to', res_path)
mmcv.dump(lyft_submissions, res_path)
mmengine.dump(lyft_submissions, res_path)
return res_path
def lyft_evaluate(self,
......
......@@ -5,6 +5,7 @@ from os import path as osp
from typing import Dict, List, Optional, Sequence, Tuple, Union
import mmcv
import mmengine
import numpy as np
import pyquaternion
import torch
......@@ -235,7 +236,7 @@ class NuScenesMetric(BaseMetric):
nusc_eval.main(render_curves=False)
# record metrics
metrics = mmcv.load(osp.join(output_dir, 'metrics_summary.json'))
metrics = mmengine.load(osp.join(output_dir, 'metrics_summary.json'))
detail = dict()
metric_prefix = f'{result_name}_NuScenes'
for name in classes:
......@@ -464,7 +465,7 @@ class NuScenesMetric(BaseMetric):
mmcv.mkdir_or_exist(jsonfile_prefix)
res_path = osp.join(jsonfile_prefix, 'results_nusc.json')
print('Results writes to', res_path)
mmcv.dump(nusc_submissions, res_path)
mmengine.dump(nusc_submissions, res_path)
return res_path
def _format_lidar_bbox(self,
......@@ -538,7 +539,7 @@ class NuScenesMetric(BaseMetric):
mmcv.mkdir_or_exist(jsonfile_prefix)
res_path = osp.join(jsonfile_prefix, 'results_nusc.json')
print('Results writes to', res_path)
mmcv.dump(nusc_submissions, res_path)
mmengine.dump(nusc_submissions, res_path)
return res_path
......
......@@ -4,6 +4,7 @@ from os import path as osp
from typing import Dict, List, Optional, Union
import mmcv
import mmengine
import numpy as np
import torch
from mmcv.utils import print_log
......@@ -443,7 +444,7 @@ class WaymoMetric(KittiMetric):
out = f'{pklfile_prefix}.pkl'
else:
out = pklfile_prefix
mmcv.dump(det_annos, out)
mmengine.dump(det_annos, out)
print(f'Result is saved to {out}.')
return det_annos
......@@ -68,7 +68,6 @@ def register_all_modules(init_default_scope: bool = True) -> None:
Defaults to True.
""" # noqa
import mmdet3d.datasets # noqa: F401,F403
import mmdet3d.engine.scheduler # noqa: F401,F403
import mmdet3d.evaluation.metrics # noqa: F401,F403
import mmdet3d.structures # noqa: F401,F403
import mmdet3d.visualization # noqa: F401,F403
......
# Copyright (c) OpenMMLab. All rights reserved.
import os
import mmcv
import mmengine
import numpy as np
from tools.dataset_converters.s3dis_data_utils import S3DISData, S3DISSegData
......@@ -54,17 +54,17 @@ def create_indoor_info_file(data_path,
infos_train = train_dataset.get_infos(
num_workers=workers, has_label=True)
mmcv.dump(infos_train, train_filename, 'pkl')
mmengine.dump(infos_train, train_filename, 'pkl')
print(f'{pkl_prefix} info train file is saved to {train_filename}')
infos_val = val_dataset.get_infos(num_workers=workers, has_label=True)
mmcv.dump(infos_val, val_filename, 'pkl')
mmengine.dump(infos_val, val_filename, 'pkl')
print(f'{pkl_prefix} info val file is saved to {val_filename}')
if pkl_prefix == 'scannet':
infos_test = test_dataset.get_infos(
num_workers=workers, has_label=False)
mmcv.dump(infos_test, test_filename, 'pkl')
mmengine.dump(infos_test, test_filename, 'pkl')
print(f'{pkl_prefix} info test file is saved to {test_filename}')
# generate infos for the semantic segmentation task
......@@ -100,7 +100,7 @@ def create_indoor_info_file(data_path,
info = dataset.get_infos(num_workers=workers, has_label=True)
filename = os.path.join(save_path,
f'{pkl_prefix}_infos_{split}.pkl')
mmcv.dump(info, filename, 'pkl')
mmengine.dump(info, filename, 'pkl')
print(f'{pkl_prefix} info {split} file is saved to {filename}')
seg_dataset = S3DISSegData(
data_root=data_path,
......
......@@ -3,6 +3,7 @@ from collections import OrderedDict
from pathlib import Path
import mmcv
import mmengine
import numpy as np
from nuscenes.utils.geometry_utils import view_points
......@@ -196,7 +197,7 @@ def create_kitti_info_file(data_path,
_calculate_num_points_in_gt(data_path, kitti_infos_train, relative_path)
filename = save_path / f'{pkl_prefix}_infos_train.pkl'
print(f'Kitti info train file is saved to {filename}')
mmcv.dump(kitti_infos_train, filename)
mmengine.dump(kitti_infos_train, filename)
kitti_infos_val = get_kitti_image_info(
data_path,
training=True,
......@@ -208,10 +209,10 @@ def create_kitti_info_file(data_path,
_calculate_num_points_in_gt(data_path, kitti_infos_val, relative_path)
filename = save_path / f'{pkl_prefix}_infos_val.pkl'
print(f'Kitti info val file is saved to {filename}')
mmcv.dump(kitti_infos_val, filename)
mmengine.dump(kitti_infos_val, filename)
filename = save_path / f'{pkl_prefix}_infos_trainval.pkl'
print(f'Kitti info trainval file is saved to {filename}')
mmcv.dump(kitti_infos_train + kitti_infos_val, filename)
mmengine.dump(kitti_infos_train + kitti_infos_val, filename)
kitti_infos_test = get_kitti_image_info(
data_path,
......@@ -224,7 +225,7 @@ def create_kitti_info_file(data_path,
relative_path=relative_path)
filename = save_path / f'{pkl_prefix}_infos_test.pkl'
print(f'Kitti info test file is saved to {filename}')
mmcv.dump(kitti_infos_test, filename)
mmengine.dump(kitti_infos_test, filename)
def create_waymo_info_file(data_path,
......@@ -288,19 +289,19 @@ def create_waymo_info_file(data_path,
num_points_in_gt_calculater.calculate(waymo_infos_train)
filename = save_path / f'{pkl_prefix}_infos_train.pkl'
print(f'Waymo info train file is saved to {filename}')
mmcv.dump(waymo_infos_train, filename)
mmengine.dump(waymo_infos_train, filename)
waymo_infos_val = waymo_infos_gatherer_trainval.gather(val_img_ids)
num_points_in_gt_calculater.calculate(waymo_infos_val)
filename = save_path / f'{pkl_prefix}_infos_val.pkl'
print(f'Waymo info val file is saved to {filename}')
mmcv.dump(waymo_infos_val, filename)
mmengine.dump(waymo_infos_val, filename)
filename = save_path / f'{pkl_prefix}_infos_trainval.pkl'
print(f'Waymo info trainval file is saved to {filename}')
mmcv.dump(waymo_infos_train + waymo_infos_val, filename)
mmengine.dump(waymo_infos_train + waymo_infos_val, filename)
waymo_infos_test = waymo_infos_gatherer_test.gather(test_img_ids)
filename = save_path / f'{pkl_prefix}_infos_test.pkl'
print(f'Waymo info test file is saved to {filename}')
mmcv.dump(waymo_infos_test, filename)
mmengine.dump(waymo_infos_test, filename)
def _create_reduced_point_cloud(data_path,
......@@ -322,7 +323,7 @@ def _create_reduced_point_cloud(data_path,
front_camera_id (int, optional): The referenced/front camera ID.
Default: 2.
"""
kitti_infos = mmcv.load(info_path)
kitti_infos = mmengine.load(info_path)
for info in mmcv.track_iter_progress(kitti_infos):
pc_info = info['point_cloud']
......@@ -419,7 +420,7 @@ def export_2d_annotation(root_path, info_path, mono3d=True):
Default: True.
"""
# get bbox annotations for camera
kitti_infos = mmcv.load(info_path)
kitti_infos = mmengine.load(info_path)
cat2Ids = [
dict(id=kitti_categories.index(cat_name), name=cat_name)
for cat_name in kitti_categories
......@@ -454,7 +455,7 @@ def export_2d_annotation(root_path, info_path, mono3d=True):
json_prefix = f'{info_path[:-4]}_mono3d'
else:
json_prefix = f'{info_path[:-4]}'
mmcv.dump(coco_2d_dict, f'{json_prefix}.coco.json')
mmengine.dump(coco_2d_dict, f'{json_prefix}.coco.json')
def get_2d_boxes(info, occluded, mono3d=True):
......
......@@ -4,6 +4,7 @@ from logging import warning
from os import path as osp
import mmcv
import mmengine
import numpy as np
from lyft_dataset_sdk.lyftdataset import LyftDataset as Lyft
from pyquaternion import Quaternion
......@@ -77,18 +78,18 @@ def create_lyft_infos(root_path,
data = dict(infos=train_lyft_infos, metadata=metadata)
info_name = f'{info_prefix}_infos_test'
info_path = osp.join(root_path, f'{info_name}.pkl')
mmcv.dump(data, info_path)
mmengine.dump(data, info_path)
else:
print(f'train sample: {len(train_lyft_infos)}, \
val sample: {len(val_lyft_infos)}')
data = dict(infos=train_lyft_infos, metadata=metadata)
train_info_name = f'{info_prefix}_infos_train'
info_path = osp.join(root_path, f'{train_info_name}.pkl')
mmcv.dump(data, info_path)
mmengine.dump(data, info_path)
data['infos'] = val_lyft_infos
val_info_name = f'{info_prefix}_infos_val'
info_val_path = osp.join(root_path, f'{val_info_name}.pkl')
mmcv.dump(data, info_val_path)
mmengine.dump(data, info_val_path)
def _fill_trainval_infos(lyft,
......@@ -234,7 +235,7 @@ def export_2d_annotation(root_path, info_path, version):
'CAM_BACK_LEFT',
'CAM_BACK_RIGHT',
]
lyft_infos = mmcv.load(info_path)['infos']
lyft_infos = mmengine.load(info_path)['infos']
lyft = Lyft(
data_path=osp.join(root_path, version),
json_path=osp.join(root_path, version, version),
......@@ -268,4 +269,4 @@ def export_2d_annotation(root_path, info_path, version):
coco_info['id'] = coco_ann_id
coco_2d_dict['annotations'].append(coco_info)
coco_ann_id += 1
mmcv.dump(coco_2d_dict, f'{info_path[:-4]}.coco.json')
mmengine.dump(coco_2d_dict, f'{info_path[:-4]}.coco.json')
......@@ -4,6 +4,7 @@ import base64
from os import path as osp
import mmcv
import mmengine
import numpy as np
from nuimages import NuImages
from nuimages.utils.utils import mask_decode, name_to_index_mapping
......@@ -210,7 +211,7 @@ def export_nuim_to_coco(nuim, data_root, out_dir, extra_tag, version, nproc):
mmcv.mkdir_or_exist(out_dir)
out_file = osp.join(out_dir, f'{extra_tag}_{version}.json')
print(f'Annotation dumped to {out_file}')
mmcv.dump(coco_format_json, out_file)
mmengine.dump(coco_format_json, out_file)
def main():
......
......@@ -5,6 +5,7 @@ from os import path as osp
from typing import List, Tuple, Union
import mmcv
import mmengine
import numpy as np
from nuscenes.nuscenes import NuScenes
from nuscenes.utils.geometry_utils import view_points
......@@ -87,18 +88,18 @@ def create_nuscenes_infos(root_path,
data = dict(infos=train_nusc_infos, metadata=metadata)
info_path = osp.join(root_path,
'{}_infos_test.pkl'.format(info_prefix))
mmcv.dump(data, info_path)
mmengine.dump(data, info_path)
else:
print('train sample: {}, val sample: {}'.format(
len(train_nusc_infos), len(val_nusc_infos)))
data = dict(infos=train_nusc_infos, metadata=metadata)
info_path = osp.join(root_path,
'{}_infos_train.pkl'.format(info_prefix))
mmcv.dump(data, info_path)
mmengine.dump(data, info_path)
data['infos'] = val_nusc_infos
info_val_path = osp.join(root_path,
'{}_infos_val.pkl'.format(info_prefix))
mmcv.dump(data, info_val_path)
mmengine.dump(data, info_val_path)
def get_available_scenes(nusc):
......@@ -353,7 +354,7 @@ def export_2d_annotation(root_path, info_path, version, mono3d=True):
'CAM_BACK_LEFT',
'CAM_BACK_RIGHT',
]
nusc_infos = mmcv.load(info_path)['infos']
nusc_infos = mmengine.load(info_path)['infos']
nusc = NuScenes(version=version, dataroot=root_path, verbose=True)
# info_2d_list = []
cat2Ids = [
......@@ -396,7 +397,7 @@ def export_2d_annotation(root_path, info_path, version, mono3d=True):
json_prefix = f'{info_path[:-4]}_mono3d'
else:
json_prefix = f'{info_path[:-4]}'
mmcv.dump(coco_2d_dict, f'{json_prefix}.coco.json')
mmengine.dump(coco_2d_dict, f'{json_prefix}.coco.json')
def get_2d_boxes(nusc,
......
......@@ -4,6 +4,7 @@ from concurrent import futures as futures
from os import path as osp
import mmcv
import mmengine
import numpy as np
......@@ -171,7 +172,7 @@ class S3DISSegData(object):
num_points=4096,
label_weight_func=None):
self.data_root = data_root
self.data_infos = mmcv.load(ann_file)
self.data_infos = mmengine.load(ann_file)
self.split = split
self.num_points = num_points
......
......@@ -4,6 +4,7 @@ from concurrent import futures as futures
from os import path as osp
import mmcv
import mmengine
import numpy as np
......@@ -218,7 +219,7 @@ class ScanNetSegData(object):
num_points=8192,
label_weight_func=None):
self.data_root = data_root
self.data_infos = mmcv.load(ann_file)
self.data_infos = mmengine.load(ann_file)
self.split = split
assert split in ['train', 'val', 'test']
self.num_points = num_points
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment