misc updates

Summary:
- remove the `None` support for `merge_from_list`
- fix logging when initializing diskcacje
- Don't inherit `_FakeListObj` fron `list`, so looping over it can raise error.

Reviewed By: sstsai-adl

Differential Revision: D34952714

fbshipit-source-id: f636e408b79ed77904f257f189fcef216cb2efbc

d2go/config/config.py

@@ -20,8 +20,6 @@ CONFIG_CUSTOM_PARSE_REGISTRY = Registry("CONFIG_CUSTOM_PARSE")
 
 def _opts_to_dict(opts: List[str]):
     ret = {}
-    if opts is None:
-        return ret
     for full_key, v in zip(opts[0::2], opts[1::2]):
         keys = full_key.split(".")
         cur = ret

d2go/data/disk_cache.py

@@ -57,14 +57,13 @@ class DiskCachedDatasetFromList(data.Dataset):
             return np.frombuffer(buffer, dtype=np.uint8)
 
         logger.info(
-            "Serializing {} elements to byte tensors and concatenating them all ...".format(
-                len(self._lst)
-            )
+            "Serializing {} elements to byte tensors ...".format(len(self._lst))
         )
         self._lst = [_serialize(x) for x in self._lst]
+        total_size = sum(len(x) for x in self._lst)
         # TODO: only enabling DiskCachedDataset for large enough dataset
         logger.info(
-            "Serialized dataset takes {:.2f} MiB".format(len(self._lst) / 1024 ** 2)
+            "Serialized dataset takes {:.2f} MiB".format(total_size / 1024 ** 2)
         )
         self._initialize_diskcache()
 
@@ -138,9 +137,11 @@ class DiskCachedDatasetFromList(data.Dataset):
             self._chuck_size = _local_master_gather(
                 lambda: self._chuck_size, check_equal=True
             )[0]
+            logger.info("Gathered chuck size: {}".format(self._chuck_size))
 
         # free the memory of self._lst
         self._size = _local_master_gather(lambda: len(self._lst), check_equal=True)[0]
+        logger.info("Gathered list size: {}".format(self._size))
         del self._lst
 
     def _write_to_local_db(self, task):

d2go/data/utils.py

@@ -384,7 +384,7 @@ def update_cfg_if_using_adhoc_dataset(cfg):
     return cfg
 
 
-class _FakeListObj(list):
+class _FakeListObj(object):
     def __init__(self, size):
         self.size = size
 
@@ -392,7 +392,9 @@ class _FakeListObj(list):
         return self.size
 
     def __getitem__(self, idx):
-        raise NotImplementedError()
+        raise NotImplementedError(
+            "This is a fake list, accessing this list should not happen"
+        )
 
 
 def local_master_get_detection_dataset_dicts(*args, **kwargs):