populate meta-arch registry when importing d2go

Summary:
Pull Request resolved: https://github.com/facebookresearch/d2go/pull/130

We want to make sure that after importing `d2go.modeling` all the meta-arch is registered.

Reviewed By: Maninae

Differential Revision: D31904303

fbshipit-source-id: 3f32b65b764b2458e2fb9c4e0bbd99824b37ecfc

d2go/export/api.py

@@ -39,7 +39,6 @@ else:
 
 import torch
 import torch.nn as nn
-from d2go.modeling.quantization import post_training_quantize
 from detectron2.utils.file_io import PathManager
 from mobile_cv.arch.utils import fuse_utils
 from mobile_cv.common.misc.file_utils import make_temp_directory
@@ -106,10 +105,13 @@ def convert_predictor(
                 "The model is not quantized during training, running post"
                 " training quantization ..."
             )
+            # delayed import to avoid circular import since d2go.modeling depends on d2go.export
+            from d2go.modeling.quantization import post_training_quantize
+
             pytorch_model = post_training_quantize(cfg, pytorch_model, data_loader)
             # only check bn exists in ptq as qat still has bn inside fused ops
             if fuse_utils.check_bn_exist(pytorch_model):
-                logger.warn(f"Post training quantized model has bn inside fused ops")
+                logger.warn("Post training quantized model has bn inside fused ops")
         logger.info(f"Converting quantized model {cfg.QUANTIZATION.BACKEND}...")
 
         if cfg.QUANTIZATION.EAGER_MODE:

d2go/modeling/__init__.py

@@ -4,5 +4,12 @@
 
 # NOTE: making necessary imports to register with Registery
 from . import backbone  # noqa
+from . import meta_arch  # noqa
 from . import modeldef  # noqa
-from .meta_arch.build import build_model  # noqa
+
+# namespace forwarding
+from .meta_arch.build import build_model
+
+__all__ = [
+    "build_model",
+]

d2go/modeling/meta_arch/__init__.py

+#!/usr/bin/env python3
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+
+# NOTE: making necessary imports to register with Registry

tests/misc/test_registration.py

+#!/usr/bin/env python3
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+
+import importlib
+import logging
+import pkgutil
+import unittest
+
+# NOTE: don't import anything related to D2/D2Go so that the test of one registry is
+# isolated from others
+
+
+logger = logging.getLogger(__name__)
+
+
+# copied from https://stackoverflow.com/questions/3365740/how-to-import-all-submodules
+def import_submodules(package, recursive=True):
+    """Import all submodules of a module, recursively, including subpackages
+
+    :param package: package (name or actual module)
+    :type package: str | module
+    :rtype: dict[str, types.ModuleType]
+    """
+    if isinstance(package, str):
+        package = importlib.import_module(package)
+    results = {}
+    for _loader, name, is_pkg in pkgutil.walk_packages(package.__path__):
+        full_name = package.__name__ + "." + name
+        results[full_name] = importlib.import_module(full_name)
+        if recursive and is_pkg:
+            results.update(import_submodules(full_name))
+    return results
+
+
+class BaseRegistryPopulationTests(object):
+    """
+    Test D2Go's core registries are populated once top-level module is imported.
+    """
+
+    def get_registered_items(self):
+        """return a list of registered items"""
+        raise NotImplementedError()
+
+    def import_all_modules(self):
+        """import all related modules"""
+        raise NotImplementedError()
+
+    def test_is_polulated(self):
+        registered_before_import_all = self.get_registered_items()
+        self.import_all_modules()
+        registered_after_import_all = self.get_registered_items()
+        self.assertEqual(registered_before_import_all, registered_after_import_all)
+
+
+class TestMetaArchRegistryPopulation(unittest.TestCase, BaseRegistryPopulationTests):
+    def setUp(self):
+        import d2go.modeling.meta_arch
+
+        self._package = d2go.modeling.meta_arch
+
+    def get_registered_items(self):
+        from detectron2.modeling import META_ARCH_REGISTRY
+
+        return [k for k, v in META_ARCH_REGISTRY]
+
+    def import_all_modules(self):
+        import_submodules(self._package)