Update config to support predefined variables (#348)

* update config with predefined variables

* rm redun
Signed-off-by: lixuanyi <lixuanyi@sensetime.com>

* add test for config
Signed-off-by: lixuanyi <lixuanyi@sensetime.com>

* support all types
Signed-off-by: lixuanyi <lixuanyi@sensetime.com>

* newline at the end
Signed-off-by: lixuanyi <lixuanyi@sensetime.com>

* update
Signed-off-by: lixuanyi <lixuanyi@sensetime.com>

* extract code into a function and add docs
Signed-off-by: lixuanyi <lixuanyi@sensetime.com>

* fix and add tests
Signed-off-by: lixuanyi <lixuanyi@sensetime.com>

* add unit tests and fix

* fix

* fix minor

* fix test

docs/utils.md

@@ -26,6 +26,39 @@ To load and use configs
 ...      d='string')
 ```
 
+For all format configs, some predefined variables are supported. It will convert the variable in `{{ var }}` with its real value.
+
+Currently, it supports four predefined variables:
+
+`{{ fileDirname }}` - the current opened file's dirname, e.g. /home/your-username/your-project/folder
+
+`{{ fileBasename }}` - the current opened file's basename, e.g. file.ext
+
+`{{ fileBasenameNoExtension }}` - the current opened file's basename with no file extension, e.g. file
+
+`{{ fileExtname }}` - the current opened file's extension, e.g. .ext
+
+These variable names are referred from https://code.visualstudio.com/docs/editor/variables-reference.
+
+Here is one examples of config with predefined variables.
+
+`config_a.py`
+
+```python
+a = 1
+b = './work_dir/{{ fileBasenameNoExtension }}'
+c = '{{ fileExtname }}'
+```
+
+```python
+>>> cfg = Config.fromfile('./config_a.py')
+>>> print(cfg)
+>>> dict(a=1,
+...      b='./work_dir/config_a',
+...      c='.py')
+```
+
+
 For all format configs, inheritance is supported. To reuse fields in other config files,
 specify `_base_='./config_a.py'` or a list of configs `_base_=['./config_a.py', './config_b.py']`.
 Here are 4 examples of config inheritance.

mmcv/utils/config.py

 # Copyright (c) Open-MMLab. All rights reserved.
 import ast
 import os.path as osp
+import re
 import shutil
 import sys
 import tempfile
@@ -91,16 +92,43 @@ class Config:
                               f'file {filename}')
 
     @staticmethod
-    def _file2dict(filename):
+    def _substitute_predefined_vars(filename, temp_config_name):
+        file_dirname = osp.dirname(filename)
+        file_basename = osp.basename(filename)
+        file_basename_no_extension = osp.splitext(file_basename)[0]
+        file_extname = osp.splitext(filename)[1]
+        support_templates = dict(
+            fileDirname=file_dirname,
+            fileBasename=file_basename,
+            fileBasenameNoExtension=file_basename_no_extension,
+            fileExtname=file_extname)
+        config_file = open(filename).read()
+        for key, value in support_templates.items():
+            regexp = r'\{\{\s*' + str(key) + r'\s*\}\}'
+            config_file = re.sub(regexp, value, config_file)
+        with open(temp_config_name, 'w') as tmp_config_file:
+            tmp_config_file.write(config_file)
+
+    @staticmethod
+    def _file2dict(filename, use_predefined_variables=True):
         filename = osp.abspath(osp.expanduser(filename))
         check_file_exist(filename)
-        if filename.endswith('.py'):
-            with tempfile.TemporaryDirectory() as temp_config_dir:
-                temp_config_file = tempfile.NamedTemporaryFile(
-                    dir=temp_config_dir, suffix='.py')
-                temp_config_name = osp.basename(temp_config_file.name)
-                shutil.copyfile(filename,
-                                osp.join(temp_config_dir, temp_config_name))
+        fileExtname = osp.splitext(filename)[1]
+        if fileExtname not in ['.py', '.json', '.yaml', 'yml']:
+            raise IOError('Only py/yml/yaml/json type are supported now!')
+
+        with tempfile.TemporaryDirectory() as temp_config_dir:
+            temp_config_file = tempfile.NamedTemporaryFile(
+                dir=temp_config_dir, suffix=fileExtname)
+            temp_config_name = osp.basename(temp_config_file.name)
+            # Substitute predefined variables
+            if use_predefined_variables:
+                Config._substitute_predefined_vars(filename,
+                                                   temp_config_file.name)
+            else:
+                shutil.copyfile(filename, temp_config_file.name)
+
+            if filename.endswith('.py'):
                 temp_module_name = osp.splitext(temp_config_name)[0]
                 sys.path.insert(0, temp_config_dir)
                 Config._validate_py_syntax(filename)
@@ -113,13 +141,11 @@ class Config:
                 }
                 # delete imported module
                 del sys.modules[temp_module_name]
-                # close temp file
-                temp_config_file.close()
-        elif filename.endswith(('.yml', '.yaml', '.json')):
-            import mmcv
-            cfg_dict = mmcv.load(filename)
-        else:
-            raise IOError('Only py/yml/yaml/json type are supported now!')
+            elif filename.endswith(('.yml', '.yaml', '.json')):
+                import mmcv
+                cfg_dict = mmcv.load(temp_config_file.name)
+            # close temp file
+            temp_config_file.close()
 
         cfg_text = filename + '\n'
         with open(filename, 'r') as f:
@@ -173,8 +199,9 @@ class Config:
         return b
 
     @staticmethod
-    def fromfile(filename):
-        cfg_dict, cfg_text = Config._file2dict(filename)
+    def fromfile(filename, use_predefined_variables=True):
+        cfg_dict, cfg_text = Config._file2dict(filename,
+                                               use_predefined_variables)
         return Config(cfg_dict, cfg_text=cfg_text, filename=filename)
 
     @staticmethod

tests/data/config/h.py

+item1 = '{{fileBasename}}'
+item2 = '{{ fileDirname}}'
+item3 = 'abc_{{ fileBasenameNoExtension }}'

tests/data/config/o.json

+{
+    "item1": "{{ fileDirname }}"
+}

tests/data/config/p.yaml

+item1: '{{ fileDirname }}'

tests/test_config.py

@@ -60,6 +60,74 @@ def test_construct():
         assert cfg.dump() == open(dump_file, 'r').read()
         assert Config.fromfile(dump_file)
 
+    # test h.py
+    cfg_file = osp.join(osp.dirname(__file__), 'data/config/h.py')
+    cfg_dict = dict(
+        item1='h.py',
+        item2=f'{osp.dirname(__file__)}/data/config',
+        item3='abc_h')
+    cfg = Config(cfg_dict, filename=cfg_file)
+    assert isinstance(cfg, Config)
+    assert cfg.filename == cfg_file
+    assert cfg.text == open(cfg_file, 'r').read()
+    assert cfg.dump() == cfg.pretty_text
+    with tempfile.TemporaryDirectory() as temp_config_dir:
+        dump_file = osp.join(temp_config_dir, 'h.py')
+        cfg.dump(dump_file)
+        assert cfg.dump() == open(dump_file, 'r').read()
+        assert Config.fromfile(dump_file)
+        assert Config.fromfile(dump_file)['item1'] == cfg_dict['item1']
+        assert Config.fromfile(dump_file)['item2'] == cfg_dict['item2']
+        assert Config.fromfile(dump_file)['item3'] == cfg_dict['item3']
+
+    # test no use_predefined_variable
+    cfg_dict = dict(
+        item1='{{fileBasename}}',
+        item2='{{ fileDirname}}',
+        item3='abc_{{ fileBasenameNoExtension }}')
+    assert Config.fromfile(cfg_file, False)
+    assert Config.fromfile(cfg_file, False)['item1'] == cfg_dict['item1']
+    assert Config.fromfile(cfg_file, False)['item2'] == cfg_dict['item2']
+    assert Config.fromfile(cfg_file, False)['item3'] == cfg_dict['item3']
+
+    # test p.yaml
+    cfg_file = osp.join(osp.dirname(__file__), 'data/config/p.yaml')
+    cfg_dict = dict(item1=f'{osp.dirname(__file__)}/data/config')
+    cfg = Config(cfg_dict, filename=cfg_file)
+    assert isinstance(cfg, Config)
+    assert cfg.filename == cfg_file
+    assert cfg.text == open(cfg_file, 'r').read()
+    assert cfg.dump() == yaml.dump(cfg_dict)
+    with tempfile.TemporaryDirectory() as temp_config_dir:
+        dump_file = osp.join(temp_config_dir, 'p.yaml')
+        cfg.dump(dump_file)
+        assert cfg.dump() == open(dump_file, 'r').read()
+        assert Config.fromfile(dump_file)
+        assert Config.fromfile(dump_file)['item1'] == cfg_dict['item1']
+
+    # test no use_predefined_variable
+    assert Config.fromfile(cfg_file, False)
+    assert Config.fromfile(cfg_file, False)['item1'] == '{{ fileDirname }}'
+
+    # test o.json
+    cfg_file = osp.join(osp.dirname(__file__), 'data/config/o.json')
+    cfg_dict = dict(item1=f'{osp.dirname(__file__)}/data/config')
+    cfg = Config(cfg_dict, filename=cfg_file)
+    assert isinstance(cfg, Config)
+    assert cfg.filename == cfg_file
+    assert cfg.text == open(cfg_file, 'r').read()
+    assert cfg.dump() == json.dumps(cfg_dict)
+    with tempfile.TemporaryDirectory() as temp_config_dir:
+        dump_file = osp.join(temp_config_dir, 'o.json')
+        cfg.dump(dump_file)
+        assert cfg.dump() == open(dump_file, 'r').read()
+        assert Config.fromfile(dump_file)
+        assert Config.fromfile(dump_file)['item1'] == cfg_dict['item1']
+
+    # test no use_predefined_variable
+    assert Config.fromfile(cfg_file, False)
+    assert Config.fromfile(cfg_file, False)['item1'] == '{{ fileDirname }}'
+
 
 def test_fromfile():
     for filename in ['a.py', 'a.b.py', 'b.json', 'c.yaml']: