[Feature] Add windows CI (#1023)

* add windows CI

* clean versions

* only allow pt1.7 on windows

* fix windows install issue

* add win cpu

* fix win command

* clean unnecessary command

* resolve turbojpeg & tempfile on win

* replace os.readlink with os.path.realpath

* fix windows ci

* close file before removing it

* fix windows ci

* fix symlink on windows

* fix windows ci

* fix windows ci

* fix windows ci

* fix windows ci

* fix windows ci

* fix windows ci

* fix windows ci

* fix windows ci

* fix windows ci

* modify according to comment
Co-authored-by: zhouzaida <zhouzaida@163.com>

.github/workflows/build.yml

@@ -287,6 +287,72 @@ jobs:
           name: codecov-umbrella
           fail_ci_if_error: false
 
+  build_windows_without_ops:
+    runs-on: windows-latest
+    env:
+      MMCV_WITH_OPS: 0
+    strategy:
+      matrix:
+        torch: [1.7.1, 1.8.0, 1.9.0]
+        include:
+          - torch: 1.7.1
+            torchvision: 0.8.2
+          - torch: 1.8.0
+            torchvision: 0.9.0
+          - torch: 1.9.0
+            torchvision: 0.10.0
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+      - name: Install Pillow
+        run: pip install Pillow==6.2.2
+        if: ${{matrix.torchvision == '0.4.2'}}
+      - name: Install PyTorch
+        run: pip install torch==${{matrix.torch}}+cpu torchvision==${{matrix.torchvision}}+cpu --no-cache-dir -f https://download.pytorch.org/whl/torch_stable.html
+      - name: Build and install
+        run: pip install -e .
+      - name: Validate the installation
+        run: python -c "import mmcv"
+      - name: Run unittests
+        run: |
+          pip install -r requirements/test.txt
+          pytest tests/ --ignore=tests/test_ops --ignore tests/test_utils/test_progressbar.py --ignore tests/test_utils/test_timer.py --ignore tests/test_image/test_io.py
+
+  build_windows:
+    runs-on: windows-latest
+    strategy:
+      matrix:
+        torch: [1.7.1, 1.8.0, 1.9.0]
+        include:
+          - torch: 1.7.1
+            torchvision: 0.8.2
+          - torch: 1.8.0
+            torchvision: 0.9.0
+          - torch: 1.9.0
+            torchvision: 0.10.0
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+      - name: Install Pillow
+        run: pip install Pillow==6.2.2
+        if: ${{matrix.torchvision == '0.4.2'}}
+      - name: Install PyTorch
+        run: pip install torch==${{matrix.torch}}+cpu torchvision==${{matrix.torchvision}}+cpu --no-cache-dir -f https://download.pytorch.org/whl/torch_stable.html
+      - name: Build and install
+        run: pip install -e .
+      - name: Validate the installation
+        run: python -c "import mmcv"
+      - name: Run unittests
+        run: |
+          pip install -r requirements/test.txt
+          pytest tests/ --ignore tests/test_utils/test_progressbar.py --ignore tests/test_utils/test_timer.py --ignore tests/test_image/test_io.py
+
   build_macos:
     runs-on: macos-latest
     strategy:

mmcv/runner/hooks/logger/pavi.py

@@ -89,8 +89,10 @@ class PaviLoggerHook(LoggerHook):
     def after_run(self, runner):
         if self.add_last_ckpt:
             ckpt_path = osp.join(runner.work_dir, 'latest.pth')
-            if osp.isfile(ckpt_path):
+            if osp.islink(ckpt_path):
                 ckpt_path = osp.join(runner.work_dir, os.readlink(ckpt_path))
+
+            if osp.isfile(ckpt_path):
                 # runner.epoch += 1 has been done before `after_run`.
                 iteration = runner.epoch if self.by_epoch else runner.iter
                 return self.writer.add_snapshot_file(

mmcv/video/processing.py

@@ -141,7 +141,7 @@ def concat_video(video_list,
         log_level (str): Logging level of ffmpeg.
         print_cmd (bool): Whether to print the final ffmpeg command.
     """
-    _, tmp_filename = tempfile.mkstemp(suffix='.txt', text=True)
+    tmp_filehandler, tmp_filename = tempfile.mkstemp(suffix='.txt', text=True)
     with open(tmp_filename, 'w') as f:
         for filename in video_list:
             f.write(f'file {osp.abspath(filename)}\n')
@@ -156,4 +156,5 @@ def concat_video(video_list,
         print_cmd,
         pre_options='-f concat -safe 0',
         **options)
+    os.close(tmp_filehandler)
     os.remove(tmp_filename)

tests/test_runner/test_hooks.py

@@ -6,6 +6,7 @@ CommandLine:
 """
 import logging
 import os.path as osp
+import platform
 import random
 import re
 import shutil
@@ -212,9 +213,14 @@ def test_pavi_hook():
         'learning_rate': 0.02,
         'momentum': 0.95
     }, 1)
+    # in windows environment, the latest checkpoint is copied from epoch_1.pth
+    if platform.system() == 'Windows':
+        snapshot_file_path = osp.join(runner.work_dir, 'latest.pth')
+    else:
+        snapshot_file_path = osp.join(runner.work_dir, 'epoch_1.pth')
     hook.writer.add_snapshot_file.assert_called_with(
         tag=runner.work_dir.split('/')[-1],
-        snapshot_file_path=osp.join(runner.work_dir, 'epoch_1.pth'),
+        snapshot_file_path=snapshot_file_path,
         iteration=1)
 
 

tests/test_runner/test_runner.py

@@ -2,6 +2,7 @@
 import logging
 import os
 import os.path as osp
+import platform
 import random
 import string
 import tempfile
@@ -169,7 +170,12 @@ def test_save_checkpoint(runner_class):
             first_ckp_path = osp.join(root, 'iter_1.pth')
 
         assert osp.exists(first_ckp_path)
+
+        if platform.system() != 'Windows':
             assert osp.realpath(latest_path) == osp.realpath(first_ckp_path)
+        else:
+            # use copy instead of symlink on windows
+            pass
 
         torch.load(latest_path)
 

tests/test_utils/test_config.py

@@ -5,6 +5,7 @@ import os
 import os.path as osp
 import shutil
 import tempfile
+from pathlib import Path
 
 import pytest
 import yaml
@@ -66,10 +67,14 @@ def test_construct():
 
     # test h.py
     cfg_file = osp.join(data_path, 'config/h.py')
-    cfg_dict = dict(
-        item1='h.py',
-        item2=f'{osp.dirname(__file__)}/data/config',
-        item3='abc_h')
+    path = osp.join(osp.dirname(__file__), 'data', 'config')
+    # the value of osp.dirname(__file__) may be `D:\a\xxx` in windows
+    # environment. When dumping the cfg_dict to file, `D:\a\xxx` will be
+    # converted to `D:\x07\xxx` and it will cause unexpected result when
+    # checking whether `D:\a\xxx` equals to `D:\x07\xxx`. Therefore, we forcely
+    # convert a string representation of the path with forward slashes (/)
+    path = Path(path).as_posix()
+    cfg_dict = dict(item1='h.py', item2=path, item3='abc_h')
     cfg = Config(cfg_dict, filename=cfg_file)
     assert isinstance(cfg, Config)
     assert cfg.filename == cfg_file
@@ -96,7 +101,7 @@ def test_construct():
 
     # test p.yaml
     cfg_file = osp.join(data_path, 'config/p.yaml')
-    cfg_dict = dict(item1=f'{osp.dirname(__file__)}/data/config')
+    cfg_dict = dict(item1=osp.join(osp.dirname(__file__), 'data', 'config'))
     cfg = Config(cfg_dict, filename=cfg_file)
     assert isinstance(cfg, Config)
     assert cfg.filename == cfg_file
@@ -115,7 +120,7 @@ def test_construct():
 
     # test o.json
     cfg_file = osp.join(data_path, 'config/o.json')
-    cfg_dict = dict(item1=f'{osp.dirname(__file__)}/data/config')
+    cfg_dict = dict(item1=osp.join(osp.dirname(__file__), 'data', 'config'))
     cfg = Config(cfg_dict, filename=cfg_file)
     assert isinstance(cfg, Config)
     assert cfg.filename == cfg_file
@@ -490,17 +495,19 @@ def test_reserved_key():
 
 
 def test_syntax_error():
-    temp_cfg_file = tempfile.NamedTemporaryFile(suffix='.py')
+    # the name can not be used to open the file a second time in windows,
+    # so `delete` should be set as `False` and we need to manually remove it
+    # more details can be found at https://github.com/open-mmlab/mmcv/pull/1077
+    temp_cfg_file = tempfile.NamedTemporaryFile(suffix='.py', delete=False)
     temp_cfg_path = temp_cfg_file.name
     # write a file with syntax error
     with open(temp_cfg_path, 'w') as f:
         f.write('a=0b=dict(c=1)')
     with pytest.raises(
-            SyntaxError,
-            match='There are syntax errors in config '
-            f'file {temp_cfg_path}'):
+            SyntaxError, match='There are syntax errors in config file'):
         Config.fromfile(temp_cfg_path)
     temp_cfg_file.close()
+    os.remove(temp_cfg_path)
 
 
 def test_pickle_support():

tests/test_utils/test_logging.py

 import logging
+import os
 import platform
 import tempfile
 from unittest.mock import patch
@@ -28,15 +29,21 @@ def test_get_logger_rank0():
     assert len(logger.handlers) == 1
     assert logger.handlers[0].level == logging.DEBUG
 
-    with tempfile.NamedTemporaryFile() as f:
+    # the name can not be used to open the file a second time in windows,
+    # so `delete` should be set as `False` and we need to manually remove it
+    # more details can be found at https://github.com/open-mmlab/mmcv/pull/1077
+    with tempfile.NamedTemporaryFile(delete=False) as f:
         logger = get_logger('rank0.pkg3', log_file=f.name)
         assert isinstance(logger, logging.Logger)
         assert len(logger.handlers) == 2
         assert isinstance(logger.handlers[0], logging.StreamHandler)
         assert isinstance(logger.handlers[1], logging.FileHandler)
-
         logger_pkg3 = get_logger('rank0.pkg3')
         assert id(logger_pkg3) == id(logger)
+        # flushing and closing all handlers in order to remove `f.name`
+        logging.shutdown()
+
+    os.remove(f.name)
 
     logger_pkg3 = get_logger('rank0.pkg3.subpkg')
     assert logger_pkg3.handlers == logger_pkg3.handlers
@@ -52,11 +59,18 @@ def test_get_logger_rank1():
     assert isinstance(logger.handlers[0], logging.StreamHandler)
     assert logger.handlers[0].level == logging.INFO
 
-    with tempfile.NamedTemporaryFile() as f:
+    # the name can not be used to open the file a second time in windows,
+    # so `delete` should be set as `False` and we need to manually remove it
+    # more details can be found at https://github.com/open-mmlab/mmcv/pull/1077
+    with tempfile.NamedTemporaryFile(delete=False) as f:
         logger = get_logger('rank1.pkg2', log_file=f.name)
         assert isinstance(logger, logging.Logger)
         assert len(logger.handlers) == 1
         assert logger.handlers[0].level == logging.INFO
+        # flushing and closing all handlers in order to remove `f.name`
+        logging.shutdown()
+
+    os.remove(f.name)
 
 
 def test_print_log_print(capsys):
@@ -79,7 +93,10 @@ def test_print_log_logger(caplog):
     print_log('welcome', logger='mmcv', level=logging.ERROR)
     assert caplog.record_tuples[-1] == ('mmcv', logging.ERROR, 'welcome')
 
-    with tempfile.NamedTemporaryFile() as f:
+    # the name can not be used to open the file a second time in windows,
+    # so `delete` should be set as `False` and we need to manually remove it
+    # more details can be found at https://github.com/open-mmlab/mmcv/pull/1077
+    with tempfile.NamedTemporaryFile(delete=False) as f:
         logger = get_logger('abc', log_file=f.name)
         print_log('welcome', logger=logger)
         assert caplog.record_tuples[-1] == ('abc', logging.INFO, 'welcome')
@@ -89,6 +106,10 @@ def test_print_log_logger(caplog):
             match = re.fullmatch(regex_time + r' - abc - INFO - welcome\n',
                                  log_text)
             assert match is not None
+        # flushing and closing all handlers in order to remove `f.name`
+        logging.shutdown()
+
+    os.remove(f.name)
 
 
 def test_print_log_exception():

tests/test_utils/test_path.py

@@ -38,9 +38,12 @@ def test_scandir():
     ])
     assert set(mmcv.scandir(folder, '.png')) == set()
 
+    # path of sep is `\\` in windows but `/` in linux, so osp.join should be
+    # used to join string for compatibility
     filenames_recursive = [
-        'a.bin', '1.txt', '2.txt', '1.json', '2.json', 'sub/1.json',
-        'sub/1.txt', '.file'
+        'a.bin', '1.txt', '2.txt', '1.json', '2.json',
+        osp.join('sub', '1.json'),
+        osp.join('sub', '1.txt'), '.file'
     ]
     # .file starts with '.' and is a file so it will not be scanned
     assert set(mmcv.scandir(folder, recursive=True)) == set(

tests/test_video/test_optflow.py

@@ -51,10 +51,11 @@ def test_flowwrite():
     flow = np.random.rand(100, 100, 2).astype(np.float32)
 
     # write to a .flo file
-    _, filename = tempfile.mkstemp()
+    tmp_filehandler, filename = tempfile.mkstemp()
     mmcv.flowwrite(flow, filename)
     flow_from_file = mmcv.flowread(filename)
     assert_array_equal(flow, flow_from_file)
+    os.close(tmp_filehandler)
     os.remove(filename)
 
     # write to two .jpg files

tests/test_video/test_processing.py

 # Copyright (c) OpenMMLab. All rights reserved.
 import os
 import os.path as osp
+import platform
 import tempfile
 
+import pytest
+
 import mmcv
 
 
@@ -13,6 +16,7 @@ class TestVideoEditor:
         cls.video_path = osp.join(osp.dirname(__file__), '../data/test.mp4')
         cls.num_frames = 168
 
+    @pytest.mark.skipif(platform.system() == 'Windows', reason='skip windows')
     def test_cut_concat_video(self):
         part1_file = osp.join(tempfile.gettempdir(), '.mmcv_test1.mp4')
         part2_file = osp.join(tempfile.gettempdir(), '.mmcv_test2.mp4')
@@ -31,6 +35,7 @@ class TestVideoEditor:
         os.remove(part2_file)
         os.remove(out_file)
 
+    @pytest.mark.skipif(platform.system() == 'Windows', reason='skip windows')
     def test_resize_video(self):
         out_file = osp.join(tempfile.gettempdir(), '.mmcv_test.mp4')
         mmcv.resize_video(

tests/test_video/test_reader.py

@@ -199,7 +199,8 @@ class TestVideoReader:
             start=10,
             end=50,
             show_progress=False)
-        v = mmcv.VideoReader(out_filename)
+
+        with mmcv.VideoReader(out_filename) as v:
             assert v.fps == 25
             assert len(v) == 40
 
@@ -207,4 +208,3 @@ class TestVideoReader:
                 filename = f'{frame_dir}/{i:06d}.jpg'
                 os.remove(filename)
             shutil.rmtree(frame_dir)
-        os.remove(out_filename)