result = self.execute_module(ome_default_args)
        assert result['msg'] == "Successfully uploaded application certificate."

# Dell EMC OpenManage Ansible Modules

from __future__ import (absolute_import, division, print_function)

__metaclass__ = type

import json

import pytest
from ansible.module_utils.six.moves.urllib.error import HTTPError, URLError
from ansible.module_utils.urls import ConnectionError, SSLValidationError
from io import StringIO
from ansible.module_utils._text import to_text
from ssl import SSLError
from ansible_collections.dellemc.openmanage.plugins.modules import ome_application_certificate
from ansible_collections.dellemc.openmanage.tests.unit.plugins.modules.common import FakeAnsibleModule, Constants

MODULE_PATH = 'ansible_collections.dellemc.openmanage.plugins.modules.'

@pytest.fixture
def ome_connection_mock_for_application_certificate(mocker, ome_response_mock):
    connection_class_mock = mocker.patch(
        MODULE_PATH + 'ome_application_certificate.RestOME')
    ome_connection_mock_obj = connection_class_mock.return_value.__enter__.return_value
    ome_connection_mock_obj.invoke_request.return_value = ome_response_mock
    return ome_connection_mock_obj

class TestOmeAppCSR(FakeAnsibleModule):
    module = ome_application_certificate

    @pytest.mark.parametrize("exc_type",
                             [ValueError, SSLError, TypeError, ConnectionError, HTTPError, URLError])
    def test_ome_application_certificate_main_error_cases(self, exc_type, mocker, ome_default_args,
                                                          ome_connection_mock_for_application_certificate,
                                                          ome_response_mock):
        json_str = to_text(json.dumps({"info": "error_details"}))
        args = {"command": "generate_csr", "distinguished_name": "hostname.com",
                "department_name": "Remote Access Group", "business_name": "Dell Inc.",
                "locality": "Round Rock", "country_state": "Texas", "country": "US",
                "email": "example@nnthink.com"}
        ome_default_args.update(args)
        if exc_type == URLError:
            mocker.patch(MODULE_PATH + 'ome_application_certificate.get_resource_parameters',
                         side_effect=exc_type("TEST"))
            result = self._run_module(ome_default_args)
            assert result["unreachable"] is True
        elif exc_type not in [HTTPError, SSLValidationError]:
            mocker.patch(MODULE_PATH + 'ome_application_certificate.get_resource_parameters',
                         side_effect=exc_type("exception message"))
            result = self._run_module_with_fail_json(ome_default_args)
            assert result['failed'] is True
        else:
            mocker.patch(MODULE_PATH + 'ome_application_certificate.get_resource_parameters',
                         side_effect=exc_type('http://testhost.com', 400,
                                              'http error message',
                                              {"accept-type": "application/json"},
                                              StringIO(json_str)))
            result = self._run_module_with_fail_json(ome_default_args)
            assert result['failed'] is True
        assert 'csr_status' not in result
        assert 'msg' in result

    def test_get_resource_parameters_generate(self, mocker, ome_default_args,
                                              ome_connection_mock_for_application_certificate,
                                              ome_response_mock):
        args = {"command": "generate_csr", "distinguished_name": "hostname.com",
                "department_name": "Remote Access Group", "business_name": "Dell Inc.",
                "locality": "Round Rock", "country_state": "Texas", "country": "US",
                "email": "support@dell.com"}
        f_module = self.get_module_mock(params=args)
        result = self.module.get_resource_parameters(f_module)
        assert result[0] == "POST"
        assert result[1] == "ApplicationService/Actions/ApplicationService.GenerateCSR"
        assert result[2] == {'DistinguishedName': 'hostname.com', 'Locality': 'Round Rock',
                             'DepartmentName': 'Remote Access Group', 'BusinessName': 'Dell Inc.',
                             'State': 'Texas', 'Country': 'US', 'Email': 'support@dell.com'}

    def test_upload_csr_fail01(self, mocker, ome_default_args, ome_connection_mock_for_application_certificate,
                               ome_response_mock):
        args = {"command": "upload", "upload_file": "/path/certificate.cer"}
        f_module = self.get_module_mock(params=args)
        with pytest.raises(Exception) as exc:
            self.module.get_resource_parameters(f_module)
        assert exc.value.args[0] == "No such file or directory."

    def test_upload_csr_success(self, mocker, ome_default_args, ome_connection_mock_for_application_certificate,
                                ome_response_mock):
        payload = "--BEGIN-REQUEST--"
        mocker.patch(MODULE_PATH + 'ome_application_certificate.get_resource_parameters',
                     return_value=("POST", "ApplicationService/Actions/ApplicationService.UploadCertificate", payload))
        ome_default_args.update({"command": "upload", "upload_file": "/path/certificate.cer"})
        ome_response_mock.success = True

    def test_generate_csr(self, mocker, ome_default_args, ome_connection_mock_for_application_certificate,
                          ome_response_mock):
        csr_json = {"CertificateData": "--BEGIN-REQUEST--"}
        payload = {"DistinguishedName": "hostname.com", "DepartmentName": "Remote Access Group",
                   "BusinessName": "Dell Inc.", "Locality": "Round Rock", "State": "Texas",
                   "Country": "US", "Email": "support@dell.com"}
        mocker.patch(MODULE_PATH + 'ome_application_certificate.get_resource_parameters',
                     return_value=("POST", "ApplicationService/Actions/ApplicationService.GenerateCSR", payload))
        ome_default_args.update({"command": "generate_csr", "distinguished_name": "hostname.com",
                                 "department_name": "Remote Access Group", "business_name": "Dell Inc.",
                                 "locality": "Round Rock", "country_state": "Texas", "country": "US",
                                 "email": "support@dell.com"})
        ome_response_mock.success = True
        ome_response_mock.json_data = csr_json
        result = self.execute_module(ome_default_args)
        assert result['msg'] == "Successfully generated certificate signing request."
        assert result['csr_status'] == {'CertificateData': '--BEGIN-REQUEST--'}


# Dell EMC OpenManage Ansible Modules

from __future__ import (absolute_import, division, print_function)

__metaclass__ = type

import json

import pytest
from ansible.module_utils.six.moves.urllib.error import HTTPError, URLError
from ansible.module_utils.urls import ConnectionError, SSLValidationError
from io import StringIO
from ansible.module_utils._text import to_text
from ssl import SSLError
from ansible_collections.dellemc.openmanage.plugins.modules import ome_application_certificate
from ansible_collections.dellemc.openmanage.tests.unit.plugins.modules.common import FakeAnsibleModule, Constants

MODULE_PATH = 'ansible_collections.dellemc.openmanage.plugins.modules.'

@pytest.fixture
def ome_connection_mock_for_application_certificate(mocker, ome_response_mock):
    connection_class_mock = mocker.patch(
        MODULE_PATH + 'ome_application_certificate.RestOME')
    ome_connection_mock_obj = connection_class_mock.return_value.__enter__.return_value
    ome_connection_mock_obj.invoke_request.return_value = ome_response_mock
    return ome_connection_mock_obj

class TestOmeAppCSR(FakeAnsibleModule):
    module = ome_application_certificate

    @pytest.mark.parametrize("exc_type",
                             [ValueError, SSLError, TypeError, ConnectionError, HTTPError, URLError])
    def test_ome_application_certificate_main_error_cases(self, exc_type, mocker, ome_default_args,
                                                          ome_connection_mock_for_application_certificate,
                                                          ome_response_mock):
        json_str = to_text(json.dumps({"info": "error_details"}))
        args = {"command": "generate_csr", "distinguished_name": "hostname.com",
                "department_name": "Remote Access Group", "business_name": "Dell Inc.",
                "locality": "Round Rock", "country_state": "Texas", "country": "US",
                "email": "example@nnthink.com"}
        ome_default_args.update(args)
        if exc_type == URLError:
            mocker.patch(MODULE_PATH + 'ome_application_certificate.get_resource_parameters',
                         side_effect=exc_type("TEST"))
            result = self._run_module(ome_default_args)
            assert result["unreachable"] is True
        elif exc_type not in [HTTPError, SSLValidationError]:
            mocker.patch(MODULE_PATH + 'ome_application_certificate.get_resource_parameters',
                         side_effect=exc_type("exception message"))
            result = self._run_module_with_fail_json(ome_default_args)
            assert result['failed'] is True
        else:
            mocker.patch(MODULE_PATH + 'ome_application_certificate.get_resource_parameters',
                         side_effect=exc_type('http://testhost.com', 400,
                                              'http error message',
                                              {"accept-type": "application/json"},
                                              StringIO(json_str)))
            result = self._run_module_with_fail_json(ome_default_args)
            assert result['failed'] is True
        assert 'csr_status' not in result
        assert 'msg' in result

    def test_get_resource_parameters_generate(self, mocker, ome_default_args,
                                              ome_connection_mock_for_application_certificate,
                                              ome_response_mock):
        args = {"command": "generate_csr", "distinguished_name": "hostname.com",
                "department_name": "Remote Access Group", "business_name": "Dell Inc.",
                "locality": "Round Rock", "country_state": "Texas", "country": "US",
                "email": "support@dell.com"}
        f_module = self.get_module_mock(params=args)
        result = self.module.get_resource_parameters(f_module)
        assert result[0] == "POST"
        assert result[1] == "ApplicationService/Actions/ApplicationService.GenerateCSR"
        assert result[2] == {'DistinguishedName': 'hostname.com', 'Locality': 'Round Rock',
                             'DepartmentName': 'Remote Access Group', 'BusinessName': 'Dell Inc.',
                             'State': 'Texas', 'Country': 'US', 'Email': 'support@dell.com'}

    def test_upload_csr_fail01(self, mocker, ome_default_args, ome_connection_mock_for_application_certificate,
                               ome_response_mock):
        args = {"command": "upload", "upload_file": "/path/certificate.cer"}
        f_module = self.get_module_mock(params=args)
        with pytest.raises(Exception) as exc:
            self.module.get_resource_parameters(f_module)
        assert exc.value.args[0] == "No such

    def test_upload_csr_success(self, mocker, ome_default_args, ome_connection_mock_for_application_certificate,
                                ome_response_mock):
        payload = "--BEGIN-REQUEST--"
        mocker.patch(MODULE_PATH + 'ome_application_certificate.get_resource_parameters',
                     return_value=("POST", "ApplicationService/Actions/ApplicationService.UploadCertificate", payload))
        ome_default_args.update({"command": "upload", "upload_file": "/path/certificate.cer"})
        ome_response_mock.success = True
        result = self.execute_module(ome_default_args)
        assert result['msg'] == "Successfully uploaded application certificate."

    def test_generate_csr(self, mocker, ome_default_args, ome_connection_mock_for_application_certificate,
                          ome_response_mock):
        csr_json = {"CertificateData": "--BEGIN-REQUEST--"}
        payload = {"DistinguishedName": "hostname.com", "DepartmentName": "Remote Access Group",
                   "BusinessName": "Dell Inc.", "Locality": "Round Rock", "State": "Texas",
                   "Country": "US", "Email": "support@dell.com"}
        mocker.patch(MODULE_PATH + 'ome_application_certificate.get_resource_parameters',
                     return_value=("POST", "ApplicationService/Actions/ApplicationService.GenerateCSR", payload))
        ome_default_args.update({"command": "generate_csr", "distinguished_name": "hostname.com",
                                 "department_name": "Remote Access Group", "business_name": "Dell Inc.",
                                 "locality": "Round Rock", "country_state": "Texas", "country": "US",
                                 "email": "support@dell.com"})
        ome_response_mock.success = True
        ome_response_mock.json_data = csr_json
        result = self.execute_module(ome_default_args)
        assert result['msg'] == "Successfully generated certificate signing request."
        assert result['csr_status'] == {'CertificateData': '--BEGIN-REQUEST--'}


"""
Tests for splitter objects.
"""
import unittest
import numpy as np
import deepchem as dc

class TestTaskSplitters(unittest.TestCase):
  """
  Test some basic splitters.
  """

  def test_multitask_train_valid_test_split(self):
    """
    Test TaskSplitter train/valid/test split on multitask dataset.
    """
    n_samples = 100
    n_features = 10
    n_tasks = 10
    X = np.random.rand(n_samples, n_features)
    p = .05  # proportion actives
    y = np.random.binomial(1, p, size=(n_samples, n_tasks))
    dataset = dc.data.NumpyDataset(X, y)

    task_splitter = dc.splits.TaskSplitter()
    train, valid, test = task_splitter.train_valid_test_split(
        dataset, frac_train=.4, frac_valid=.3, frac_test=.3)

    assert len(train.get_task_names()) == 4
    assert len(valid.get_task_names()) == 3
    assert len(test.get_task_names()) == 3

  def test_multitask_K_fold_split(self):
    """
    Test TaskSplitter K-fold split on multitask dataset.
    """
    n_samples = 100
    n_features = 10
    n_tasks = 10
    X = np.random.rand(n_samples, n_features)
    p = .05  # proportion actives
    y = np.random.binomial(1, p, size=(n_samples, n_tasks))
    dataset = dc.data.NumpyDataset(X, y)
    K = 5

    task_splitter = dc.splits.TaskSplitter()
    fold_datasets = task_splitter.k_fold_split(dataset, K)

    for fold_dataset in fold_datasets:
      assert len(fold_dataset.get_task_names()) == 2

  def test_uneven_k_fold_split(self):
    """
    Test k-fold-split works when K does not divide n_tasks.
    """
    n_samples = 100
    n_features = 10
    n_tasks = 17
    X = np.random.rand(n_samples, n_features)
    p = .05  # proportion actives
    y = np.random.binomial(1, p, size=(n_samples, n_tasks))
    dataset = dc.data.NumpyDataset(X, y)
    K = 4
    task_splitter = dc.splits.TaskSplitter()
    fold_datasets = task_splitter.k_fold_split(dataset, K)

    for fold in range(K - 1):
      fold_dataset = fold_datasets[fold]
      assert len(fold_dataset.get_task_names()) == 4
    assert len(fold_datasets[-1].get_task_names()) == 5

"""
    Test train/valid/test split works when proportions don't divide n_tasks.
    """
    n_samples = 100
    n_features = 10
    n_tasks = 11
    X = np.random.rand(n_samples, n_features)
    p = .05  # proportion actives
    y = np.random.binomial(1, p, size=(n_samples, n_tasks))
    dataset = dc.data.NumpyDataset(X, y)

    task_splitter = dc.splits.TaskSplitter()
    train, valid, test = task_splitter.train_valid_test_split(
        dataset, frac_train=.4, frac_valid=.3, frac_test=.3)

    assert len(train.get_task_names()) == 4
    assert len(valid.get_task_names()) == 3
    # Note that the extra task goes to test
    assert len(test.get_task_names()) == 4

  def test_merge_fold_datasets(self):
    """
    Test that (K-1) folds can be merged into train dataset.
    """
    n_samples = 100
    n_features = 10
    n_tasks = 10
    X = np.random.rand(n_samples, n_features)
    p = .05  # proportion actives
    y = np.random.binomial(1, p, size=(n_samples, n_tasks))
    w = np.ones((n_samples, n_tasks))
    dataset = dc.data.NumpyDataset(X, y, w)
    K = 5

    task_splitter = dc.splits.TaskSplitter()
    fold_datasets = task_splitter.k_fold_split(dataset, K)
    # Number tasks per fold
    n_per_fold = 2

    for fold in range(K):
      train_inds = list(set(range(K)) - set([fold]))
      train_fold_datasets = [fold_datasets[ind] for ind in train_inds]
      train_dataset = dc.splits.merge_fold_datasets(train_fold_datasets)

      # Find the tasks that correspond to this test fold
      train_tasks = list(
          set(range(10)) -
          set(range(fold * n_per_fold, (fold + 1) * n_per_fold)))

      # Assert that all arrays look like they should
      np.testing.assert_array_equal(train_dataset.X, X)
      np.testing.assert_array_equal(train_dataset.y, y[:, train_tasks])
      np.testing.assert_array_equal(train_dataset.w, w[:, train_tasks])
      np.testing.assert_array_equal(train_dataset.X, X)


"""
    Test that (K-1) folds can be merged into train dataset.
    """
    n_samples = 100
    n_features = 10
    n_tasks = 10
    X = np.random.rand(n_samples, n_features)
    p = .05

"""
Tests for splitter objects.
"""
import unittest
import numpy as np
import deepchem as dc

class TestTaskSplitters(unittest.TestCase):
  """
  Test some basic splitters.
  """

  def test_multitask_train_valid_test_split(self):
    """
    Test TaskSplitter train/valid/test split on multitask dataset.
    """
    n_samples = 100
    n_features = 10
    n_tasks = 10
    X = np.random.rand(n_samples, n_features)
    p = .05  # proportion actives
    y = np.random.binomial(1, p, size=(n_samples, n_tasks))
    dataset = dc.data.NumpyDataset(X, y)

    task_splitter = dc.splits.TaskSplitter()
    train, valid, test = task_splitter.train_valid_test_split(
        dataset, frac_train=.4, frac_valid=.3, frac_test=.3)

    assert len(train.get_task_names()) == 4
    assert len(valid.get_task_names()) == 3
    assert len(test.get_task_names()) == 3

  def test_multitask_K_fold_split(self):
    """
    Test TaskSplitter K-fold split on multitask dataset.
    """
    n_samples = 100
    n_features = 10
    n_tasks = 10
    X = np.random.rand(n_samples, n_features)
    p = .05  # proportion actives
    y = np.random.binomial(1, p, size=(n_samples, n_tasks))
    dataset = dc.data.NumpyDataset(X, y)
    K = 5

    task_splitter = dc.splits.TaskSplitter()
    fold_datasets = task_splitter.k_fold_split(dataset, K)

    for fold_dataset in fold_datasets:
      assert len(fold_dataset.get_task_names()) == 2

  def test_uneven_k_fold_split(self):
    """
    Test k-fold-split works when K does not divide n_tasks.
    """
    n_samples = 100
    n_features = 10
    n_tasks = 17
    X = np.random.rand(n_samples, n_features)
    p = .05  # proportion actives
    y = np.random.binomial(1, p, size=(n_samples, n_tasks))
    dataset = dc.data.NumpyDataset(X, y)
    K = 4
    task_splitter = dc.splits.TaskSplitter()
    fold_datasets = task_splitter.k_fold_split(dataset, K)

    for fold in range(K - 1):
      fold_dataset = fold_datasets[fold]
      assert len(fold_dataset.get_task_names()) == 4
    assert len(fold_datasets[-1].get_task_names()) == 5

  def test_uneven_train_valid_test_split(self):
    """
    Test train/valid/test split works when proportions don't divide n_tasks.
    """
    n_samples = 100
    n_features = 10
    n_tasks = 11
    X = np.random.rand(n_samples, n_features)
    p = .05  # proportion actives
    y = np.random.binomial(1, p, size=(n_samples, n_tasks))
    dataset = dc.data.NumpyDataset(X, y)

    task_splitter = dc.splits.TaskSplitter()
    train, valid, test = task_splitter.train_valid_test_split(
        dataset, frac_train=.4, frac_valid=.3, frac_test=.3)

    assert len(train.get_task_names()) == 4
    assert len(valid.get_task_names()) == 3
    # Note that the extra task goes to test
    assert len(test.get_task_names()) == 4

  def test_merge_fold_datasets(self):

  # proportion actives
    y = np.random.binomial(1, p, size=(n_samples, n_tasks))
    w = np.ones((n_samples, n_tasks))
    dataset = dc.data.NumpyDataset(X, y, w)
    K = 5

    task_splitter = dc.splits.TaskSplitter()
    fold_datasets = task_splitter.k_fold_split(dataset, K)
    # Number tasks per fold
    n_per_fold = 2

    for fold in range(K):
      train_inds = list(set(range(K)) - set([fold]))
      train_fold_datasets = [fold_datasets[ind] for ind in train_inds]
      train_dataset = dc.splits.merge_fold_datasets(train_fold_datasets)

      # Find the tasks that correspond to this test fold
      train_tasks = list(
          set(range(10)) -
          set(range(fold * n_per_fold, (fold + 1) * n_per_fold)))

      # Assert that all arrays look like they should
      np.testing.assert_array_equal(train_dataset.X, X)
      np.testing.assert_array_equal(train_dataset.y, y[:, train_tasks])
      np.testing.assert_array_equal(train_dataset.w, w[:, train_tasks])
      np.testing.assert_array_equal(train_dataset.X, X)


# proportion actives
    y = np.random.binomial(1, p, size=(n_samples, n_tasks))
    w = np.ones((n_samples, n_tasks))
    dataset = dc.data.NumpyDataset(X, y, w)
    K = 5

"""
Tests for splitter objects.
"""
import unittest
import numpy as np
import deepchem as dc

class TestTaskSplitters(unittest.TestCase):
  """
  Test some basic splitters.
  """

  def test_multitask_train_valid_test_split(self):
    """
    Test TaskSplitter train/valid/test split on multitask dataset.
    """
    n_samples = 100
    n_features = 10
    n_tasks = 10
    X = np.random.rand(n_samples, n_features)
    p = .05  # proportion actives
    y = np.random.binomial(1, p, size=(n_samples, n_tasks))
    dataset = dc.data.NumpyDataset(X, y)

    task_splitter = dc.splits.TaskSplitter()
    train, valid, test = task_splitter.train_valid_test_split(
        dataset, frac_train=.4, frac_valid=.3, frac_test=.3)

    assert len(train.get_task_names()) == 4
    assert len(valid.get_task_names()) == 3
    assert len(test.get_task_names()) == 3

  def test_multitask_K_fold_split(self):
    """
    Test TaskSplitter K-fold split on multitask dataset.
    """
    n_samples = 100
    n_features = 10
    n_tasks = 10
    X = np.random.rand(n_samples, n_features)
    p = .05  # proportion actives
    y = np.random.binomial(1, p, size=(n_samples, n_tasks))
    dataset = dc.data.NumpyDataset(X, y)
    K = 5

    task_splitter = dc.splits.TaskSplitter()
    fold_datasets = task_splitter.k_fold_split(dataset, K)

    for fold_dataset in fold_datasets:
      assert len(fold_dataset.get_task_names()) == 2

  def test_uneven_k_fold_split(self):
    """
    Test k-fold-split works when K does not divide n_tasks.
    """
    n_samples = 100
    n_features = 10
    n_tasks = 17
    X = np.random.rand(n_samples, n_features)
    p = .05  # proportion actives
    y = np.random.binomial(1, p, size=(n_samples, n_tasks))
    dataset = dc.data.NumpyDataset(X, y)
    K = 4
    task_splitter = dc.splits.TaskSplitter()
    fold_datasets = task_splitter.k_fold_split(dataset, K)

    for fold in range(K - 1):
      fold_dataset = fold_datasets[fold]
      assert len(fold_dataset.get_task_names()) == 4
    assert len(fold_datasets[-1].get_task_names()) == 5

  def test_uneven_train_valid_test_split(self):
    """
    Test train/valid/test split works when proportions don't divide n_tasks.
    """
    n_samples = 100
    n_features = 10
    n_tasks = 11
    X = np.random.rand(n_samples, n_features)
    p = .05  # proportion actives
    y = np.random.binomial(1, p, size=(n_samples, n_tasks))
    dataset = dc.data.NumpyDataset(X, y)

    task_splitter = dc.splits.TaskSplitter()
    train, valid, test = task_splitter.train_valid_test_split(
        dataset, frac_train=.4, frac_valid=.3, frac_test=.3)

    assert len(train.get_task_names()) == 4
    assert len(valid.get_task_names()) == 3
    # Note that the extra task goes to test
    assert len(test.get_task_names()) == 4

  def test_merge_fold_datasets(self):
    """
    Test that (K-1) folds can be merged into train dataset.
    """
    n_samples = 100
    n_features = 10
    n_tasks = 10
    X = np.random.rand(n_samples, n_features)
    p = .05

    task_splitter = dc.splits.TaskSplitter()
    fold_datasets = task_splitter.k_fold_split(dataset, K)
    # Number tasks per fold
    n_per_fold = 2

    for fold in range(K):
      train_inds = list(set(range(K)) - set([fold]))
      train_fold_datasets = [fold_datasets[ind] for ind in train_inds]
      train_dataset = dc.splits.merge_fold_datasets(train_fold_datasets)

      # Find the tasks that correspond to this test fold
      train_tasks = list(
          set(range(10)) -
          set(range(fold * n_per_fold, (fold + 1) * n_per_fold)))

      # Assert that all arrays look like they should
      np.testing.assert_array_equal(train_dataset.X, X)
      np.testing.assert_array_equal(train_dataset.y, y[:, train_tasks])
      np.testing.assert_array_equal(train_dataset.w, w[:, train_tasks])
      np.testing.assert_array_equal(train_dataset.X, X)


task_splitter = dc.splits.TaskSplitter()
    fold_datasets = task_splitter.k_fold_split(dataset, K)
    # Number tasks per fold
    n_per_fold = 2

    for fold in range(K):
      train_inds = list(set(range(K)) - set([fold]))
      train_fold_datasets = [fold_datasets[ind] for ind in train_inds]
      train_dataset = dc.splits.merge_fold_datasets(train_fold_datasets)

      # Find the tasks that correspond to this test fold
      train_tasks = list(
          set(range(10)) -
          set(range(fold * n_per_fold, (fold + 1) * n_per_fold)))

      # Assert that all arrays look like they should
      np.testing.assert_array_equal(train_dataset.X, X)
      np.testing.assert_array_equal(train_dataset.y, y[:, train_tasks])
      np.testing.assert_array_equal(train_dataset.w, w[:, train_tasks])
      np.testing.assert_array_equal(train_dataset.X, X)

"""
Tests for splitter objects.
"""
import unittest
import numpy as np
import deepchem as dc

class TestTaskSplitters(unittest.TestCase):
  """
  Test some basic splitters.
  """

  def test_multitask_train_valid_test_split(self):
    """
    Test TaskSplitter train/valid/test split on multitask dataset.
    """
    n_samples = 100
    n_features = 10
    n_tasks = 10
    X = np.random.rand(n_samples, n_features)
    p = .05  # proportion actives
    y = np.random.binomial(1, p, size=(n_samples, n_tasks))
    dataset = dc.data.NumpyDataset(X, y)

    task_splitter = dc.splits.TaskSplitter()
    train, valid, test = task_splitter.train_valid_test_split(
        dataset, frac_train=.4, frac_valid=.3, frac_test=.3)

    assert len(train.get_task_names()) == 4
    assert len(valid.get_task_names()) == 3
    assert len(test.get_task_names()) == 3

  def test_multitask_K_fold_split(self):
    """
    Test TaskSplitter K-fold split on multitask dataset.
    """
    n_samples = 100
    n_features = 10
    n_tasks = 10
    X = np.random.rand(n_samples, n_features)
    p = .05  # proportion actives
    y = np.random.binomial(1, p, size=(n_samples, n_tasks))
    dataset = dc.data.NumpyDataset(X, y)
    K = 5

    task_splitter = dc.splits.TaskSplitter()
    fold_datasets = task_splitter.k_fold_split(dataset, K)

    for fold_dataset in fold_datasets:
      assert len(fold_dataset.get_task_names()) == 2

  def test_uneven_k_fold_split(self):
    """
    Test k-fold-split works when K does not divide n_tasks.
    """
    n_samples = 100
    n_features = 10
    n_tasks = 17
    X = np.random.rand(n_samples, n_features)
    p = .05  # proportion actives
    y = np.random.binomial(1, p, size=(n_samples, n_tasks))
    dataset = dc.data.NumpyDataset(X, y)
    K = 4
    task_splitter = dc.splits.TaskSplitter()
    fold_datasets = task_splitter.k_fold_split(dataset, K)

    for fold in range(K - 1):
      fold_dataset = fold_datasets[fold]
      assert len(fold_dataset.get_task_names()) == 4
    assert len(fold_datasets[-1].get_task_names()) == 5

  def test_uneven_train_valid_test_split(self):
    """
    Test train/valid/test split works when proportions don't divide n_tasks.
    """
    n_samples = 100
    n_features = 10
    n_tasks = 11
    X = np.random.rand(n_samples, n_features)
    p = .05  # proportion actives
    y = np.random.binomial(1, p, size=(n_samples, n_tasks))
    dataset = dc.data.NumpyDataset(X, y)

    task_splitter = dc.splits.TaskSplitter()
    train, valid, test = task_splitter.train_valid_test_split(
        dataset, frac_train=.4, frac_valid=.3, frac_test=.3)

    assert len(train.get_task_names()) == 4
    assert len(valid.get_task_names()) == 3
    # Note that the extra task goes to test
    assert len(test.get_task_names()) == 4

  def test_merge_fold_datasets(self):
    """
    Test that (K-1) folds can be merged into train dataset.
    """
    n_samples = 100
    n_features = 10
    n_tasks = 10
    X = np.random.rand(n_samples, n_features)
    p = .05  # proportion actives
    y = np.random.binomial(1, p, size=(n_samples, n_tasks))
    w = np.ones((n_samples, n_tasks))
    dataset = dc.data.NumpyDataset(X, y, w)
    K = 5


"""
Tests for splitter objects.
"""
import unittest
import numpy as np
import deepchem as dc

class TestTaskSplitters(unittest.TestCase):
  """
  Test some basic splitters.
  """

  def test_multitask_train_valid_test_split(self):
    """
    Test TaskSplitter train/valid/test split on multitask dataset.
    """
    n_samples = 100
    n_features = 10
    n_tasks = 10
    X = np.random.rand(n_samples, n_features)
    p = .05  # proportion actives
    y = np.random.binomial(1, p, size=(n_samples, n_tasks))
    dataset = dc.data.NumpyDataset(X, y)

    task_splitter = dc.splits.TaskSplitter()
    train, valid, test = task_splitter.train_valid_test_split(
        dataset, frac_train=.4, frac_valid=.3, frac_test=.3)

    assert len(train.get_task_names()) == 4
    assert len(valid.get_task_names()) == 3
    assert len(test.get_task_names()) == 3

  def test_multitask_K_fold_split(self):
    """
    Test TaskSplitter K-fold split on multitask dataset.
    """
    n_samples = 100
    n_features = 10
    n_tasks = 10
    X = np.random.rand(n_samples, n_features)
    p = .05  # proportion actives
    y = np.random.binomial(1, p, size=(n_samples, n_tasks))
    dataset = dc.data.NumpyDataset(X, y)
    K = 5

    task_splitter = dc.splits.TaskSplitter()
    fold_datasets = task_splitter.k_fold_split(dataset, K)

    for fold_dataset in fold_datasets:
      assert len(fold_dataset.get_task_names()) == 2

  def test_uneven_k_fold_split(self):
    """
    Test k-fold-split works when K does not divide n_tasks.
    """
    n_samples = 100
    n_features = 10
    n_tasks = 17
    X = np.random.rand(n_samples, n_features)
    p = .05  # proportion actives
    y = np.random.binomial(1, p, size=(n_samples, n_tasks))
    dataset = dc.data.NumpyDataset(X, y)
    K = 4
    task_splitter = dc.splits.TaskSplitter()
    fold_datasets = task_splitter.k_fold_split(dataset, K)

    for fold in range(K - 1):
      fold_dataset = fold_datasets[fold]
      assert len(fold_dataset.get_task_names()) == 4
    assert len(fold_datasets[-1].get_task_names()) == 5

  def test_uneven_train_valid_test_split(self):
    """
    Test train/valid/test split works when proportions don't divide n_tasks.
    """
    n_samples = 100
    n_features = 10
    n_tasks = 1

    X = np.random.rand(n_samples, n_features)
    p = .05  # proportion actives
    y = np.random.binomial(1, p, size=(n_samples, n_tasks))
    dataset = dc.data.NumpyDataset(X, y)

    task_splitter = dc.splits.TaskSplitter()
    train, valid, test = task_splitter.train_valid_test_split(
        dataset, frac_train=.4, frac_valid=.3, frac_test=.3)

    assert len(train.get_task_names()) == 4
    assert len(valid.get_task_names()) == 3
    # Note that the extra task goes to test
    assert len(test.get_task_names()) == 4

  def test_merge_fold_datasets(self):
    """
    Test that (K-1) folds can be merged into train dataset.
    """
    n_samples = 100
    n_features = 10
    n_tasks = 10
    X = np.random.rand(n_samples, n_features)
    p = .05  # proportion actives
    y = np.random.binomial(1, p, size=(n_samples, n_tasks))
    w = np.ones((n_samples, n_tasks))
    dataset = dc.data.NumpyDataset(X, y, w)
    K = 5

    task_splitter = dc.splits.TaskSplitter()
    fold_datasets = task_splitter.k_fold_split(dataset, K)
    # Number tasks per fold
    n_per_fold = 2

    for fold in range(K):
      train_inds = list(set(range(K)) - set([fold]))
      train_fold_datasets = [fold_datasets[ind] for ind in train_inds]
      train_dataset = dc.splits.merge_fold_datasets(train_fold_datasets)

      # Find the tasks that correspond to this test fold
      train_tasks = list(
          set(range(10)) -
          set(range(fold * n_per_fold, (fold + 1) * n_per_fold)))

      # Assert that all arrays look like they should
      np.testing.assert_array_equal(train_dataset.X, X)
      np.testing.assert_array_equal(train_dataset.y, y[:, train_tasks])
      np.testing.assert_array_equal(train_dataset.w, w[:, train_tasks])
      np.testing.assert_array_equal(train_dataset.X, X)


    n_samples = 100
    n_features = 10
    n_tasks = 10
    X = np.random.rand(n_samples, n_features)
    p = .05  # proportion actives
    y = np.random.binomial(1, p, size=(n_samples, n_tasks))
    dataset = dc.data.NumpyDataset(X, y)
    K = 5

    task_splitter = dc.splits.TaskSplitter()
    fold_datasets = task_splitter.k_fold_split(dataset, K)

    for fold_dataset in fold_datasets:
      assert len(fold_dataset.get_task_names()) == 2

"""
Tests for splitter objects.
"""
import unittest
import numpy as np
import deepchem as dc

class TestTaskSplitters(unittest.TestCase):
  """
  Test some basic splitters.
  """

  def test_multitask_train_valid_test_split(self):
    """
    Test TaskSplitter train/valid/test split on multitask dataset.
    """
    n_samples = 100
    n_features = 10
    n_tasks = 10
    X = np.random.rand(n_samples, n_features)
    p = .05  # proportion actives
    y = np.random.binomial(1, p, size=(n_samples, n_tasks))
    dataset = dc.data.NumpyDataset(X, y)

    task_splitter = dc.splits.TaskSplitter()
    train, valid, test = task_splitter.train_valid_test_split(
        dataset, frac_train=.4, frac_valid=.3, frac_test=.3)

    assert len(train.get_task_names()) == 4
    assert len(valid.get_task_names()) == 3
    assert len(test.get_task_names()) == 3

  def test_multitask_K_fold_split(self):
    """
    Test TaskSplitter K-fold split on multitask dataset.
    """

  def test_uneven_k_fold_split(self):
    """
    Test k-fold-split works when K does not divide n_tasks.
    """
    n_samples = 100
    n_features = 10
    n_tasks = 17
    X = np.random.rand(n_samples, n_features)
    p = .05  # proportion actives
    y = np.random.binomial(1, p, size=(n_samples, n_tasks))
    dataset = dc.data.NumpyDataset(X, y)
    K = 4
    task_splitter = dc.splits.TaskSplitter()
    fold_datasets = task_splitter.k_fold_split(dataset, K)

    for fold in range(K - 1):
      fold_dataset = fold_datasets[fold]
      assert len(fold_dataset.get_task_names()) == 4
    assert len(fold_datasets[-1].get_task_names()) == 5

  def test_uneven_train_valid_test_split(self):
    """
    Test train/valid/test split works when proportions don't divide n_tasks.
    """
    n_samples = 100
    n_features = 10
    n_tasks = 11
    X = np.random.rand(n_samples, n_features)
    p = .05  # proportion actives
    y = np.random.binomial(1, p, size=(n_samples, n_tasks))
    dataset = dc.data.NumpyDataset(X, y)

    task_splitter = dc.splits.TaskSplitter()
    train, valid, test = task_splitter.train_valid_test_split(
        dataset, frac_train=.4, frac_valid=.3, frac_test=.3)

    assert len(train.get_task_names()) == 4
    assert len(valid.get_task_names()) == 3
    # Note that the extra task goes to test
    assert len(test.get_task_names()) == 4

  def test_merge_fold_datasets(self):
    """
    Test that (K-1) folds can be merged into train dataset.
    """
    n_samples = 100
    n_features = 10
    n_tasks = 10
    X = np.random.rand(n_samples, n_features)
    p = .05  # proportion actives
    y = np.random.binomial(1, p, size=(n_samples, n_tasks))
    w = np.ones((n_samples, n_tasks))
    dataset = dc.data.NumpyDataset(X, y, w)
    K = 5

    task_splitter = dc.splits.TaskSplitter()
    fold_datasets = task_splitter.k_fold_split(dataset, K)
    # Number tasks per fold
    n_per_fold = 2

    for fold in range(K):
      train_inds = list(set(range(K)) - set([fold]))
      train_fold_datasets = [fold_datasets[ind] for ind in train_inds]
      train_dataset = dc.splits.merge_fold_datasets(train_fold_datasets)

      # Find the tasks that correspond to this test fold
      train_tasks = list(
          set(range(10)) -
          set(range(fold * n_per_fold, (fold + 1) * n_per_fold)))

      # Assert that all arrays look like they should
      np.testing.assert_array_equal(train_dataset.X, X)
      np.testing.assert_array_equal(train_dataset.y, y[:, train_tasks])
      np.testing.assert_array_equal(train_dataset.w, w[:, train_tasks])
      np.testing.assert_array_equal(train_dataset.X, X)


for fold in range(K - 1):
      fold_dataset = fold_datasets[fold]
      assert len(fold_dataset.get_task_names()) == 4

"""
Tests for splitter objects.
"""
import unittest
import numpy as np
import deepchem as dc

class TestTaskSplitters(unittest.TestCase):
  """
  Test some basic splitters.
  """

  def test_multitask_train_valid_test_split(self):
    """
    Test TaskSplitter train/valid/test split on multitask dataset.
    """
    n_samples = 100
    n_features = 10
    n_tasks = 10
    X = np.random.rand(n_samples, n_features)
    p = .05  # proportion actives
    y = np.random.binomial(1, p, size=(n_samples, n_tasks))
    dataset = dc.data.NumpyDataset(X, y)

    task_splitter = dc.splits.TaskSplitter()
    train, valid, test = task_splitter.train_valid_test_split(
        dataset, frac_train=.4, frac_valid=.3, frac_test=.3)

    assert len(train.get_task_names()) == 4
    assert len(valid.get_task_names()) == 3
    assert len(test.get_task_names()) == 3

  def test_multitask_K_fold_split(self):
    """
    Test TaskSplitter K-fold split on multitask dataset.
    """
    n_samples = 100
    n_features = 10
    n_tasks = 10
    X = np.random.rand(n_samples, n_features)
    p = .05  # proportion actives
    y = np.random.binomial(1, p, size=(n_samples, n_tasks))
    dataset = dc.data.NumpyDataset(X, y)
    K = 5

    task_splitter = dc.splits.TaskSplitter()
    fold_datasets = task_splitter.k_fold_split(dataset, K)

    for fold_dataset in fold_datasets:
      assert len(fold_dataset.get_task_names()) == 2

  def test_uneven_k_fold_split(self):
    """
    Test k-fold-split works when K does not divide n_tasks.
    """
    n_samples = 100
    n_features = 10
    n_tasks = 17
    X = np.random.rand(n_samples, n_features)
    p = .05  # proportion actives
    y = np.random.binomial(1, p, size=(n_samples, n_tasks))
    dataset = dc.data.NumpyDataset(X, y)
    K = 4
    task_splitter = dc.splits.TaskSplitter()
    fold_datasets = task_splitter.k_fold_split(dataset, K)

    assert len(fold_datasets[-1].get_task_names()) == 5

  def test_uneven_train_valid_test_split(self):
    """
    Test train/valid/test split works when proportions don't divide n_tasks.
    """
    n_samples = 100
    n_features = 10
    n_tasks = 11
    X = np.random.rand(n_samples, n_features)
    p = .05  # proportion actives
    y = np.random.binomial(1, p, size=(n_samples, n_tasks))
    dataset = dc.data.NumpyDataset(X, y)

    task_splitter = dc.splits.TaskSplitter()
    train, valid, test = task_splitter.train_valid_test_split(
        dataset, frac_train=.4, frac_valid=.3, frac_test=.3)

    assert len(train.get_task_names()) == 4
    assert len(valid.get_task_names()) == 3
    # Note that the extra task goes to test
    assert len(test.get_task_names()) == 4

  def test_merge_fold_datasets(self):
    """
    Test that (K-1) folds can be merged into train dataset.
    """
    n_samples = 100
    n_features = 10
    n_tasks = 10
    X = np.random.rand(n_samples, n_features)
    p = .05  # proportion actives
    y = np.random.binomial(1, p, size=(n_samples, n_tasks))
    w = np.ones((n_samples, n_tasks))
    dataset = dc.data.NumpyDataset(X, y, w)
    K = 5

    task_splitter = dc.splits.TaskSplitter()
    fold_datasets = task_splitter.k_fold_split(dataset, K)
    # Number tasks per fold
    n_per_fold = 2

    for fold in range(K):
      train_inds = list(set(range(K)) - set([fold]))
      train_fold_datasets = [fold_datasets[ind] for ind in train_inds]
      train_dataset = dc.splits.merge_fold_datasets(train_fold_datasets)

      # Find the tasks that correspond to this test fold
      train_tasks = list(
          set(range(10)) -
          set(range(fold * n_per_fold, (fold + 1) * n_per_fold)))

      # Assert that all arrays look like they should
      np.testing.assert_array_equal(train_dataset.X, X)
      np.testing.assert_array_equal(train_dataset.y, y[:, train_tasks])
      np.testing.assert_array_equal(train_dataset.w, w[:, train_tasks])
      np.testing.assert_array_equal(train_dataset.X, X)


"""RLock example

Allow to release only for thread that is created the lock
"""

import threading
import time

class Box(object):

    lock = threading.RLock()

    def __init__(self):
        self.total_items = 0

Box.lock.acquire()
        self.total_items += n
        Box.lock.release()

    def add(self):
        Box.lock.acquire()
        self.execute(1)
        Box.lock.release()

    def remove(self):
        Box.lock.acquire()
        self.execute(-1)
        Box.lock.release()

def adder(box, items):
    while items > 0:
        print('Adding 1 item to the box\n')
        box.add()
        time.sleep(5)
        items -= 1

def remover(box, items):
    while items > 0:
        print('Removing 1 item from the box\n')
        box.remove()
        time.sleep(5)
        items -= 1

if __name__ == '__main__':
    items = 5

    print('Putting', 5, 'items in the box')

    box = Box()

    t1 = threading.Thread(target=adder, args=(box, items))
    t2 = threading.Thread(target=remover, args=(box, items))

    t1.start()
    t2.start()

    t1.join()
    t2.join()

    print(box.total_items, 'items still remain in the box')


while items > 0:
        print('Adding 1 item to the box\n')
        box.add()
        time.sleep(5)
        items -= 1

"""RLock example

Allow to release only for thread that is created the lock
"""

import threading
import time

class Box(object):

    lock = threading.RLock()

    def __init__(self):
        self.total_items = 0

    def execute(self, n):
        Box.lock.acquire()
        self.total_items += n
        Box.lock.release()

    def add(self):
        Box.lock.acquire()
        self.execute(1)
        Box.lock.release()

    def remove(self):
        Box.lock.acquire()
        self.execute(-1)
        Box.lock.release()

def adder(box, items):

def remover(box, items):
    while items > 0:
        print('Removing 1 item from the box\n')
        box.remove()
        time.sleep(5)
        items -= 1

if __name__ == '__main__':
    items = 5

    print('Putting', 5, 'items in the box')

    box = Box()

    t1 = threading.Thread(target=adder, args=(box, items))
    t2 = threading.Thread(target=remover, args=(box, items))

    t1.start()
    t2.start()

    t1.join()
    t2.join()

    print(box.total_items, 'items still remain in the box')


"""RLock example

Allow to release only for thread that is created the lock
"""

import threading
import time

class Box(object):

    lock = threading.RLock()

    def __init__(self):
        self.total_items = 0

    def execute(self, n):
        Box.lock.acquire()
        self.total_items += n
        Box.lock.release()

    def add(self):
        Box.lock.acquire()
        self.execute(1)
        Box.lock.release()

    def remove(self):
        Box.lock.acquire()
        self.execute

        Box.lock.release()

def adder(box, items):
    while items > 0:
        print('Adding 1 item to the box\n')
        box.add()
        time.sleep(5)
        items -= 1

def remover(box, items):
    while items > 0:
        print('Removing 1 item from the box\n')
        box.remove()
        time.sleep(5)
        items -= 1

if __name__ == '__main__':
    items = 5

    print('Putting', 5, 'items in the box')

    box = Box()

    t1 = threading.Thread(target=adder, args=(box, items))
    t2 = threading.Thread(target=remover, args=(box, items))

    t1.start()
    t2.start()

    t1.join()
    t2.join()

    print(box.total_items, 'items still remain in the box')


if __name__ == '__main__':
    items = 5

    print('Putting', 5, 'items in the box')

    box = Box()

    t1 = threading.Thread(target=adder, args=(box, items))
    t2 = threading.Thread(target=remover, args=(box, items))

    t1.start()
    t2.start()

    t1.join()
    t2.join()

    print(box.total_items, 'items still remain in the box')

"""RLock example

Allow to release only for thread that is created the lock
"""

import threading
import time

class Box(object):

    lock = threading.RLock()

    def __init__(self):
        self.total_items = 0

    def execute(self, n):
        Box.lock.acquire()
        self.total_items += n
        Box.lock.release()

    def add(self):
        Box.lock.acquire()
        self.execute(1)
        Box.lock.release()

    def remove(self):
        Box.lock.acquire()
        self.execute(-1)
        Box.lock.release()

def adder(box, items):
    while items > 0:
        print('Adding 1 item to the box\n')
        box.add()
        time.sleep(5)
        items -= 1

def remover(box, items):
    while items > 0:
        print('Removing 1 item from the box\n')
        box.remove()
        time.sleep(5)
        items -= 1


import numpy as np

from ..element_h1 import ElementH1
from ...refdom import RefLine

class ElementLineP0(ElementH1):
    """Piecewise constant element."""

    interior_dofs = 1
    maxdeg = 0
    dofnames = ['u']
    doflocs = np.array([[.5]])
    refdom = RefLine

if i == 0:
            return 1. + 0. * X[0], 0. * X
        else:
            self._index_error()


if i == 0:
            return 1. + 0. * X[0], 0. * X
        else:
            self._index_error()

import numpy as np

from ..element_h1 import ElementH1
from ...refdom import RefLine

class ElementLineP0(ElementH1):
    """Piecewise constant element."""

    interior_dofs = 1
    maxdeg = 0
    dofnames = ['u']
    doflocs = np.array([[.5]])
    refdom = RefLine

    def lbasis(self, X, i):


import datetime
import logging
import time
from collections import OrderedDict
from contextlib import contextmanager

import torch

from vidgen.utils.comm import is_main_process
from vidgen.utils.logger import log_every_n_seconds

class DatasetEvaluator:
    """
    Base class for a dataset evaluator.

    The function :func:`inference_on_dataset` runs the model over
    all samples in the dataset, and have a DatasetEvaluator to process the inputs/outputs.

    This class will accumulate information of the inputs/outputs (by :meth:`process`),
    and produce evaluation results in the end (by :meth:`evaluate`).
    """

    def reset(self):
        """
        Preparation for a new round of evaluation.
        Should be called before starting a round of evaluation.
        """
        pass

    def process(self, input, output):
        """
        Process an input/output pair.

        Args:
            input: the input that's used to call the model.
            output: the return value of `model(input)`
        """
        pass

    def evaluate(self):
        """
        Evaluate/summarize the performance, after processing all input/output pairs.

        Returns:
            dict:
                A new evaluator class can return a dict of arbitrary format
                as long as the user can process the results.
                In our train_net.py, we expect the following format:

                * key: the name of the task (e.g., bbox)
                * value: a dict of {metric name: score}, e.g.: {"AP50": 80}
        """
        pass

class DatasetEvaluators(DatasetEvaluator):
    def __init__(self, evaluators):
        assert len(evaluators)
        super().__init__()
        self._evaluators = evaluators

    def reset(self):
        for evaluator in self._evaluators:
            evaluator.reset()

    def process(self, input, output):
        for evaluator in self._evaluators:
            evaluator.process(input, output)

results = OrderedDict()
        for evaluator in self._evaluators:
            result = evaluator.evaluate()
            if is_main_process() and result is not None:
                for k, v in result.items():
                    assert (
                        k not in results
                    ), "Different evaluators produce results with the same key {}".format(k)
                    results[k] = v
        return results

def inference_on_dataset(model, data_loader, evaluator):
    """
    Run model on the data_loader and evaluate the metrics with evaluator.
    Also benchmark the inference speed of `model.forward` accurately.
    The model will be used in eval mode.

    Args:
        model (nn.Module): a module which accepts an object from
            `data_loader` and returns some outputs. It will be temporarily set to `eval` mode.

            If you wish to evaluate a model in `training` mode instead, you can
            wrap the given model and override its behavior of `.eval()` and `.train()`.
        data_loader: an iterable object with a length.
            The elements it generates will be the inputs to the model.
        evaluator (DatasetEvaluator): the evaluator to run. Use `None` if you only want
            to benchmark, but don't want to do any evaluation.

    Returns:
        The return value of `evaluator.evaluate()`
    """
    num_devices = torch.distributed.get_world_size() if torch.distributed.is_initialized() else 1
    logger = logging.getLogger(__name__)
    logger.info("Start inference on {} images".format(len(data_loader)))

    total = len(data_loader)  # inference data loader must have a fixed length
    if evaluator is None:
        # create a no-op evaluator
        evaluator = DatasetEvaluators([])
    evaluator.reset()

    num_warmup = min(5, total - 1)
    start_time = time.perf_counter()
    total_compute_time = 0
    with inference_context(model), torch.no_grad():
        for idx, inputs in enumerate(data_loader):
            if idx == num_warmup:
                start_time = time.perf_counter()
                total_compute_time = 0

            start_compute_time = time.perf_counter()
            outputs = model(inputs)
            if torch.cuda.is_available():
                torch.cuda.synchronize()
            total_compute_time += time.perf_counter() - start_compute_time
            evaluator.process(inputs, outputs)

            iters_after_start = idx + 1 - num_warmup * int(idx >= num_warmup)
            seconds_per_img = total_compute_time / iters_after_start
            if idx >= num_warmup * 2 or seconds_per_img > 5:
                total_seconds_per_img = (time.perf_counter() - start_time) / iters_after_start
                eta = datetime.timedelta(seconds=int(total_seconds_per_img * (total - idx - 1)))
                log_every_n_seconds(
                    logging.INFO,
                    "Inference done {}/{}. {:.4f} s / img. ETA={}".format(
                        idx + 1, total, seconds_per_img, str(eta)
                    ),
                    n=5,
                )

    # Measure the time only for this worker (before the synchronization barrier)
    total_time = time.perf_counter() - start_time
    total_time_str = str(datetime.timedelta(seconds=total_time))
    # NOTE this format is parsed by grep
    logger.info(
        "Total inference time: {} ({:.6f} s / img per device, on {} devices)".format(
            total_time_str, total_time / (total - num_warmup), num_devices
        )
    )
    total_compute_time_str = str(datetime.timedelta(seconds=int(total_compute_time)))
    logger.info(
        "Total inference pure compute time: {} ({:.6f} s / img per device, on {} devices)".format(
            total_compute_time_str, total_compute_time / (total - num_warmup), num_devices
        )
    )

    results = evaluator.evaluate()
    # An evaluator may return None when not in main process.
    # Replace it by an empty dict instead to make it easier for downstream code to handle
    if results is None:
        results = {}
    return results

@contextmanager
def inference_context(model):
    """
    A context where the model is temporarily changed to eval mode,
    and restored to previous mode afterwards.

    Args:
        model: a torch Module
    """
    training_mode = model.training
    model.eval()
    yield
    model.train(training_mode)


assert len(evaluators)
        super().__init__()
        self._evaluators = evaluators

import datetime
import logging
import time
from collections import OrderedDict
from contextlib import contextmanager

import torch

from vidgen.utils.comm import is_main_process
from vidgen.utils.logger import log_every_n_seconds

class DatasetEvaluator:
    """
    Base class for a dataset evaluator.

    The function :func:`inference_on_dataset` runs the model over
    all samples in the dataset, and have a DatasetEvaluator to process the inputs/outputs.

    This class will accumulate information of the inputs/outputs (by :meth:`process`),
    and produce evaluation results in the end (by :meth:`evaluate`).
    """

    def reset(self):
        """
        Preparation for a new round of evaluation.
        Should be called before starting a round of evaluation.
        """
        pass

    def process(self, input, output):
        """
        Process an input/output pair.

        Args:
            input: the input that's used to call the model.
            output: the return value of `model(input)`
        """
        pass

    def evaluate(self):
        """
        Evaluate/summarize the performance, after processing all input/output pairs.

        Returns:
            dict:
                A new evaluator class can return a dict of arbitrary format
                as long as the user can process the results.
                In our train_net.py, we expect the following format:

                * key: the name of the task (e.g., bbox)
                * value: a dict of {metric name: score}, e.g.: {"AP50": 80}
        """
        pass

class DatasetEvaluators(DatasetEvaluator):
    def __init__(self, evaluators):

    def reset(self):
        for evaluator in self._evaluators:
            evaluator.reset()

    def process(self, input, output):
        for evaluator in self._evaluators:
            evaluator.process(input, output)

    def evaluate(self):
        results = OrderedDict()
        for evaluator in self._evaluators:
            result = evaluator.evaluate()
            if is_main_process() and result is not None:
                for k, v in result.items():
                    assert (
                        k not in results
                    ), "Different evaluators produce results with the same key {}".format(k)
                    results[k] = v
        return results

def inference_on_dataset(model, data_loader, evaluator):
    """
    Run model on the data_loader and evaluate the metrics with evaluator.
    Also benchmark the inference speed of `model.forward` accurately.
    The model will be used in eval mode.

    Args:
        model (nn.Module): a module which accepts an object from
            `data_loader` and returns some outputs. It will be temporarily set to `eval` mode.

            If you wish to evaluate a model in `training` mode instead, you can
            wrap the given model and override its behavior of `.eval()` and `.train()`.
        data_loader: an iterable object with a length.
            The elements it generates will be the inputs to the model.
        evaluator (DatasetEvaluator): the evaluator to run. Use `None` if you only want
            to benchmark, but don't want to do any evaluation.

    Returns:
        The return value of `evaluator.evaluate()`
    """
    num_devices = torch.distributed.get_world_size() if torch.distributed.is_initialized() else 1
    logger = logging.getLogger(__name__)
    logger.info("Start inference on {} images".format(len(data_loader)))

    total = len(data_loader)  # inference data loader must have a fixed length
    if evaluator is None:
        # create a no-op evaluator
        evaluator = DatasetEvaluators([])
    evaluator.reset()

    num_warmup = min(5, total - 1)
    start_time = time.perf_counter()
    total_compute_time = 0
    with inference_context(model), torch.no_grad():
        for idx, inputs in enumerate(data_loader):
            if idx == num_warmup:
                start_time = time.perf_counter()
                total_compute_time = 0

            start_compute_time = time.perf_counter()
            outputs = model(inputs)
            if torch.cuda.is_available():
                torch.cuda.synchronize()
            total_compute_time += time.perf_counter() - start_compute_time
            evaluator.process(inputs, outputs)

            iters_after_start = idx + 1 - num_warmup * int(idx >= num_warmup)
            seconds_per_img = total_compute_time / iters_after_start
            if idx >= num_warmup * 2 or seconds_per_img > 5:
                total_seconds_per_img = (time.perf_counter() - start_time) / iters_after_start
                eta = datetime.timedelta(seconds=int(total_seconds_per_img * (total - idx - 1)))
                log_every_n_seconds(
                    logging.INFO,
                    "Inference done {}/{}. {:.4f} s / img. ETA={}".format(
                        idx + 1, total, seconds_per_img, str(eta)
                    ),
                    n=5,
                )

    # Measure the time only for this worker (before the synchronization barrier)
    total_time = time.perf_counter() - start_time
    total_time_str = str(datetime.timedelta(seconds=total_time))
    # NOTE this format is parsed by grep
    logger.info(
        "Total inference time: {} ({:.6f} s / img per device, on {} devices)".format(
            total_time_str, total_time / (total - num_warmup), num_devices
        )
    )
    total_compute_time_str = str(datetime.timedelta(seconds=int(total_compute_time)))
    logger.info(
        "Total inference pure compute time: {} ({:.6f} s / img per device, on {} devices)".format(
            total_compute_time_str, total_compute_time / (total - num_warmup), num_devices
        )
    )

    results = evaluator.evaluate()
    # An evaluator may return None when not in main process.
    # Replace it by an empty dict instead to make it easier for downstream code to handle
    if results is None:
        results = {}
    return results

@contextmanager
def inference_context(model):
    """
    A context where the model is temporarily changed to eval mode,
    and restored to previous mode afterwards.

    Args:
        model: a torch Module
    """
    training_mode = model.training
    model.eval()
    yield
    model.train(training_mode)


"""
    A context where the model is temporarily changed to eval mode,
    and restored to previous mode afterwards.

    Args:
        model: a torch Module
    """
    training_mode = model.training

import datetime
import logging
import time
from collections import OrderedDict
from contextlib import contextmanager

import torch

from vidgen.utils.comm import is_main_process
from vidgen.utils.logger import log_every_n_seconds

class DatasetEvaluator:
    """
    Base class for a dataset evaluator.

    The function :func:`inference_on_dataset` runs the model over
    all samples in the dataset, and have a DatasetEvaluator to process the inputs/outputs.

    This class will accumulate information of the inputs/outputs (by :meth:`process`),
    and produce evaluation results in the end (by :meth:`evaluate`).
    """

    def reset(self):
        """
        Preparation for a new round of evaluation.
        Should be called before starting a round of evaluation.
        """
        pass

    def process(self, input, output):
        """
        Process an input/output pair.

        Args:
            input: the input that's used to call the model.
            output: the return value of `model(input)`
        """
        pass

    def evaluate(self):
        """
        Evaluate/summarize the performance, after processing all input/output pairs.

        Returns:
            dict:
                A new evaluator class can return a dict of arbitrary format
                as long as the user can process the results.
                In our train_net.py, we expect the following format:

                * key: the name of the task (e.g., bbox)
                * value: a dict of {metric name: score}, e.g.: {"AP50": 80}
        """
        pass

class DatasetEvaluators(DatasetEvaluator):
    def __init__(self, evaluators):
        assert len(evaluators)
        super().__init__()
        self._evaluators = evaluators

    def reset(self):
        for evaluator in self._evaluators:
            evaluator.reset()

    def process(self, input, output):
        for evaluator in self._evaluators:
            evaluator.process(input, output)

    def evaluate(self):
        results = OrderedDict()
        for evaluator in self._evaluators:
            result = evaluator.evaluate()
            if is_main_process() and result is not None:
                for k, v in result.items():
                    assert (
                        k not in results
                    ), "Different evaluators produce results with the same key {}".format(k)
                    results[k] = v
        return results

def inference_on_dataset(model, data_loader, evaluator):
    """
    Run model on the data_loader and evaluate the metrics with evaluator.
    Also benchmark the inference speed of `model.forward` accurately.
    The model will be used in eval mode.

    Args:
        model (nn.Module): a module which accepts an object from
            `data_loader` and returns some outputs. It will be temporarily set to `eval` mode.

            If you wish to evaluate a model in `training` mode instead, you can
            wrap the given model and override its behavior of `.eval()` and `.train()`.
        data_loader: an iterable object with a length.
            The elements it generates will be the inputs to the model.
        evaluator (DatasetEvaluator): the evaluator to run. Use `None` if you only want
            to benchmark, but don't want to do any evaluation.

    Returns:
        The return value of `evaluator.evaluate()`
    """
    num_devices = torch.distributed.get_world_size() if torch.distributed.is_initialized() else 1
    logger = logging.getLogger(__name__)
    logger.info("Start inference on {} images".format(len(data_loader)))

    total = len(data_loader)  # inference data loader must have a fixed length
    if evaluator is None:
        # create a no-op evaluator
        evaluator = DatasetEvaluators([])
    evaluator.reset()

    num_warmup = min(5, total - 1)
    start_time = time.perf_counter()
    total_compute_time = 0
    with inference_context(model), torch.no_grad():
        for idx, inputs in enumerate(data_loader):
            if idx == num_warmup:
                start_time = time.perf_counter()
                total_compute_time = 0

            start_compute_time = time.perf_counter()
            outputs = model(inputs)
            if torch.cuda.is_available():
                torch.cuda.synchronize()
            total_compute_time += time.perf_counter() - start_compute_time
            evaluator.process(inputs, outputs)

            iters_after_start = idx + 1 - num_warmup * int(idx >= num_warmup)
            seconds_per_img = total_compute_time / iters_after_start
            if idx >= num_warmup * 2 or seconds_per_img > 5:
                total_seconds_per_img = (time.perf_counter() - start_time) / iters_after_start
                eta = datetime.timedelta(seconds=int(total_seconds_per_img * (total - idx - 1)))
                log_every_n_seconds(
                    logging.INFO,
                    "Inference done {}/{}. {:.4f} s / img. ETA={}".format(
                        idx + 1, total, seconds_per_img, str(eta)
                    ),
                    n=5,
                )

    # Measure the time only for this worker (before the synchronization barrier)
    total_time = time.perf_counter() - start_time
    total_time_str = str(datetime.timedelta(seconds=total_time))
    # NOTE this format is parsed by grep
    logger.info(
        "Total inference time: {} ({:.6f} s / img per device, on {} devices)".format(
            total_time_str, total_time / (total - num_warmup), num_devices
        )
    )
    total_compute_time_str = str(datetime.timedelta(seconds=int(total_compute_time)))
    logger.info(
        "Total inference pure compute time: {} ({:.6f} s / img per device, on {} devices)".format(
            total_compute_time_str, total_compute_time / (total - num_warmup), num_devices
        )
    )

    results = evaluator.evaluate()
    # An evaluator may return None when not in main process.
    # Replace it by an empty dict instead to make it easier for downstream code to handle
    if results is None:
        results = {}
    return results

@contextmanager
def inference_context(model):

    model.eval()
    yield
    model.train(training_mode)


import datetime
import logging
import time
from collections import OrderedDict
from contextlib import contextmanager

import torch

from vidgen.utils.comm import is_main_process
from vidgen.utils.logger import log_every_n_seconds

class DatasetEvaluator:
    """
    Base class for a dataset evaluator.

    The function :func:`inference_on_dataset` runs the model over
    all samples in the dataset, and have a DatasetEvaluator to process the inputs/outputs.

    This class will accumulate information of the inputs/outputs (by :meth:`process`),
    and produce evaluation results in the end (by :meth:`evaluate`).
    """

    def reset(self):
        """
        Preparation for a new round of evaluation.
        Should be called before starting a round of evaluation.
        """
        pass

    def process(self, input, output):
        """
        Process an input/output pair.

        Args:
            input: the input that's used to call the model.
            output: the return value of `model(input)`
        """
        pass

    def evaluate(self):
        """
        Evaluate/summarize the performance, after processing all input/output pairs.

        Returns:
            dict:
                A new evaluator class can return a dict of arbitrary format
                as long as the user can process the results.
                In our train_net.py, we expect the following format:

                * key: the name of the task (e.g., bbox)
                * value: a dict of {metric name: score}, e.g.: {"AP50": 80}
        """
        pass

class DatasetEvaluators(DatasetEvaluator):
    def __init__(self, evaluators):
        assert len(evaluators)
        super().__init__()
        self._evaluators = evaluators

    def reset(self):
        for evaluator in self._evaluators:
            evaluator.reset()

    def process(self, input, output):
        for evaluator in self._evaluators:
            evaluator.process(input, output)

    def evaluate(self):
        results = OrderedDict()
        for evaluator in self._evaluators:
            result = evaluator.evaluate()
            if is_main_process() and result is not None:
                for k, v in result.items():
                    assert (
                        k not in results
                    ), "Different evaluators produce results with the same key {}".format(k)
                    results[k] = v
        return results

def inference_on_dataset(model, data_loader, evaluator):
    """
    Run model on the data_loader and evaluate the metrics with evaluator.
    Also benchmark the inference speed of `model.forward` accurately.
    The model will be used in eval mode.

    Args:
        model (nn.Module): a module which accepts an object from
            `data_loader` and returns some outputs. It will be temporarily set to `eval` mode.

            If you wish to evaluate a model in `training` mode instead, you can
            wrap the given model and override its behavior of `.eval()` and `.train()`.
        data_loader: an iterable object with a length.
            The elements it generates will be the inputs to the model.
        evaluator (DatasetEvaluator): the evaluator to run. Use `None` if you only want
            to benchmark, but don't want to do any evaluation.

    Returns:
        The return value of `evaluator.evaluate()`
    """
    num_devices = torch.distributed.get_world_size() if torch.distributed.is_initialized() else 1
    logger = logging.getLogger(__name__)
    logger.info("Start inference on {} images".format(len(data_loader)))

    total = len(data_loader)  # inference data loader must have a fixed length
    if evaluator is None:
        # create a no-op evaluator
        evaluator = DatasetEvaluators([])
    evaluator.reset()

    num_warmup = min(5, total - 1)
    start_time = time.perf_counter()
    total_compute_time = 0
    with inference_context(model), torch.no_grad():
        for idx, inputs in enumerate(data_loader):
            if idx == num_warmup:
                start_time = time.perf_counter()
                total_compute_time = 0

            start_compute_time = time.perf_counter()
            outputs = model(inputs)
            if torch.cuda.is_available():
                torch.cuda.synchronize()
            total_compute_time += time.perf_counter() - start_compute_time
            evaluator.process(inputs, outputs)

            iters_after_start = idx + 1 - num_warmup * int(idx >= num_warmup)
            seconds_per_img = total_compute_time / iters_after_start
            if idx >= num_warmup * 2 or seconds_per_img > 5:
                total_seconds_per_img = (time.perf_counter() - start_time) / iters_after_start
                eta = datetime.timedelta(seconds=int(total_seconds_per_img * (total - idx - 1)))
                log_every_n_seconds(
                    logging.INFO,
                    "Inference done {}/{}. {:.4f} s / img. ETA={}".format(
                        idx + 1, total, seconds_per_img, str(eta)
                    ),
                    n=5,
                )

    # Measure the time only for this worker (before the synchronization barrier)
    total_time = time.perf_counter() - start_time
    total_time_str = str(datetime.timedelta(seconds=total_time))
    # NOTE this format is parsed by grep
    logger.info(
        "Total inference time: {} ({:.6f} s / img per device, on {} devices)".format(
            total_time_str, total_time / (total - num_warmup), num_devices
        )
    )
    total_compute_time_str = str(datetime.timedelta(seconds=int(total_compute_time)))
    logger.info(
        "Total inference pure compute time: {} ({:.6f} s / img per device, on {} devices)".format(
            total_compute_time_str, total_compute_time / (total - num_warmup), num_devices
        )
    )

    results = evaluator.evaluate()
    # An evaluator may return None when not in main process.
    # Replace it by an empty dict instead to make it easier for downstream code to handle
    if results is None:
        results = {}
    return results

@contextmanager
def inference_context(model):
    """
    A context where the model is temporarily changed to eval mode,
    and restored to previous mode afterwards.

    Args:
        model: a torch Module
    """
    training_mode = model.training

    yield
    model.train(training_mode)


import datetime
import logging
import time
from collections import OrderedDict
from contextlib import contextmanager

import torch

from vidgen.utils.comm import is_main_process
from vidgen.utils.logger import log_every_n_seconds

class DatasetEvaluator:
    """
    Base class for a dataset evaluator.

    The function :func:`inference_on_dataset` runs the model over
    all samples in the dataset, and have a DatasetEvaluator to process the inputs/outputs.

    This class will accumulate information of the inputs/outputs (by :meth:`process`),
    and produce evaluation results in the end (by :meth:`evaluate`).
    """

    def reset(self):
        """
        Preparation for a new round of evaluation.
        Should be called before starting a round of evaluation.
        """
        pass

    def process(self, input, output):
        """
        Process an input/output pair.

        Args:
            input: the input that's used to call the model.
            output: the return value of `model(input)`
        """
        pass

    def evaluate(self):
        """
        Evaluate/summarize the performance, after processing all input/output pairs.

        Returns:
            dict:
                A new evaluator class can return a dict of arbitrary format
                as long as the user can process the results.
                In our train_net.py, we expect the following format:

                * key: the name of the task (e.g., bbox)
                * value: a dict of {metric name: score}, e.g.: {"AP50": 80}
        """
        pass

class DatasetEvaluators(DatasetEvaluator):
    def __init__(self, evaluators):
        assert len(evaluators)
        super().__init__()
        self._evaluators = evaluators

    def reset(self):
        for evaluator in self._evaluators:
            evaluator.reset()

    def process(self, input, output):
        for evaluator in self._evaluators:
            evaluator.process(input, output)

    def evaluate(self):
        results = OrderedDict()
        for evaluator in self._evaluators:
            result = evaluator.evaluate()
            if is_main_process() and result is not None:
                for k, v in result.items():
                    assert (
                        k not in results
                    ), "Different evaluators produce results with the same key {}".format(k)
                    results[k] = v
        return results

def inference_on_dataset(model, data_loader, evaluator):
    """
    Run model on the data_loader and evaluate the metrics with evaluator.
    Also benchmark the inference speed of `model.forward` accurately.
    The model will be used in eval mode.

    Args:
        model (nn.Module): a module which accepts an object from
            `data_loader` and returns some outputs. It will be temporarily set to `eval` mode.

            If you wish to evaluate a model in `training` mode instead, you can
            wrap the given model and override its behavior of `.eval()` and `.train()`.
        data_loader: an iterable object with a length.
            The elements it generates will be the inputs to the model.
        evaluator (DatasetEvaluator): the evaluator to run. Use `None` if you only want
            to benchmark, but don't want to do any evaluation.

    Returns:
        The return value of `evaluator.evaluate()`
    """
    num_devices = torch.distributed.get_world_size() if torch.distributed.is_initialized() else 1
    logger = logging.getLogger(__name__)
    logger.info("Start inference on {} images".format(len(data_loader)))

    total = len(data_loader)  # inference data loader must have a fixed length
    if evaluator is None:
        # create a no-op evaluator
        evaluator = DatasetEvaluators([])
    evaluator.reset()

    num_warmup = min(5, total - 1)
    start_time = time.perf_counter()
    total_compute_time = 0
    with inference_context(model), torch.no_grad():
        for idx, inputs in enumerate(data_loader):
            if idx == num_warmup:
                start_time = time.perf_counter()
                total_compute_time = 0

            start_compute_time = time.perf_counter()
            outputs = model(inputs)
            if torch.cuda.is_available():
                torch.cuda.synchronize()
            total_compute_time += time.perf_counter() - start_compute_time
            evaluator.process(inputs, outputs)

            iters_after_start = idx + 1 - num_warmup * int(idx >= num_warmup)
            seconds_per_img = total_compute_time / iters_after_start
            if idx >= num_warmup * 2 or seconds_per_img > 5:
                total_seconds_per_img = (time.perf_counter() - start_time) / iters_after_start
                eta = datetime.timedelta(seconds=int(total_seconds_per_img * (total - idx - 1)))
                log_every_n_seconds(
                    logging.INFO,
                    "Inference done {}/{}. {:.4f} s / img. ETA={}".format(
                        idx + 1, total, seconds_per_img, str(eta)
                    ),
                    n=5,
                )

    # Measure the time only for this worker (before the synchronization barrier)
    total_time = time.perf_counter() - start_time
    total_time_str = str(datetime.timedelta(seconds=total_time))
    # NOTE this format is parsed by grep
    logger.info(
        "Total inference time: {} ({:.6f} s / img per device, on {} devices)".format(
            total_time_str, total_time / (total - num_warmup), num_devices
        )
    )
    total_compute_time_str = str(datetime.timedelta(seconds=int(total_compute_time)))
    logger.info(
        "Total inference pure compute time: {} ({:.6f} s / img per device, on {} devices)".format(
            total_compute_time_str, total_compute_time / (total - num_warmup), num_devices
        )
    )

    results = evaluator.evaluate()
    # An evaluator may return None when not in main process.
    # Replace it by an empty dict instead to make it easier for downstream code to handle
    if results is None:
        results = {}
    return results

@contextmanager
def inference_context(model):
    """
    A context where the model is temporarily changed to eval mode,
    and restored to previous mode afterwards.

    Args:
        model: a torch Module
    """
    training_mode = model.training
    model.eval()


import datetime
import logging
import time
from collections import OrderedDict
from contextlib import contextmanager

import torch

from vidgen.utils.comm import is_main_process
from vidgen.utils.logger import log_every_n_seconds

class DatasetEvaluator:
    """
    Base class for a dataset evaluator.

    The function :func:`inference_on_dataset` runs the model over
    all samples in the dataset, and have a DatasetEvaluator to process the inputs/outputs.

    This class will accumulate information of the inputs/outputs (by :meth:`process`),
    and produce evaluation results in the end (by :meth:`evaluate`).
    """

    def reset(self):
        """
        Preparation for a new round of evaluation.
        Should be called before starting a round of evaluation.
        """
        pass

    def process(self, input, output):
        """
        Process an input/output pair.

        Args:
            input: the input that's used to call the model.
            output: the return value of `model(input)`
        """
        pass

    def evaluate(self):
        """
        Evaluate/summarize the performance, after processing all input/output pairs.

        Returns:
            dict:
                A new evaluator class can return a dict of arbitrary format
                as long as the user can process the results.
                In our train_net.py, we expect the following format:

                * key: the name of the task (e.g., bbox)
                * value: a dict of {metric name: score}, e.g.: {"AP50": 80}
        """
        pass

class DatasetEvaluators(DatasetEvaluator):
    def __init__(self, evaluators):
        assert len(evaluators)
        super().__init__()
        self._evaluators = evaluators

    def reset(self):
        for evaluator in self._evaluators:
            evaluator.reset()

    def process(self, input, output):
        for evaluator in self._evaluators:
            evaluator.process(input, output)

    def evaluate(self):
        results =

        for evaluator in self._evaluators:
            result = evaluator.evaluate()
            if is_main_process() and result is not None:
                for k, v in result.items():
                    assert (
                        k not in results
                    ), "Different evaluators produce results with the same key {}".format(k)
                    results[k] = v
        return results

def inference_on_dataset(model, data_loader, evaluator):
    """
    Run model on the data_loader and evaluate the metrics with evaluator.
    Also benchmark the inference speed of `model.forward` accurately.
    The model will be used in eval mode.

    Args:
        model (nn.Module): a module which accepts an object from
            `data_loader` and returns some outputs. It will be temporarily set to `eval` mode.

            If you wish to evaluate a model in `training` mode instead, you can
            wrap the given model and override its behavior of `.eval()` and `.train()`.
        data_loader: an iterable object with a length.
            The elements it generates will be the inputs to the model.
        evaluator (DatasetEvaluator): the evaluator to run. Use `None` if you only want
            to benchmark, but don't want to do any evaluation.

    Returns:
        The return value of `evaluator.evaluate()`
    """
    num_devices = torch.distributed.get_world_size() if torch.distributed.is_initialized() else 1
    logger = logging.getLogger(__name__)
    logger.info("Start inference on {} images".format(len(data_loader)))

    total = len(data_loader)  # inference data loader must have a fixed length
    if evaluator is None:
        # create a no-op evaluator
        evaluator = DatasetEvaluators([])
    evaluator.reset()

    num_warmup = min(5, total - 1)
    start_time = time.perf_counter()
    total_compute_time = 0
    with inference_context(model), torch.no_grad():
        for idx, inputs in enumerate(data_loader):
            if idx == num_warmup:
                start_time = time.perf_counter()
                total_compute_time = 0

            start_compute_time = time.perf_counter()
            outputs = model(inputs)
            if torch.cuda.is_available():
                torch.cuda.synchronize()
            total_compute_time += time.perf_counter() - start_compute_time
            evaluator.process(inputs, outputs)

            iters_after_start = idx + 1 - num_warmup * int(idx >= num_warmup)
            seconds_per_img = total_compute_time / iters_after_start
            if idx >= num_warmup * 2 or seconds_per_img > 5:
                total_seconds_per_img = (time.perf_counter() - start_time) / iters_after_start
                eta = datetime.timedelta(seconds=int(total_seconds_per_img * (total - idx - 1)))
                log_every_n_seconds(
                    logging.INFO,
                    "Inference done {}/{}. {:.4f} s / img. ETA={}".format(
                        idx + 1, total, seconds_per_img, str(eta)
                    ),
                    n=5,
                )

    # Measure the time only for this worker (before the synchronization barrier)
    total_time = time.perf_counter() - start_time
    total_time_str = str(datetime.timedelta(seconds=total_time))
    # NOTE this format is parsed by grep
    logger.info(
        "Total inference time: {} ({:.6f} s / img per device, on {} devices)".format(
            total_time_str, total_time / (total - num_warmup), num_devices
        )
    )
    total_compute_time_str = str(datetime.timedelta(seconds=int(total_compute_time)))
    logger.info(
        "Total inference pure compute time: {} ({:.6f} s / img per device, on {} devices)".format(
            total_compute_time_str, total_compute_time / (total - num_warmup), num_devices
        )
    )

    results = evaluator.evaluate()
    # An evaluator may return None when not in main process.
    # Replace it by an empty dict instead to make it easier for downstream code to handle
    if results is None:
        results = {}
    return results

@contextmanager
def inference_context(model):
    """
    A context where the model is temporarily changed to eval mode,
    and restored to previous mode afterwards.

    Args:
        model: a torch Module
    """
    training_mode = model.training
    model.eval()
    yield
    model.train(training_mode)


import datetime
import logging
import time
from collections import OrderedDict
from contextlib import contextmanager

import torch

from vidgen.utils.comm import is_main_process
from vidgen.utils.logger import log_every_n_seconds

class DatasetEvaluator:
    """
    Base class for a dataset evaluator.

    The function :func:`inference_on_dataset` runs the model over
    all samples in the dataset, and have a DatasetEvaluator to process the inputs/outputs.

    This class will accumulate information of the inputs/outputs (by :meth:`process`),
    and produce evaluation results in the end (by :meth:`evaluate`).
    """

    def reset(self):
        """
        Preparation for a new round of evaluation.
        Should be called before starting a round of evaluation.
        """

    def process(self, input, output):
        """
        Process an input/output pair.

        Args:
            input: the input that's used to call the model.
            output: the return value of `model(input)`
        """
        pass

    def evaluate(self):
        """
        Evaluate/summarize the performance, after processing all input/output pairs.

        Returns:
            dict:
                A new evaluator class can return a dict of arbitrary format
                as long as the user can process the results.
                In our train_net.py, we expect the following format:

                * key: the name of the task (e.g., bbox)
                * value: a dict of {metric name: score}, e.g.: {"AP50": 80}
        """
        pass

class DatasetEvaluators(DatasetEvaluator):
    def __init__(self, evaluators):
        assert len(evaluators)
        super().__init__()
        self._evaluators = evaluators

    def reset(self):
        for evaluator in self._evaluators:
            evaluator.reset()

    def process(self, input, output):
        for evaluator in self._evaluators:
            evaluator.process(input, output)

    def evaluate(self):
        results = OrderedDict()
        for evaluator in self._evaluators:
            result = evaluator.evaluate()
            if is_main_process() and result is not None:
                for k, v in result.items():
                    assert (
                        k not in results
                    ), "Different evaluators produce results with the same key {}".format(k)
                    results[k] = v
        return results

def inference_on_dataset(model, data_loader, evaluator):
    """
    Run model on the data_loader and evaluate the metrics with evaluator.
    Also benchmark the inference speed of `model.forward` accurately.
    The model will be used in eval mode.

    Args:
        model (nn.Module): a module which accepts an object from
            `data_loader` and returns some outputs. It will be temporarily set to `eval` mode.

            If you wish to evaluate a model in `training` mode instead, you can
            wrap the given model and override its behavior of `.eval()` and `.train()`.
        data_loader: an iterable object with a length.
            The elements it generates will be the inputs to the model.
        evaluator (DatasetEvaluator): the evaluator to run. Use `None` if you only want
            to benchmark, but don't want to do any evaluation.

    Returns:
        The return value of `evaluator.evaluate()`
    """
    num_devices = torch.distributed.get_world_size() if torch.distributed.is_initialized() else 1
    logger = logging.getLogger(__name__)
    logger.info("Start inference on {} images".format(len(data_loader)))

    total = len(data_loader)  # inference data loader must have a fixed length
    if evaluator is None:
        # create a no-op evaluator
        evaluator = DatasetEvaluators([])
    evaluator.reset()

    num_warmup = min(5, total - 1)
    start_time = time.perf_counter()
    total_compute_time = 0
    with inference_context(model), torch.no_grad():
        for idx, inputs in enumerate(data_loader):
            if idx == num_warmup:
                start_time = time.perf_counter()
                total_compute_time = 0

            start_compute_time = time.perf_counter()
            outputs = model(inputs)
            if torch.cuda.is_available():
                torch.cuda.synchronize()
            total_compute_time += time.perf_counter() - start_compute_time
            evaluator.process(inputs, outputs)

            iters_after_start = idx + 1 - num_warmup * int(idx >= num_warmup)
            seconds_per_img = total_compute_time / iters_after_start
            if idx >= num_warmup * 2 or seconds_per_img > 5:
                total_seconds_per_img = (time.perf_counter() - start_time) / iters_after_start
                eta = datetime.timedelta(seconds=int(total_seconds_per_img * (total - idx - 1)))
                log_every_n_seconds(
                    logging.INFO,
                    "Inference done {}/{}. {:.4f} s / img. ETA={}".format(
                        idx + 1, total, seconds_per_img, str(eta)
                    ),
                    n=5,
                )

    # Measure the time only for this worker (before the synchronization barrier)
    total_time = time.perf_counter() - start_time
    total_time_str = str(datetime.timedelta(seconds=total_time))
    # NOTE this format is parsed by grep
    logger.info(
        "Total inference time: {} ({:.6f} s / img per device, on {} devices)".format(
            total_time_str, total_time / (total - num_warmup), num_devices
        )
    )
    total_compute_time_str = str(datetime.timedelta(seconds=int(total_compute_time)))
    logger.info(
        "Total inference pure compute time: {} ({:.6f} s / img per device, on {} devices)".format(
            total_compute_time_str, total_compute_time / (total - num_warmup), num_devices
        )
    )

    results = evaluator.evaluate()
    # An evaluator may return None when not in main process.
    # Replace it by an empty dict instead to make it easier for downstream code to handle
    if results is None:
        results = {}
    return results

@contextmanager
def inference_context(model):
    """
    A context where the model is temporarily changed to eval mode,
    and restored to previous mode afterwards.

    Args:
        model: a torch Module
    """
    training_mode = model.training
    model.eval()
    yield
    model.train(training_mode)


if evaluator is None:
        # create a no-op evaluator
        evaluator = DatasetEvaluators([])

import datetime
import logging
import time
from collections import OrderedDict
from contextlib import contextmanager

import torch

from vidgen.utils.comm import is_main_process
from vidgen.utils.logger import log_every_n_seconds

class DatasetEvaluator:
    """
    Base class for a dataset evaluator.

    The function :func:`inference_on_dataset` runs the model over
    all samples in the dataset, and have a DatasetEvaluator to process the inputs/outputs.

    This class will accumulate information of the inputs/outputs (by :meth:`process`),
    and produce evaluation results in the end (by :meth:`evaluate`).
    """

    def reset(self):
        """
        Preparation for a new round of evaluation.
        Should be called before starting a round of evaluation.
        """
        pass

    def process(self, input, output):
        """
        Process an input/output pair.

        Args:
            input: the input that's used to call the model.
            output: the return value of `model(input)`
        """
        pass

    def evaluate(self):
        """
        Evaluate/summarize the performance, after processing all input/output pairs.

        Returns:
            dict:
                A new evaluator class can return a dict of arbitrary format
                as long as the user can process the results.
                In our train_net.py, we expect the following format:

                * key: the name of the task (e.g., bbox)
                * value: a dict of {metric name: score}, e.g.: {"AP50": 80}
        """
        pass

class DatasetEvaluators(DatasetEvaluator):
    def __init__(self, evaluators):
        assert len(evaluators)
        super().__init__()
        self._evaluators = evaluators

    def reset(self):
        for evaluator in self._evaluators:
            evaluator.reset()

    def process(self, input, output):
        for evaluator in self._evaluators:
            evaluator.process(input, output)

    def evaluate(self):
        results = OrderedDict()
        for evaluator in self._evaluators:
            result = evaluator.evaluate()
            if is_main_process() and result is not None:
                for k, v in result.items():
                    assert (
                        k not in results
                    ), "Different evaluators produce results with the same key {}".format(k)
                    results[k] = v
        return results

def inference_on_dataset(model, data_loader, evaluator):
    """
    Run model on the data_loader and evaluate the metrics with evaluator.
    Also benchmark the inference speed of `model.forward` accurately.
    The model will be used in eval mode.

    Args:
        model (nn.Module): a module which accepts an object from
            `data_loader` and returns some outputs. It will be temporarily set to `eval` mode.

            If you wish to evaluate a model in `training` mode instead, you can
            wrap the given model and override its behavior of `.eval()` and `.train()`.
        data_loader: an iterable object with a length.
            The elements it generates will be the inputs to the model.
        evaluator (DatasetEvaluator): the evaluator to run. Use `None` if you only want
            to benchmark, but don't want to do any evaluation.

    Returns:
        The return value of `evaluator.evaluate()`
    """
    num_devices = torch.distributed.get_world_size() if torch.distributed.is_initialized() else 1
    logger = logging.getLogger(__name__)
    logger.info("Start inference on {} images".format(len(data_loader)))

    total = len(data_loader)  # inference data loader must have a fixed length

    evaluator.reset()

    num_warmup = min(5, total - 1)
    start_time = time.perf_counter()
    total_compute_time = 0
    with inference_context(model), torch.no_grad():
        for idx, inputs in enumerate(data_loader):
            if idx == num_warmup:
                start_time = time.perf_counter()
                total_compute_time = 0

            start_compute_time = time.perf_counter()
            outputs = model(inputs)
            if torch.cuda.is_available():
                torch.cuda.synchronize()
            total_compute_time += time.perf_counter() - start_compute_time
            evaluator.process(inputs, outputs)

            iters_after_start = idx + 1 - num_warmup * int(idx >= num_warmup)
            seconds_per_img = total_compute_time / iters_after_start
            if idx >= num_warmup * 2 or seconds_per_img > 5:
                total_seconds_per_img = (time.perf_counter() - start_time) / iters_after_start
                eta = datetime.timedelta(seconds=int(total_seconds_per_img * (total - idx - 1)))
                log_every_n_seconds(
                    logging.INFO,
                    "Inference done {}/{}. {:.4f} s / img. ETA={}".format(
                        idx + 1, total, seconds_per_img, str(eta)
                    ),
                    n=5,
                )

    # Measure the time only for this worker (before the synchronization barrier)
    total_time = time.perf_counter() - start_time
    total_time_str = str(datetime.timedelta(seconds=total_time))
    # NOTE this format is parsed by grep
    logger.info(
        "Total inference time: {} ({:.6f} s / img per device, on {} devices)".format(
            total_time_str, total_time / (total - num_warmup), num_devices
        )
    )
    total_compute_time_str = str(datetime.timedelta(seconds=int(total_compute_time)))
    logger.info(
        "Total inference pure compute time: {} ({:.6f} s / img per device, on {} devices)".format(
            total_compute_time_str, total_compute_time / (total - num_warmup), num_devices
        )
    )

    results = evaluator.evaluate()
    # An evaluator may return None when not in main process.
    # Replace it by an empty dict instead to make it easier for downstream code to handle
    if results is None:
        results = {}
    return results

@contextmanager
def inference_context(model):
    """
    A context where the model is temporarily changed to eval mode,
    and restored to previous mode afterwards.

    Args:
        model: a torch Module
    """
    training_mode = model.training
    model.eval()
    yield
    model.train(training_mode)