y = super().predict(X, batch_size=batch_size)
        y[y > (2 * self.y_max_)] = 2 * self.y_max_
        if self.y_min_ < 0:
            y[y < (2 * self.y_min_)] = 2 * self.y_min_
        elif self.y_min_ > 0:
            y[y < (0.5 * self.y_min_)] = 0.5 * self.y_min_
        return y

import copy
from itertools import product
from typing import Optional, Union

import numpy as np
from sklearn.base import RegressorMixin

from ConfigSpace.forbidden import ForbiddenEqualsClause, ForbiddenAndConjunction

from autosklearn.pipeline.components.data_preprocessing import DataPreprocessorChoice

from ConfigSpace.configuration_space import ConfigurationSpace, Configuration
from autosklearn.pipeline.components import regression as \
    regression_components
from autosklearn.pipeline.components import feature_preprocessing as \
    feature_preprocessing_components
from autosklearn.pipeline.base import BasePipeline
from autosklearn.pipeline.constants import SPARSE

class SimpleRegressionPipeline(RegressorMixin, BasePipeline):
    """This class implements the regression task.

    It implements a pipeline, which includes one preprocessing step and one
    regression algorithm. It can render a search space including all known
    regression and preprocessing algorithms.

    Contrary to the sklearn API it is not possible to enumerate the
    possible parameters in the __init__ function because we only know the
    available regressors at runtime. For this reason the user must
    specifiy the parameters by passing an instance of
    ConfigSpace.configuration_space.Configuration.

    Parameters
    ----------
    config : ConfigSpace.configuration_space.Configuration
        The configuration to evaluate.

    random_state : Optional[int | RandomState]
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance
        used by `np.random`.

    Attributes
    ----------
    _estimator : The underlying scikit-learn regression model. This
        variable is assigned after a call to the
        :meth:`autosklearn.pipeline.regression.SimpleRegressionPipeline.fit`
        method.

    _preprocessor : The underlying scikit-learn preprocessing algorithm. This
        variable is only assigned if a preprocessor is specified and
        after a call to the
        :meth:`autosklearn.pipeline.regression.SimpleRegressionPipeline.fit`
        method.

    See also
    --------

    References
    ----------

    Examples
    --------

    """
    def __init__(
        self,
        config: Optional[Configuration] = None,
        steps=None,
        dataset_properties=None,
        include=None,
        exclude=None,
        random_state: Optional[Union[int, np.random.RandomState]] = None,
        init_params=None
    ):
        self._output_dtype = np.float32
        if dataset_properties is None:
            dataset_properties = dict()
        if 'target_type' not in dataset_properties:
            dataset_properties['target_type'] = 'regression'
        super().__init__(
            config=config, steps=steps,
            dataset_properties=dataset_properties,
            include=include, exclude=exclude, random_state=random_state,
            init_params=init_params)

    def fit_estimator(self, X, y, **fit_params):
        self.y_max_ = np.nanmax(y)
        self.y_min_ = np.nanmin(y)
        return super(SimpleRegressionPipeline, self).fit_estimator(
            X, y, **fit_params)

    def iterative_fit(self, X, y, n_iter=1, **fit_params):
        self.y_max_ = np.nanmax(y)
        self.y_min_ = np.nanmin(y)
        return super(SimpleRegressionPipeline, self).iterative_fit(
            X, y, n_iter=n_iter, **fit_params)

    def predict(self, X, batch_size=None):

    def _get_hyperparameter_search_space(self, include=None, exclude=None,
                                         dataset_properties=None):
        """Return the configuration space for the CASH problem.

        Parameters
        ----------
        include : dict
            If include is given, only the modules specified for nodes
            are used. Specify them by their module name; e.g., to include
            only the SVM use :python:`include={'regressor':['svr']}`.

        exclude : dict
            If exclude is given, only the components specified for nodes
            are used. Specify them by their module name; e.g., to include
            all regressors except the SVM use
            :python:`exclude=['regressor': 'svr']`.

        Returns
        -------
        cs : ConfigSpace.configuration_space.Configuration
            The configuration space describing the SimpleRegressionClassifier.
        """
        cs = ConfigurationSpace()

        if dataset_properties is None or not isinstance(dataset_properties, dict):
            dataset_properties = dict()
        if 'target_type' not in dataset_properties:
            dataset_properties['target_type'] = 'regression'
        if dataset_properties['target_type'] != 'regression':
            dataset_properties['target_type'] = 'regression'

        if 'sparse' not in dataset_properties:
            # This dataset is probably dense
            dataset_properties['sparse'] = False

        cs = self._get_base_search_space(
            cs=cs, dataset_properties=dataset_properties,
            exclude=exclude, include=include, pipeline=self.steps)

        regressors = cs.get_hyperparameter('regressor:__choice__').choices
        preprocessors = cs.get_hyperparameter('feature_preprocessor:__choice__').choices
        available_regressors = self._final_estimator.get_available_components(
            dataset_properties)

        possible_default_regressor = copy.copy(list(
            available_regressors.keys()))
        default = cs.get_hyperparameter('regressor:__choice__').default_value
        del possible_default_regressor[
            possible_default_regressor.index(default)]

        # A regressor which can handle sparse data after the densifier is
        for key in regressors:
            if SPARSE in available_regressors[key].get_properties(dataset_properties=None)['input']:
                if 'densifier' in preprocessors:
                    while True:
                        try:
                            forb_reg = ForbiddenEqualsClause(
                                cs.get_hyperparameter('regressor:__choice__'), key)
                            forb_fpp = ForbiddenEqualsClause(cs.get_hyperparameter(
                                'feature_preprocessor:__choice__'), 'densifier')
                            cs.add_forbidden_clause(
                                ForbiddenAndConjunction(forb_reg, forb_fpp))
                            # Success
                            break
                        except ValueError:
                            # Change the default and try again
                            try:
                                default = possible_default_regressor.pop()
                            except IndexError:
                                raise ValueError(
                                    "Cannot find a legal default configuration.")
                            cs.get_hyperparameter(
                                'regressor:__choice__').default_value = default

        # which would take too long
        # Combinations of tree-based models with feature learning:
        regressors_ = [
            "adaboost", "ard_regression", "decision_tree",
            "extra_trees", "gaussian_process", "gradient_boosting",
            "k_nearest_neighbors", "libsvm_svr", "mlp", "random_forest"
        ]
        feature_learning_ = ["kitchen_sinks", "kernel_pca", "nystroem_sampler"]

        for r, f in product(regressors_, feature_learning_):
            if r not in regressors:
                continue
            if f not in preprocessors:
                continue
            while True:
                try:
                    cs.add_forbidden_clause(ForbiddenAndConjunction(
                        ForbiddenEqualsClause(cs.get_hyperparameter(
                            "regressor:__choice__"), r),
                        ForbiddenEqualsClause(cs.get_hyperparameter(
                            "feature_preprocessor:__choice__"), f)))
                    break
                except KeyError:
                    break
                except ValueError:
                    # Change the default and try again
                    try:
                        default = possible_default_regressor.pop()
                    except IndexError:
                        raise ValueError(
                            "Cannot find a legal default configuration.")
                    cs.get_hyperparameter(
                        'regressor:__choice__').default_value = default

        self.configuration_space = cs
        self.dataset_properties = dataset_properties
        return cs

    def _get_estimator_components(self):
        return regression_components._regressors

    def _get_pipeline_steps(self, dataset_properties, init_params=None):
        steps = []

        default_dataset_properties = {'target_type': 'regression'}
        if dataset_properties is not None and isinstance(dataset_properties, dict):
            default_dataset_properties.update(dataset_properties)

        steps.extend([
            [
                'data_preprocessor',
                DataPreprocessorChoice(
                    dataset_properties=default_dataset_properties,
                    random_state=self.random_state
                )
            ],
            [
                'feature_preprocessor',
                feature_preprocessing_components.FeaturePreprocessorChoice(
                    dataset_properties=default_dataset_properties,
                    random_state=self.random_state
                )
            ],
            [
                'regressor',
                regression_components.RegressorChoice(
                    default_dataset_properties,
                    random_state=self.random_state
                )
            ]
        ])

        return steps

    def _get_estimator_hyperparameter_name(self):
        return "regressor"


import copy
from itertools import product
from typing import Optional, Union

import numpy as np
from sklearn.base import RegressorMixin

from ConfigSpace.forbidden import ForbiddenEqualsClause, ForbiddenAndConjunction

from autosklearn.pipeline.components.data_preprocessing import DataPreprocessorChoice

from ConfigSpace.configuration_space import ConfigurationSpace, Configuration
from autosklearn.pipeline.components import regression as \
    regression_components
from autosklearn.pipeline.components import feature_preprocessing as \
    feature_preprocessing_components
from autosklearn.pipeline.base import BasePipeline
from autosklearn.pipeline.constants import SPARSE

class SimpleRegressionPipeline(RegressorMixin, BasePipeline):
    """This class implements the regression task.

    It implements a pipeline, which includes one preprocessing step and one
    regression algorithm. It can render a search space including all known
    regression and preprocessing algorithms.

    Contrary to the sklearn API it is not possible to enumerate the
    possible parameters in the __init__ function because we only know the
    available regressors at runtime. For this reason the user must
    specifiy the parameters by passing an instance of
    ConfigSpace.configuration_space.Configuration.

    Parameters
    ----------
    config : ConfigSpace.configuration_space.Configuration
        The configuration to evaluate.

    random_state : Optional[int | RandomState]
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance
        used by `np.random`.

    Attributes
    ----------
    _estimator : The underlying scikit-learn regression model. This
        variable is assigned after a call to the
        :meth:`autosklearn.pipeline.regression.SimpleRegressionPipeline.fit`
        method.

    _preprocessor : The underlying scikit-learn preprocessing algorithm. This
        variable is only assigned if a preprocessor is specified and
        after a call to the
        :meth:`autosklearn.pipeline.regression.SimpleRegressionPipeline.fit`
        method.

    See also
    --------

    References
    ----------

    Examples
    --------

    """
    def __init__(
        self,
        config: Optional[Configuration] = None,
        steps=None,
        dataset_properties=None,
        include=None,
        exclude=None,
        random_state: Optional[Union[int, np.random.RandomState]] = None,
        init_params=None
    ):
        self._output_dtype = np.float32
        if dataset_properties is None:
            dataset_properties = dict()
        if 'target_type' not in dataset_properties:
            dataset_properties['target_type'] = 'regression'
        super().__init__(
            config=config, steps=steps,
            dataset_properties=dataset_properties,
            include=include, exclude=exclude, random_state=random_state,
            init_params=init_params)

    def fit_estimator(self, X, y, **fit_params):
        self.y_max_ = np.nanmax(y)
        self.y_min_ = np.nanmin(y)
        return super(SimpleRegressionPipeline, self).fit_estimator(
            X, y, **fit_params)

    def iterative_fit(self, X, y, n_iter=1, **fit_params):
        self.y_max_ = np.nanmax(y)
        self.y_min_ = np.nanmin(y)
        return super(SimpleRegressionPipeline, self).iterative_fit(
            X, y, n_iter=n_iter, **fit_params)

    def predict(self, X, batch_size=None):
        y = super().predict(X, batch_size=batch_size)
        y[y > (2 * self.y_max_)] = 2 * self.y_max_
        if self.y_min_ < 0:
            y[y < (2 * self.y_min_)] = 2 * self.y_min_
        elif self.y_min_ > 0:
            y[y < (0.5 * self.y_min_)] = 0.5 * self.y_min_
        return y

    def _get_hyperparameter_search_space(self, include=None, exclude=None,
                                         dataset_properties=None):
        """Return the configuration space for the CASH problem.

        Parameters
        ----------
        include : dict
            If include is given, only the modules specified for nodes
            are used. Specify them by their module name; e.g., to include
            only the SVM use :python:`include={'regressor':['svr']}`.

        exclude : dict
            If exclude is given, only the components specified for nodes
            are used. Specify them by their module name; e.g., to include
            all regressors except the SVM use
            :python:`exclude=['regressor': 'svr']`.

        Returns
        -------
        cs : ConfigSpace.configuration_space.Configuration
            The configuration space describing the SimpleRegressionClassifier.
        """
        cs = ConfigurationSpa

        if dataset_properties is None or not isinstance(dataset_properties, dict):
            dataset_properties = dict()
        if 'target_type' not in dataset_properties:
            dataset_properties['target_type'] = 'regression'
        if dataset_properties['target_type'] != 'regression':
            dataset_properties['target_type'] = 'regression'

        if 'sparse' not in dataset_properties:
            # This dataset is probably dense
            dataset_properties['sparse'] = False

        cs = self._get_base_search_space(
            cs=cs, dataset_properties=dataset_properties,
            exclude=exclude, include=include, pipeline=self.steps)

        regressors = cs.get_hyperparameter('regressor:__choice__').choices
        preprocessors = cs.get_hyperparameter('feature_preprocessor:__choice__').choices
        available_regressors = self._final_estimator.get_available_components(
            dataset_properties)

        possible_default_regressor = copy.copy(list(
            available_regressors.keys()))
        default = cs.get_hyperparameter('regressor:__choice__').default_value
        del possible_default_regressor[
            possible_default_regressor.index(default)]

        # A regressor which can handle sparse data after the densifier is
        for key in regressors:
            if SPARSE in available_regressors[key].get_properties(dataset_properties=None)['input']:
                if 'densifier' in preprocessors:
                    while True:
                        try:
                            forb_reg = ForbiddenEqualsClause(
                                cs.get_hyperparameter('regressor:__choice__'), key)
                            forb_fpp = ForbiddenEqualsClause(cs.get_hyperparameter(
                                'feature_preprocessor:__choice__'), 'densifier')
                            cs.add_forbidden_clause(
                                ForbiddenAndConjunction(forb_reg, forb_fpp))
                            # Success
                            break
                        except ValueError:
                            # Change the default and try again
                            try:
                                default = possible_default_regressor.pop()
                            except IndexError:
                                raise ValueError(
                                    "Cannot find a legal default configuration.")
                            cs.get_hyperparameter(
                                'regressor:__choice__').default_value = default

        # which would take too long
        # Combinations of tree-based models with feature learning:
        regressors_ = [
            "adaboost", "ard_regression", "decision_tree",
            "extra_trees", "gaussian_process", "gradient_boosting",
            "k_nearest_neighbors", "libsvm_svr", "mlp", "random_forest"
        ]
        feature_learning_ = ["kitchen_sinks", "kernel_pca", "nystroem_sampler"]

        for r, f in product(regressors_, feature_learning_):
            if r not in regressors:
                continue
            if f not in preprocessors:
                continue
            while True:
                try:
                    cs.add_forbidden_clause(ForbiddenAndConjunction(
                        ForbiddenEqualsClause(cs.get_hyperparameter(
                            "regressor:__choice__"), r),
                        ForbiddenEqualsClause(cs.get_hyperparameter(
                            "feature_preprocessor:__choice__"), f)))
                    break
                except KeyError:
                    break
                except ValueError:
                    # Change the default and try again
                    try:
                        default = possible_default_regressor.pop()
                    except IndexError:
                        raise ValueError(
                            "Cannot find a legal default configuration.")
                    cs.get_hyperparameter(
                        'regressor:__choice__').default_value = default

        self.configuration_space = cs
        self.dataset_properties = dataset_properties
        return cs

    def _get_estimator_components(self):
        return regression_components._regressors

    def _get_pipeline_steps(self, dataset_properties, init_params=None):
        steps = []

        default_dataset_properties = {'target_type': 'regression'}
        if dataset_properties is not None and isinstance(dataset_properties, dict):
            default_dataset_properties.update(dataset_properties)

        steps.extend([
            [
                'data_preprocessor',
                DataPreprocessorChoice(
                    dataset_properties=default_dataset_properties,
                    random_state=self.random_state
                )
            ],
            [
                'feature_preprocessor',
                feature_preprocessing_components.FeaturePreprocessorChoice(
                    dataset_properties=default_dataset_properties,
                    random_state=self.random_state
                )
            ],
            [
                'regressor',
                regression_components.RegressorChoice(
                    default_dataset_properties,
                    random_state=self.random_state
                )
            ]
        ])

        return steps

    def _get_estimator_hyperparameter_name(self):
        return "regressor"


import copy
from itertools import product
from typing import Optional, Union

import numpy as np
from sklearn.base import RegressorMixin

from ConfigSpace.forbidden import ForbiddenEqualsClause, ForbiddenAndConjunction

from autosklearn.pipeline.components.data_preprocessing import DataPreprocessorChoice

from ConfigSpace.configuration_space import ConfigurationSpace, Configuration
from autosklearn.pipeline.components import regression as \
    regression_components
from autosklearn.pipeline.components import feature_preprocessing as \
    feature_preprocessing_components
from autosklearn.pipeline.base import BasePipeline
from autosklearn.pipeline.constants import SPARSE

class SimpleRegressionPipeline(RegressorMixin, BasePipeline):
    """This class implements the regression task.

    It implements a pipeline, which includes one preprocessing step and one
    regression algorithm. It can render a search space including all known
    regression and preprocessing algorithms.

    Contrary to the sklearn API it is not possible to enumerate the
    possible parameters in the __init__ function because we only know the
    available regressors at runtime. For this reason the user must
    specifiy the parameters by passing an instance of
    ConfigSpace.configuration_space.Configuration.

    Parameters
    ----------
    config : ConfigSpace.configuration_space.Configuration
        The configuration to evaluate.

    random_state : Optional[int | RandomState]
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance
        used by `np.random`.

    Attributes
    ----------
    _estimator : The underlying scikit-learn regression model. This
        variable is assigned after a call to the
        :meth:`autosklearn.pipeline.regression.SimpleRegressionPipeline.fit`
        method.

    _preprocessor : The underlying scikit-learn preprocessing algorithm. This
        variable is only assigned if a preprocessor is specified and
        after a call to the
        :meth:`autosklearn.pipeline.regression.SimpleRegressionPipeline.fit`
        method.

    See also
    --------

    References
    ----------

    Examples
    --------

    """
    def __init__(
        self,
        config: Optional[Configuration] = None,
        steps=None,
        dataset_properties=None,
        include=None,
        exclude=None,
        random_state: Optional[Union[int, np.random.RandomState]] = None,
        init_params=None
    ):
        self._output_dtype = np.float32
        if dataset_properties is None:
            dataset_properties = dict()
        if 'target_type' not in dataset_properties:
            dataset_properties['target_type'] = 'regression'
        super().__init__(
            config=config, steps=steps,
            dataset_properties=dataset_properties,
            include=include, exclude=exclude, random_state=random_state,
            init_params=init_params)

    def fit_estimator(self, X, y, **fit_params):
        self.y_max_ = np.nanmax(y)
        self.y_min_ = np.nanmin(y)
        return super(SimpleRegressionPipeline, self).fit_estimator(
            X, y, **fit_params)

    def iterative_fit(self, X, y, n_iter=1, **fit_params):
        self.y_max_ = np.nanmax(y)
        self.y_min_ = np.nanmin(y)
        return super(SimpleRegressionPipeline, self).iterative_fit(
            X, y, n_iter=n_iter, **fit_params)

    def predict(self, X, batch_size=None):
        y = super().predict(X, batch_size=batch_size)
        y[y > (2 * self.y_max_)] = 2 * self.y_max_
        if self.y_min_ < 0:
            y[y < (2 * self.y_min_)] = 2 * self.y_min_
        elif self.y_min_ > 0:
            y[y < (0.5 * self.y_min_)] = 0.5 * self.y_min_
        return y

    def _get_hyperparameter_search_space(self, include=None, exclude=None,
                                         dataset_properties=None):
        """Return the configuration space for the CASH problem.

        Parameters
        ----------
        include : dict
            If include is given, only the modules specified for nodes
            are used. Specify them by their module name; e.g., to include
            only the SVM use :python:`include={'regressor':['svr']}`.

        exclude : dict
            If exclude is given, only the components specified for nodes
            are used. Specify them by their module name; e.g., to include
            all regressors except the SVM use
            :python:`exclude=['regressor': 'svr']`.

        Returns
        -------
        cs : ConfigSpace.configuration_space.Configuration
            The configuration space describing the SimpleRegressionClassifier.
        """
        cs = ConfigurationSpace()

        if dataset_properties is None or not isinstance(dataset_properties, dict):
            dataset_properties = dict()
        if 'target_type' not in dataset_properties:
            dataset_properties['target_type'] = 'regression'
        if dataset_properties['target_type'] != 'regression':
            dataset_properties['target_type'] = 'regression'

        if 'sparse' not in dataset_properties:
            # This dataset is probably dense
            dataset_properties['sparse'] = False

        cs = self._get_base_search_space(
            cs=cs, dataset_properties=dataset_properties,
            exclude=exclude, include=include, pipeline=self.steps)

        regressors = cs.get_hyperparameter('regressor:__choice__').choices
        preprocessors = cs.get_hyperparameter('feature_preprocessor:__choice__').choices
        available_regressors = self._final_estimator.get_available_components(
            dataset_properties)

        possible_default_regressor = copy.copy(list(
            available_regressors.keys()))
        default = cs.get_hyperparameter('regressor:__choice__').default_value
        del possible_default_regressor[
            possible_default_regressor.index(default)]

        # A regressor which can handle sparse data after the densifier is
        for key in regressors:
            if SPARSE in available_regressors[key].get_properties(dataset_properties=None)['input']:
                if 'densifier' in preprocessors:
                    while True:
                        try:
                            forb_reg = ForbiddenEqualsClause(
                                cs.get_hyperparameter('regressor:__choice__'), key)
                            forb_fpp = ForbiddenEqualsClause(cs.get_hyperparameter(
                                'feature_preprocessor:__choice__'), 'densifier')
                            cs.add_forbidden_clause(
                                ForbiddenAndConjunction(forb_reg, forb_fpp))
                            # Success
                            break
                        except ValueError:
                            # Change the default and try again
                            try:
                                default = possible_default_regressor.pop()
                            except IndexError:
                                raise ValueError(
                                    "Cannot find a legal default configuration.")
                            cs.get_hyperparameter(
                                'regressor:__choice__').default_value = default

        # which would take too long
        # Combinations of tree-based models with feature learning:
        regressors_ = [
            "adaboost", "ard_regression", "decision_tree",
            "extra_trees", "gaussian_process", "gradient_boosting",
            "k_nearest_neighbors", "libsvm_svr", "mlp", "random_forest"
        ]
        feature_learning_ = ["kitchen_sinks", "kernel_pca", "nystroem_sampler"]

        for r, f in product(regressors_, feature_learning_):
            if r not in regressors:
                continue
            if f not in preprocessors:
                continue
            while True:
                try:
                    cs.add_forbidden_clause(ForbiddenAndConjunction(
                        ForbiddenEqualsClause(cs.get_hyperparameter(
                            "regressor:__choice__"), r),
                        ForbiddenEqualsClause(cs.get_hyperparameter(
                            "feature_preprocessor:__choice__"), f)))
                    break
                except KeyError:
                    break
                except ValueError:
                    # Change the default and try again

                    cs.get_hyperparameter(
                        'regressor:__choice__').default_value = default

        self.configuration_space = cs
        self.dataset_properties = dataset_properties
        return cs

    def _get_estimator_components(self):
        return regression_components._regressors

    def _get_pipeline_steps(self, dataset_properties, init_params=None):
        steps = []

        default_dataset_properties = {'target_type': 'regression'}
        if dataset_properties is not None and isinstance(dataset_properties, dict):
            default_dataset_properties.update(dataset_properties)

        steps.extend([
            [
                'data_preprocessor',
                DataPreprocessorChoice(
                    dataset_properties=default_dataset_properties,
                    random_state=self.random_state
                )
            ],
            [
                'feature_preprocessor',
                feature_preprocessing_components.FeaturePreprocessorChoice(
                    dataset_properties=default_dataset_properties,
                    random_state=self.random_state
                )
            ],
            [
                'regressor',
                regression_components.RegressorChoice(
                    default_dataset_properties,
                    random_state=self.random_state
                )
            ]
        ])

        return steps

    def _get_estimator_hyperparameter_name(self):
        return "regressor"


def apply_gradients(self, *args, **kwargs):

# File: optimizer.py

from contextlib import contextmanager
import tensorflow as tf

from ..tfutils.common import get_tf_version_tuple
from ..compat import tfv1
from ..utils.develop import HIDE_DOC
from .gradproc import FilterNoneGrad, GradientProcessor

__all__ = ['apply_grad_processors', 'ProxyOptimizer',
           'PostProcessOptimizer', 'VariableAssignmentOptimizer',
           'AccumGradOptimizer']

class ProxyOptimizer(tfv1.train.Optimizer):
    """
    A transparent proxy which delegates all methods of :class:`tf.train.Optimizer`
    """
    def __init__(self, opt, name='ProxyOptimizer'):
        assert isinstance(opt, tfv1.train.Optimizer), opt
        super(ProxyOptimizer, self).__init__(False, name)
        self._opt = opt

    @HIDE_DOC
    def compute_gradients(self, *args, **kwargs):
        return self._opt.compute_gradients(*args, **kwargs)

    @HIDE_DOC
    def get_slot(self, *args, **kwargs):
        return self._opt.get_slot(*args, **kwargs)

    @HIDE_DOC
    def get_slot_names(self, *args, **kwargs):
        return self._opt.get_slot_names(*args, **kwargs)

    @HIDE_DOC

return self._opt.apply_gradients(*args, **kwargs)

def apply_grad_processors(opt, gradprocs):
    """
    Wrapper around optimizers to apply gradient processors.

    Args:
        opt (tf.train.Optimizer):
        gradprocs (list[GradientProcessor]): gradient processors to add to the
            optimizer.

    Returns:
        a :class:`tf.train.Optimizer` instance which runs the gradient
        processors before updating the variables.
    """
    assert isinstance(gradprocs, (list, tuple)), gradprocs
    for gp in gradprocs:
        assert isinstance(gp, GradientProcessor), gp

    class _ApplyGradientProcessor(ProxyOptimizer):
        def __init__(self, opt, gradprocs):
            self._gradprocs = gradprocs[:]
            super(_ApplyGradientProcessor, self).__init__(opt)

        def apply_gradients(self, grads_and_vars,
                            global_step=None, name=None):
            g = self._apply(grads_and_vars)
            return self._opt.apply_gradients(g, global_step, name)

        def _apply(self, g):
            for proc in self._gradprocs:
                g = proc.process(g)
            return g

    return _ApplyGradientProcessor(opt, gradprocs)

class PostProcessOptimizer(ProxyOptimizer):
    """
    An optimizer which applies some "post-processing operation" per variable
    (e.g. clipping, quantization) after the gradient update.
    """
    def __init__(self, opt, func, colocate=True):
        """
        Args:
            opt (tf.train.Optimizer):
            func (tf.Variable -> tf.Operation or None): the operation needed
                to perform for this variable after the gradient update.
            colocate (boolean): colocate the function with the variable. No effect since TF 1.13.
        """
        super(PostProcessOptimizer, self).__init__(opt)
        self._func = func
        self._colocate = colocate

    @HIDE_DOC
    def apply_gradients(self, grads_and_vars, global_step=None, name=None):
        update_op = super(PostProcessOptimizer, self).apply_gradients(
            grads_and_vars, global_step)
        ops = []
        with tf.control_dependencies([update_op]):
            for _, var in grads_and_vars:
                with self._maybe_colocate(var):
                    op = self._func(var)
                    if op is not None:
                        assert isinstance(op, tf.Operation), op
                        ops.append(op)
        update_op = tf.group(update_op, *ops, name=name)
        return update_op

    @contextmanager
    def _maybe_colocate(self, var):
        G = tf.get_default_graph()
        if self._colocate and get_tf_version_tuple() <= (1, 12):
            with G.colocate_with(var):
                yield
        else:
            yield

class VariableAssignmentOptimizer(PostProcessOptimizer):
    """
    An optimizer which assigns each variable a new value (e.g. clipping,
    quantization) after the gradient update.
    """
    def __init__(self, opt, func):
        """
        Args:
            opt (tf.train.Optimizer):
            func (tf.Variable -> tf.Tensor or None): the new value to be
                assigned to this variable after the gradient update.
        """
        def f(v):
            t = func(v)
            if t is None:
                return t
            return tf.assign(v, t, use_locking=False).op
        super(VariableAssignmentOptimizer, self).__init__(opt, f)

class AccumGradOptimizer(ProxyOptimizer):
    """
    An optimizer which accumulates gradients across :math:`k` :meth:`minimize` executions,
    and apply them together in every :math:`k` th :meth:`minimize` execution.
    This is roughly the same as using a :math:`k` times larger batch size plus a
    :math:`k` times larger learning rate, but uses much less memory.

    Note that this implementation may not support all models.
    E.g., it doesn't support sparse gradient update.
    """

    def __init__(self, opt, niter):
        """
        Args:
            opt (tf.train.Optimizer): the underlying sub-optimizer.
            niter (int): number of iterations to accumulate gradients.
        """
        super(AccumGradOptimizer, self).__init__(opt, 'AccumGrad')
        self._niter = int(niter)

    def _create_accum_slots(self, var_list):
        slots = []
        for v in var_list:
            # TODO an option to not colocate the accumulators with variables (to save more memory)
            s = self._zeros_slot(v, "accum", self._name)
            slots.append(s)
        return slots

    @HIDE_DOC
    def apply_gradients(self, grads_and_vars, global_step=None, name=None):
        assert global_step is None, \
            "AccumGradOptimizer doesn't support the option global_step! " \
            "Please maintain it yourself."
        grads_and_vars = FilterNoneGrad().process(grads_and_vars)
        vs = []
        for g, v in grads_and_vars:
            assert isinstance(g, tf.Tensor) and isinstance(v, tf.Variable), \
                "AccumGradOptimizer only works for dense update! " \
                "Types of v and g are {} and {}".format(type(v), type(g))
            vs.append(v)

        with tf.control_dependencies(None):
            slots = self._create_accum_slots(vs)
            slots_and_vars = [(s, gv[1]) for s, gv in zip(slots, grads_and_vars)]

            # Create the counter on the same device as the first variable.
            with tf.variable_scope(self._name), \
                    vs[0].graph.colocate_with(vs[0]):
                counter = tf.Variable(
                    0, name="counter", trainable=False, dtype=tf.int32)

        with tf.name_scope('AccumGradOptimizer'):
            ops = []
            for s, gv in zip(slots, grads_and_vars):
                g, v = gv
                ops.append(s.assign_add(g))
            update_counter = tf.assign_add(counter, 1, name='update_counter')
            update_slot_op = tf.group(update_counter, *ops, name='update_slot')

            def update_grad():
                update_op = self._opt.apply_gradients(slots_and_vars)
                with tf.control_dependencies([update_op]):
                    clear_ops = [tf.assign(s, tf.zeros_like(s)) for s in slots]
                return tf.group(*clear_ops, name='update_grad')

            pred = tf.equal(tf.mod(counter, self._niter), 0)
            with tf.control_dependencies([update_slot_op]):
                if name is None:
                    name = 'cond_update_grad'
                op = tf.cond(pred, update_grad, tf.no_op, name=name).op
        return op

if __name__ == '__main__':
    # run it with "python -m tensorpack.tfutils.optimizer"

    x = tf.get_variable('x', shape=[6])
    cost = tf.reduce_sum(tf.abs(x), name='cost')
    opt = tf.train.GradientDescentOptimizer(0.01)
    opt = AccumGradOptimizer(opt, 5)
    min_op = opt.minimize(cost)

    sess = tf.Session()
    sess.run(tf.global_variables_initializer())
    with sess.as_default():
        for k in range(20):
            min_op.run()
            print(x.eval())


for proc in self._gradprocs:
                g = proc.process(g)
            return g

# File: optimizer.py

from contextlib import contextmanager
import tensorflow as tf

from ..tfutils.common import get_tf_version_tuple
from ..compat import tfv1
from ..utils.develop import HIDE_DOC
from .gradproc import FilterNoneGrad, GradientProcessor

__all__ = ['apply_grad_processors', 'ProxyOptimizer',
           'PostProcessOptimizer', 'VariableAssignmentOptimizer',
           'AccumGradOptimizer']

class ProxyOptimizer(tfv1.train.Optimizer):
    """
    A transparent proxy which delegates all methods of :class:`tf.train.Optimizer`
    """
    def __init__(self, opt, name='ProxyOptimizer'):
        assert isinstance(opt, tfv1.train.Optimizer), opt
        super(ProxyOptimizer, self).__init__(False, name)
        self._opt = opt

    @HIDE_DOC
    def compute_gradients(self, *args, **kwargs):
        return self._opt.compute_gradients(*args, **kwargs)

    @HIDE_DOC
    def get_slot(self, *args, **kwargs):
        return self._opt.get_slot(*args, **kwargs)

    @HIDE_DOC
    def get_slot_names(self, *args, **kwargs):
        return self._opt.get_slot_names(*args, **kwargs)

    @HIDE_DOC
    def apply_gradients(self, *args, **kwargs):
        return self._opt.apply_gradients(*args, **kwargs)

def apply_grad_processors(opt, gradprocs):
    """
    Wrapper around optimizers to apply gradient processors.

    Args:
        opt (tf.train.Optimizer):
        gradprocs (list[GradientProcessor]): gradient processors to add to the
            optimizer.

    Returns:
        a :class:`tf.train.Optimizer` instance which runs the gradient
        processors before updating the variables.
    """
    assert isinstance(gradprocs, (list, tuple)), gradprocs
    for gp in gradprocs:
        assert isinstance(gp, GradientProcessor), gp

    class _ApplyGradientProcessor(ProxyOptimizer):
        def __init__(self, opt, gradprocs):
            self._gradprocs = gradprocs[:]
            super(_ApplyGradientProcessor, self).__init__(opt)

        def apply_gradients(self, grads_and_vars,
                            global_step=None, name=None):
            g = self._apply(grads_and_vars)
            return self._opt.apply_gradients(g, global_step, name)

        def _apply(self, g):

    return _ApplyGradientProcessor(opt, gradprocs)

class PostProcessOptimizer(ProxyOptimizer):
    """
    An optimizer which applies some "post-processing operation" per variable
    (e.g. clipping, quantization) after the gradient update.
    """
    def __init__(self, opt, func, colocate=True):
        """
        Args:
            opt (tf.train.Optimizer):
            func (tf.Variable -> tf.Operation or None): the operation needed
                to perform for this variable after the gradient update.
            colocate (boolean): colocate the function with the variable. No effect since TF 1.13.
        """
        super(PostProcessOptimizer, self).__init__(opt)
        self._func = func
        self._colocate = colocate

    @HIDE_DOC
    def apply_gradients(self, grads_and_vars, global_step=None, name=None):
        update_op = super(PostProcessOptimizer, self).apply_gradients(
            grads_and_vars, global_step)
        ops = []
        with tf.control_dependencies([update_op]):
            for _, var in grads_and_vars:
                with self._maybe_colocate(var):
                    op = self._func(var)
                    if op is not None:
                        assert isinstance(op, tf.Operation), op
                        ops.append(op)
        update_op = tf.group(update_op, *ops, name=name)
        return update_op

    @contextmanager
    def _maybe_colocate(self, var):
        G = tf.get_default_graph()
        if self._colocate and get_tf_version_tuple() <= (1, 12):
            with G.colocate_with(var):
                yield
        else:
            yield

class VariableAssignmentOptimizer(PostProcessOptimizer):
    """
    An optimizer which assigns each variable a new value (e.g. clipping,
    quantization) after the gradient update.
    """
    def __init__(self, opt, func):
        """
        Args:
            opt (tf.train.Optimizer):
            func (tf.Variable -> tf.Tensor or None): the new value to be
                assigned to this variable after the gradient update.
        """
        def f(v):
            t = func(v)
            if t is None:
                return t
            return tf.assign(v, t, use_locking=False).op
        super(VariableAssignmentOptimizer, self).__init__(opt, f)

class AccumGradOptimizer(ProxyOptimizer):
    """
    An optimizer which accumulates gradients across :math:`k` :meth:`minimize` executions,
    and apply them together in every :math:`k` th :meth:`minimize` execution.
    This is roughly the same as using a :math:`k` times larger batch size plus a
    :math:`k` times larger learning rate, but uses much less memory.

    Note that this implementation may not support all models.
    E.g., it doesn't support sparse gradient update.
    """

    def __init__(self, opt, niter):
        """
        Args:
            opt (tf.train.Optimizer): the underlying sub-optimizer.
            niter (int): number of iterations to accumulate gradients.
        """
        super(AccumGradOptimizer, self).__init__(opt, 'AccumGrad')
        self._niter = int(niter)

    def _create_accum_slots(self, var_list):
        slots = []
        for v in var_list:
            # TODO an option to not colocate the accumulators with variables (to save more memory)
            s = self._zeros_slot(v, "accum", self._name)
            slots.append(s)
        return slots

    @HIDE_DOC
    def apply_gradients(self, grads_and_vars, global_step=None, name=None):
        assert global_step is None, \
            "AccumGradOptimizer doesn't support the option global_step! " \
            "Please maintain it yourself."
        grads_and_vars = FilterNoneGrad().process(grads_and_vars)
        vs = []
        for g, v in grads_and_vars:
            assert isinstance(g, tf.Tensor) and isinstance(v, tf.Variable), \
                "AccumGradOptimizer only works for dense update! " \
                "Types of v and g are {} and {}".format(type(v), type(g))
            vs.append(v)

        with tf.control_dependencies(None):
            slots = self._create_accum_slots(vs)
            slots_and_vars = [(s, gv[1]) for s, gv in zip(slots, grads_and_vars)]

            # Create the counter on the same device as the first variable.
            with tf.variable_scope(self._name), \
                    vs[0].graph.colocate_with(vs[0]):
                counter = tf.Variable(
                    0, name="counter", trainable=False, dtype=tf.int32)

        with tf.name_scope('AccumGradOptimizer'):
            ops = []
            for s, gv in zip(slots, grads_and_vars):
                g, v = gv
                ops.append(s.assign_add(g))
            update_counter = tf.assign_add(counter, 1, name='update_counter')
            update_slot_op = tf.group(update_counter, *ops, name='update_slot')

            def update_grad():
                update_op = self._opt.apply_gradients(slots_and_vars)
                with tf.control_dependencies([update_op]):
                    clear_ops = [tf.assign(s, tf.zeros_like(s)) for s in slots]
                return tf.group(*clear_ops, name='update_grad')

            pred = tf.equal(tf.mod(counter, self._niter), 0)
            with tf.control_dependencies([update_slot_op]):
                if name is None:
                    name = 'cond_update_grad'
                op = tf.cond(pred, update_grad, tf.no_op, name=name).op
        return op

if __name__ == '__main__':
    # run it with "python -m tensorpack.tfutils.optimizer"

    x = tf.get_variable('x', shape=[6])
    cost = tf.reduce_sum(tf.abs(x), name='cost')
    opt = tf.train.GradientDescentOptimizer(0.01)
    opt = AccumGradOptimizer(opt, 5)
    min_op = opt.minimize(cost)

    sess = tf.Session()
    sess.run(tf.global_variables_initializer())
    with sess.as_default():
        for k in range(20):
            min_op.run()
            print(x.eval())


# File: optimizer.py

from contextlib import contextmanager
import tensorflow as tf

from ..tfutils.common import get_tf_version_tuple
from ..compat import tfv1
from ..utils.develop import HIDE_DOC
from .gradproc import FilterNoneGrad, GradientProcessor

__all__ = ['apply_grad_processors', 'ProxyOptimizer',
           'PostProcessOptimizer', 'VariableAssignmentOptimizer',
           'AccumGradOptimizer']

class ProxyOptimizer(tfv1.train.Optimizer):
    """
    A transparent proxy which delegates all methods of :class:`tf.train.Optimizer`
    """
    def __init__(self, opt, name='ProxyOptimizer'):
        assert isinstance(opt, tfv1.train.Optimizer), opt
        super(ProxyOptimizer, self).__init__(False, name)
        self._opt = opt

    @HIDE_DOC
    def compute_gradients(self, *args, **kwargs):
        return self._opt.compute_gradients(*args, **kwargs)

    @HIDE_DOC
    def get_slot(self, *args, **kwargs):
        return self._opt.get_slot(*args, **kwargs)

    @HIDE_DOC
    def get_slot_names(self, *args, **kwargs):
        return self._opt.get_slot_names(*args, **kwargs)

    @HIDE_DOC
    def apply_gradients(self, *args, **kwargs):
        return self._opt.apply_gradients(*args, **kwargs)

def apply_grad_processors(opt, gradprocs):
    """
    Wrapper around optimizers to apply gradient processors.

    Args:
        opt (tf.train.Optimizer):
        gradprocs (list[GradientProcessor]): gradient processors to add to the
            optimizer.

    Returns:
        a :class:`tf.train.Optimizer` instance which runs the gradient
        processors before updating the variables.
    """
    assert isinstance(gradprocs, (list, tuple)), gradprocs
    for gp in gradprocs:
        assert isinstance(gp, GradientProcessor), gp

    class _ApplyGradientProcessor(ProxyOptimizer):
        def __init__(self, opt, gradprocs):
            self._gradprocs = gradprocs[:]
            super(_ApplyGradientProcessor, self).__init__(opt)

        def apply_gradients(self, grads_and_vars,
                            global_step=None, name=None):
            g = self._apply(grads_and_vars)
            return self._opt.apply_gradients(g, global_

        def _apply(self, g):
            for proc in self._gradprocs:
                g = proc.process(g)
            return g

    return _ApplyGradientProcessor(opt, gradprocs)

class PostProcessOptimizer(ProxyOptimizer):
    """
    An optimizer which applies some "post-processing operation" per variable
    (e.g. clipping, quantization) after the gradient update.
    """
    def __init__(self, opt, func, colocate=True):
        """
        Args:
            opt (tf.train.Optimizer):
            func (tf.Variable -> tf.Operation or None): the operation needed
                to perform for this variable after the gradient update.
            colocate (boolean): colocate the function with the variable. No effect since TF 1.13.
        """
        super(PostProcessOptimizer, self).__init__(opt)
        self._func = func
        self._colocate = colocate

    @HIDE_DOC
    def apply_gradients(self, grads_and_vars, global_step=None, name=None):
        update_op = super(PostProcessOptimizer, self).apply_gradients(
            grads_and_vars, global_step)
        ops = []
        with tf.control_dependencies([update_op]):
            for _, var in grads_and_vars:
                with self._maybe_colocate(var):
                    op = self._func(var)
                    if op is not None:
                        assert isinstance(op, tf.Operation), op
                        ops.append(op)
        update_op = tf.group(update_op, *ops, name=name)
        return update_op

    @contextmanager
    def _maybe_colocate(self, var):
        G = tf.get_default_graph()
        if self._colocate and get_tf_version_tuple() <= (1, 12):
            with G.colocate_with(var):
                yield
        else:
            yield

class VariableAssignmentOptimizer(PostProcessOptimizer):
    """
    An optimizer which assigns each variable a new value (e.g. clipping,
    quantization) after the gradient update.
    """
    def __init__(self, opt, func):
        """
        Args:
            opt (tf.train.Optimizer):
            func (tf.Variable -> tf.Tensor or None): the new value to be
                assigned to this variable after the gradient update.
        """
        def f(v):
            t = func(v)
            if t is None:
                return t
            return tf.assign(v, t, use_locking=False).op
        super(VariableAssignmentOptimizer, self).__init__(opt, f)

class AccumGradOptimizer(ProxyOptimizer):
    """
    An optimizer which accumulates gradients across :math:`k` :meth:`minimize` executions,
    and apply them together in every :math:`k` th :meth:`minimize` execution.
    This is roughly the same as using a :math:`k` times larger batch size plus a
    :math:`k` times larger learning rate, but uses much less memory.

    Note that this implementation may not support all models.
    E.g., it doesn't support sparse gradient update.
    """

    def __init__(self, opt, niter):
        """
        Args:
            opt (tf.train.Optimizer): the underlying sub-optimizer.
            niter (int): number of iterations to accumulate gradients.
        """
        super(AccumGradOptimizer, self).__init__(opt, 'AccumGrad')
        self._niter = int(niter)

    def _create_accum_slots(self, var_list):
        slots = []
        for v in var_list:
            # TODO an option to not colocate the accumulators with variables (to save more memory)
            s = self._zeros_slot(v, "accum", self._name)
            slots.append(s)
        return slots

    @HIDE_DOC
    def apply_gradients(self, grads_and_vars, global_step=None, name=None):
        assert global_step is None, \
            "AccumGradOptimizer doesn't support the option global_step! " \
            "Please maintain it yourself."
        grads_and_vars = FilterNoneGrad().process(grads_and_vars)
        vs = []
        for g, v in grads_and_vars:
            assert isinstance(g, tf.Tensor) and isinstance(v, tf.Variable), \
                "AccumGradOptimizer only works for dense update! " \
                "Types of v and g are {} and {}".format(type(v), type(g))
            vs.append(v)

        with tf.control_dependencies(None):
            slots = self._create_accum_slots(vs)
            slots_and_vars = [(s, gv[1]) for s, gv in zip(slots, grads_and_vars)]

            # Create the counter on the same device as the first variable.
            with tf.variable_scope(self._name), \
                    vs[0].graph.colocate_with(vs[0]):
                counter = tf.Variable(
                    0, name="counter", trainable=False, dtype=tf.int32)

        with tf.name_scope('AccumGradOptimizer'):
            ops = []
            for s, gv in zip(slots, grads_and_vars):
                g, v = gv
                ops.append(s.assign_add(g))
            update_counter = tf.assign_add(counter, 1, name='update_counter')
            update_slot_op = tf.group(update_counter, *ops, name='update_slot')

            def update_grad():
                update_op = self._opt.apply_gradients(slots_and_vars)
                with tf.control_dependencies([update_op]):
                    clear_ops = [tf.assign(s, tf.zeros_like(s)) for s in slots]
                return tf.group(*clear_ops, name='update_grad')

            pred = tf.equal(tf.mod(counter, self._niter), 0)
            with tf.control_dependencies([update_slot_op]):
                if name is None:
                    name = 'cond_update_grad'
                op = tf.cond(pred, update_grad, tf.no_op, name=name).op
        return op

if __name__ == '__main__':
    # run it with "python -m tensorpack.tfutils.optimizer"

    x = tf.get_variable('x', shape=[6])
    cost = tf.reduce_sum(tf.abs(x), name='cost')
    opt = tf.train.GradientDescentOptimizer(0.01)
    opt = AccumGradOptimizer(opt, 5)
    min_op = opt.minimize(cost)

    sess = tf.Session()
    sess.run(tf.global_variables_initializer())
    with sess.as_default():
        for k in range(20):
            min_op.run()
            print(x.eval())


        super(AccumGradOptimizer, self).__init__(opt, 'AccumGrad')
        self._niter = int(niter)

# File: optimizer.py

from contextlib import contextmanager
import tensorflow as tf

from ..tfutils.common import get_tf_version_tuple
from ..compat import tfv1
from ..utils.develop import HIDE_DOC
from .gradproc import FilterNoneGrad, GradientProcessor

__all__ = ['apply_grad_processors', 'ProxyOptimizer',
           'PostProcessOptimizer', 'VariableAssignmentOptimizer',
           'AccumGradOptimizer']

class ProxyOptimizer(tfv1.train.Optimizer):
    """
    A transparent proxy which delegates all methods of :class:`tf.train.Optimizer`
    """
    def __init__(self, opt, name='ProxyOptimizer'):
        assert isinstance(opt, tfv1.train.Optimizer), opt
        super(ProxyOptimizer, self).__init__(False, name)
        self._opt = opt

    @HIDE_DOC
    def compute_gradients(self, *args, **kwargs):
        return self._opt.compute_gradients(*args, **kwargs)

    @HIDE_DOC
    def get_slot(self, *args, **kwargs):
        return self._opt.get_slot(*args, **kwargs)

    @HIDE_DOC
    def get_slot_names(self, *args, **kwargs):
        return self._opt.get_slot_names(*args, **kwargs)

    @HIDE_DOC
    def apply_gradients(self, *args, **kwargs):
        return self._opt.apply_gradients(*args, **kwargs)

def apply_grad_processors(opt, gradprocs):
    """
    Wrapper around optimizers to apply gradient processors.

    Args:
        opt (tf.train.Optimizer):
        gradprocs (list[GradientProcessor]): gradient processors to add to the
            optimizer.

    Returns:
        a :class:`tf.train.Optimizer` instance which runs the gradient
        processors before updating the variables.
    """
    assert isinstance(gradprocs, (list, tuple)), gradprocs
    for gp in gradprocs:
        assert isinstance(gp, GradientProcessor), gp

    class _ApplyGradientProcessor(ProxyOptimizer):
        def __init__(self, opt, gradprocs):
            self._gradprocs = gradprocs[:]
            super(_ApplyGradientProcessor, self).__init__(opt)

        def apply_gradients(self, grads_and_vars,
                            global_step=None, name=None):
            g = self._apply(grads_and_vars)
            return self._opt.apply_gradients(g, global_step, name)

        def _apply(self, g):
            for proc in self._gradprocs:
                g = proc.process(g)
            return g

    return _ApplyGradientProcessor(opt, gradprocs)

class PostProcessOptimizer(ProxyOptimizer):
    """
    An optimizer which applies some "post-processing operation" per variable
    (e.g. clipping, quantization) after the gradient update.
    """
    def __init__(self, opt, func, colocate=True):
        """
        Args:
            opt (tf.train.Optimizer):
            func (tf.Variable -> tf.Operation or None): the operation needed
                to perform for this variable after the gradient update.
            colocate (boolean): colocate the function with the variable. No effect since TF 1.13.
        """
        super(PostProcessOptimizer, self).__init__(opt)
        self._func = func
        self._colocate = colocate

    @HIDE_DOC
    def apply_gradients(self, grads_and_vars, global_step=None, name=None):
        update_op = super(PostProcessOptimizer, self).apply_gradients(
            grads_and_vars, global_step)
        ops = []
        with tf.control_dependencies([update_op]):
            for _, var in grads_and_vars:
                with self._maybe_colocate(var):
                    op = self._func(var)
                    if op is not None:
                        assert isinstance(op, tf.Operation), op
                        ops.append(op)
        update_op = tf.group(update_op, *ops, name=name)
        return update_op

    @contextmanager
    def _maybe_colocate(self, var):
        G = tf.get_default_graph()
        if self._colocate and get_tf_version_tuple() <= (1, 12):
            with G.colocate_with(var):
                yield
        else:
            yield

class VariableAssignmentOptimizer(PostProcessOptimizer):
    """
    An optimizer which assigns each variable a new value (e.g. clipping,
    quantization) after the gradient update.
    """
    def __init__(self, opt, func):
        """
        Args:
            opt (tf.train.Optimizer):
            func (tf.Variable -> tf.Tensor or None): the new value to be
                assigned to this variable after the gradient update.
        """
        def f(v):
            t = func(v)
            if t is None:
                return t
            return tf.assign(v, t, use_locking=False).op
        super(VariableAssignmentOptimizer, self).__init__(opt, f)

class AccumGradOptimizer(ProxyOptimizer):
    """
    An optimizer which accumulates gradients across :math:`k` :meth:`minimize` executions,
    and apply them together in every :math:`k` th :meth:`minimize` execution.
    This is roughly the same as using a :math:`k` times larger batch size plus a
    :math:`k` times larger learning rate, but uses much less memory.

    Note that this implementation may not support all models.
    E.g., it doesn't support sparse gradient update.
    """

    def __init__(self, opt, niter):
        """
        Args:
            opt (tf.train.Optimizer): the underlying sub-optimizer.
            niter (int): number of iterations to accumulate gradients.
        """

    def _create_accum_slots(self, var_list):
        slots = []
        for v in var_list:
            # TODO an option to not colocate the accumulators with variables (to save more memory)
            s = self._zeros_slot(v, "accum", self._name)
            slots.append(s)
        return slots

    @HIDE_DOC
    def apply_gradients(self, grads_and_vars, global_step=None, name=None):
        assert global_step is None, \
            "AccumGradOptimizer doesn't support the option global_step! " \
            "Please maintain it yourself."
        grads_and_vars = FilterNoneGrad().process(grads_and_vars)
        vs = []
        for g, v in grads_and_vars:
            assert isinstance(g, tf.Tensor) and isinstance(v, tf.Variable), \
                "AccumGradOptimizer only works for dense update! " \
                "Types of v and g are {} and {}".format(type(v), type(g))
            vs.append(v)

        with tf.control_dependencies(None):
            slots = self._create_accum_slots(vs)
            slots_and_vars = [(s, gv[1]) for s, gv in zip(slots, grads_and_vars)]

            # Create the counter on the same device as the first variable.
            with tf.variable_scope(self._name), \
                    vs[0].graph.colocate_with(vs[0]):
                counter = tf.Variable(
                    0, name="counter", trainable=False, dtype=tf.int32)

        with tf.name_scope('AccumGradOptimizer'):
            ops = []
            for s, gv in zip(slots, grads_and_vars):
                g, v = gv
                ops.append(s.assign_add(g))
            update_counter = tf.assign_add(counter, 1, name='update_counter')
            update_slot_op = tf.group(update_counter, *ops, name='update_slot')

            def update_grad():
                update_op = self._opt.apply_gradients(slots_and_vars)
                with tf.control_dependencies([update_op]):
                    clear_ops = [tf.assign(s, tf.zeros_like(s)) for s in slots]
                return tf.group(*clear_ops, name='update_grad')

            pred = tf.equal(tf.mod(counter, self._niter), 0)
            with tf.control_dependencies([update_slot_op]):
                if name is None:
                    name = 'cond_update_grad'
                op = tf.cond(pred, update_grad, tf.no_op, name=name).op
        return op

if __name__ == '__main__':
    # run it with "python -m tensorpack.tfutils.optimizer"

    x = tf.get_variable('x', shape=[6])
    cost = tf.reduce_sum(tf.abs(x), name='cost')
    opt = tf.train.GradientDescentOptimizer(0.01)
    opt = AccumGradOptimizer(opt, 5)
    min_op = opt.minimize(cost)

    sess = tf.Session()
    sess.run(tf.global_variables_initializer())
    with sess.as_default():
        for k in range(20):
            min_op.run()
            print(x.eval())


for gp in gradprocs:
        assert isinstance(gp, GradientProcessor), gp

# File: optimizer.py

from contextlib import contextmanager
import tensorflow as tf

from ..tfutils.common import get_tf_version_tuple
from ..compat import tfv1
from ..utils.develop import HIDE_DOC
from .gradproc import FilterNoneGrad, GradientProcessor

__all__ = ['apply_grad_processors', 'ProxyOptimizer',
           'PostProcessOptimizer', 'VariableAssignmentOptimizer',
           'AccumGradOptimizer']

class ProxyOptimizer(tfv1.train.Optimizer):
    """
    A transparent proxy which delegates all methods of :class:`tf.train.Optimizer`
    """
    def __init__(self, opt, name='ProxyOptimizer'):
        assert isinstance(opt, tfv1.train.Optimizer), opt
        super(ProxyOptimizer, self).__init__(False, name)
        self._opt = opt

    @HIDE_DOC
    def compute_gradients(self, *args, **kwargs):
        return self._opt.compute_gradients(*args, **kwargs)

    @HIDE_DOC
    def get_slot(self, *args, **kwargs):
        return self._opt.get_slot(*args, **kwargs)

    @HIDE_DOC
    def get_slot_names(self, *args, **kwargs):
        return self._opt.get_slot_names(*args, **kwargs)

    @HIDE_DOC
    def apply_gradients(self, *args, **kwargs):
        return self._opt.apply_gradients(*args, **kwargs)

def apply_grad_processors(opt, gradprocs):
    """
    Wrapper around optimizers to apply gradient processors.

    Args:
        opt (tf.train.Optimizer):
        gradprocs (list[GradientProcessor]): gradient processors to add to the
            optimizer.

    Returns:
        a :class:`tf.train.Optimizer` instance which runs the gradient
        processors before updating the variables.
    """
    assert isinstance(gradprocs, (list, tuple)), gradprocs

    class _ApplyGradientProcessor(ProxyOptimizer):
        def __init__(self, opt, gradprocs):
            self._gradprocs = gradprocs[:]
            super(_ApplyGradientProcessor, self).__init__(opt)

        def apply_gradients(self, grads_and_vars,
                            global_step=None, name=None):
            g = self._apply(grads_and_vars)
            return self._opt.apply_gradients(g, global_step, name)

        def _apply(self, g):
            for proc in self._gradprocs:
                g = proc.process(g)
            return g

    return _ApplyGradientProcessor(opt, gradprocs)

class PostProcessOptimizer(ProxyOptimizer):
    """
    An optimizer which applies some "post-processing operation" per variable
    (e.g. clipping, quantization) after the gradient update.
    """
    def __init__(self, opt, func, colocate=True):
        """
        Args:
            opt (tf.train.Optimizer):
            func (tf.Variable -> tf.Operation or None): the operation needed
                to perform for this variable after the gradient update.
            colocate (boolean): colocate the function with the variable. No effect since TF 1.13.
        """
        super(PostProcessOptimizer, self).__init__(opt)
        self._func = func
        self._colocate = colocate

    @HIDE_DOC
    def apply_gradients(self, grads_and_vars, global_step=None, name=None):
        update_op = super(PostProcessOptimizer, self).apply_gradients(
            grads_and_vars, global_step)
        ops = []
        with tf.control_dependencies([update_op]):
            for _, var in grads_and_vars:
                with self._maybe_colocate(var):
                    op = self._func(var)
                    if op is not None:
                        assert isinstance(op, tf.Operation), op
                        ops.append(op)
        update_op = tf.group(update_op, *ops, name=name)
        return update_op

    @contextmanager
    def _maybe_colocate(self, var):
        G = tf.get_default_graph()
        if self._colocate and get_tf_version_tuple() <= (1, 12):
            with G.colocate_with(var):
                yield
        else:
            yield

class VariableAssignmentOptimizer(PostProcessOptimizer):
    """
    An optimizer which assigns each variable a new value (e.g. clipping,
    quantization) after the gradient update.
    """
    def __init__(self, opt, func):
        """
        Args:
            opt (tf.train.Optimizer):
            func (tf.Variable -> tf.Tensor or None): the new value to be
                assigned to this variable after the gradient update.
        """
        def f(v):
            t = func(v)
            if t is None:
                return t
            return tf.assign(v, t, use_locking=False).op
        super(VariableAssignmentOptimizer, self).__init__(opt, f)

class AccumGradOptimizer(ProxyOptimizer):
    """
    An optimizer which accumulates gradients across :math:`k` :meth:`minimize` executions,
    and apply them together in every :math:`k` th :meth:`minimize` execution.
    This is roughly the same as using a :math:`k` times larger batch size plus a
    :math:`k` times larger learning rate, but uses much less memory.

    Note that this implementation may not support all models.
    E.g., it doesn't support sparse gradient update.
    """

    def __init__(self, opt, niter):
        """
        Args:
            opt (tf.train.Optimizer): the underlying sub-optimizer.
            niter (int): number of iterations to accumulate gradients.
        """
        super(AccumGradOptimizer, self).__init__(opt, 'AccumGrad')
        self._niter = int(niter)

    def _create_accum_slots(self, var_list):
        slots = []
        for v in var_list:
            # TODO an option to not colocate the accumulators with variables (to save more memory)
            s = self._zeros_slot(v, "accum", self._name)
            slots.append(s)
        return slots

    @HIDE_DOC
    def apply_gradients(self, grads_and_vars, global_step=None, name=None):
        assert global_step is None, \
            "AccumGradOptimizer doesn't support the option global_step! " \
            "Please maintain it yourself."
        grads_and_vars = FilterNoneGrad().process(grads_and_vars)
        vs = []
        for g, v in grads_and_vars:
            assert isinstance(g, tf.Tensor) and isinstance(v, tf.Variable), \
                "AccumGradOptimizer only works for dense update! " \
                "Types of v and g are {} and {}".format(type(v), type(g))
            vs.append(v)

        with tf.control_dependencies(None):
            slots = self._create_accum_slots(vs)
            slots_and_vars = [(s, gv[1]) for s, gv in zip(slots, grads_and_vars)]

            # Create the counter on the same device as the first variable.
            with tf.variable_scope(self._name), \
                    vs[0].graph.colocate_with(vs[0]):
                counter = tf.Variable(
                    0, name="counter", trainable=False, dtype=tf.int32)

        with tf.name_scope('AccumGradOptimizer'):
            ops = []
            for s, gv in zip(slots, grads_and_vars):
                g, v = gv
                ops.append(s.assign_add(g))
            update_counter = tf.assign_add(counter, 1, name='update_counter')
            update_slot_op = tf.group(update_counter, *ops, name='update_slot')

            def update_grad():
                update_op = self._opt.apply_gradients(slots_and_vars)
                with tf.control_dependencies([update_op]):
                    clear_ops = [tf.assign(s, tf.zeros_like(s)) for s in slots]
                return tf.group(*clear_ops, name='update_grad')

            pred = tf.equal(tf.mod(counter, self._niter), 0)
            with tf.control_dependencies([update_slot_op]):
                if name is None:
                    name = 'cond_update_grad'
                op = tf.cond(pred, update_grad, tf.no_op, name=name).op
        return op

if __name__ == '__main__':
    # run it with "python -m tensorpack.tfutils.optimizer"

    x = tf.get_variable('x', shape=[6])
    cost = tf.reduce_sum(tf.abs(x), name='cost')
    opt = tf.train.GradientDescentOptimizer(0.01)
    opt = AccumGradOptimizer(opt, 5)
    min_op = opt.minimize(cost)

    sess = tf.Session()
    sess.run(tf.global_variables_initializer())
    with sess.as_default():
        for k in range(20):
            min_op.run()
            print(x.eval())


# Name:        wx_cairocffi
# Purpose:     wx.lib.wxcairo implementation functions for cairocffi
# Author:      Robin Dunn
# Tags:        phoenix-port, py3-port

"""
wx.lib.wxcairo implementation functions using cairocffi.
"""

import os
import os.path as op
import wx

# On Windows try to get cairocffi to import the Cairo DLLs included with
# wxPython instead of the first ones found on the system's PATH. Otherwise, if
# CAIRO is set in the environment then use that one instead.  This hack is
# accomplished by temporarily altering the PATH and then restoring it after
# cairocffi has initialized.
if os.name == 'nt':
    _save_path = os.environ.get('PATH')
    _cairo_path = os.environ.get('CAIRO')
    if not _cairo_path:
        _cairo_path = op.abspath(op.dirname(wx.__file__))
    os.environ['PATH'] = _cairo_path + os.pathsep + _save_path

import cairocffi
from cairocffi import cairo as cairo_c
from cairocffi import ffi

# Make it so subsequent `import cairo` statements still work as if it
# was really PyCairo
cairocffi.install_as_pycairo()

# Now restore the original PATH
if os.name == 'nt':
    os.environ['PATH'] = _save_path

# convenience functions, just to save a bit of typing below
def voidp(ptr):
    """Convert a SIP void* type to a ffi cdata"""
    if isinstance(ptr, ffi.CData):
        return ptr
    return ffi.cast('void *', int(ptr))

def ct_voidp(ptr):
    """Convert a SIP void* type to a ctypes c_void_p"""
    import ctypes
    return ctypes.c_void_p(int(ptr))

def _ContextFromDC(dc):
    if not isinstance(dc, wx.WindowDC) and not isinstance(dc, wx.MemoryDC):
        raise TypeError("Only window and memory DC's are supported at this time.")

    if 'wxMac' in wx.PlatformInfo:
        width, height = dc.GetSize()

        # use the CGContextRef of the DC to make the cairo surface
        cgc = dc.GetHandle()
        assert cgc is not None and int(cgc) != 0, "Unable to get CGContext from DC."
        ptr = voidp(cgc)
        surfaceptr = cairo_c.cairo_quartz_surface_create_for_cg_context(
            ptr, width, height)
        surface = cairocffi.Surface._from_pointer(surfaceptr, False)

        # Now create a cairo context for that surface
        ctx = cairocffi.Context(surface)

    elif 'wxMSW' in wx.PlatformInfo:
        # Similarly, get the HDC and create a surface from it
        hdc = voidp(dc.GetHandle())
        surfaceptr = cairo_c.cairo_win32_surface_create(hdc)
        surface = cairocffi.Surface._from_pointer(surfaceptr, False)

        # Now create a cairo context for that surface
        ctx = cairocffi.Context(surface)

    elif 'wxGTK' in wx.PlatformInfo:
        if 'gtk3' in wx.PlatformInfo:
            # With wxGTK3, GetHandle() returns a cairo context directly
            ctxptr = voidp( dc.GetHandle() )
            ctx = cairocffi.Context._from_pointer(ctxptr, True)

        else:
            # Get the GdkDrawable from the dc
            drawable = ct_voidp( dc.GetHandle() )

            # Call a GDK API to create a cairo context
            ctxptr = gdkLib.gdk_cairo_create(drawable)

            # Turn it into a Cairo context object
            ctx = cairocffi.Context._from_pointer(voidp(ctxptr), False)

    else:
        raise NotImplementedError("Help  me, I'm lost...")

    return ctx

def _FontFaceFromFont(font):
    if 'wxMac' in wx.PlatformInfo:
        cgFont = voidp(font.OSXGetCGFont())
        fontfaceptr = cairo_c.cairo_quartz_font_face_create_for_cgfont(cgFont)
        fontface = cairocffi.FontFace._from_pointer(fontfaceptr, False)

    elif 'wxMSW' in wx.PlatformInfo:
        hfont = voidp(font.GetHFONT())
        fontfaceptr = cairo_c.cairo_win32_font_face_create_for_hfont(hfont)
        fontface = cairocffi.FontFace._from_pointer(fontfaceptr, False)

    elif 'wxGTK' in wx.PlatformInfo:
        # wow, this is a hell of a lot of steps...
        desc = ct_voidp( font.GetPangoFontDescription() )

        pcfm = ct_voidp(pcLib.pango_cairo_font_map_get_default())

        pctx = ct_voidp(gdkLib.gdk_pango_context_get())

        pfnt = ct_voidp( pcLib.pango_font_map_load_font(pcfm, pctx, desc) )

        scaledfontptr = ct_voidp( pcLib.pango_cairo_font_get_scaled_font(pfnt) )

        fontfaceptr = cairo_c.cairo_scaled_font_get_font_face(voidp(scaledfontptr.value))
        fontface = cairocffi.FontFace._from_pointer(fontfaceptr, True)

        gdkLib.g_object_unref(pctx)

    else:
        raise NotImplementedError("Help  me, I'm lost...")

    return fontface

# GTK platforms we need to load some other shared libraries to help us get
# from wx objects to cairo objects.  This is using ctypes mainly because it
# was already working this way in wx_pycairo.py and so it required fewer brain
# cells to be sacrificed to port it to this module.
# TODO: consider moving this code to use cffi instead, for consistency.

if 'wxGTK' in wx.PlatformInfo:
    import ctypes
    _dlls = dict()

dll = _dlls.get(key, None)
        if dll is not None:
            return dll
        location = None
        for name in names:
            location = ctypes.util.find_library(name)
            if location:
                break
        if not location:
            raise RuntimeError(msg)
        dll = ctypes.CDLL(location)
        _dlls[key] = dll
        return dll

    def _findGDKLib():
        if 'gtk3' in wx.PlatformInfo:
            libname = 'gdk-3'
        else:
            libname = 'gdk-x11-2.0'
        return _findHelper([libname], 'gdk',
                           "Unable to find the GDK shared library")

    def _findPangoCairoLib():
        return _findHelper(['pangocairo-1.0'], 'pangocairo',
                           "Unable to find the pangocairo shared library")

    gdkLib = _findGDKLib()
    pcLib = _findPangoCairoLib()

    gdkLib.gdk_cairo_create.restype = ctypes.c_void_p
    gdkLib.gdk_pango_context_get.restype = ctypes.c_void_p

    pcLib.pango_cairo_font_map_get_default.restype = ctypes.c_void_p
    pcLib.pango_font_map_load_font.restype = ctypes.c_void_p
    pcLib.pango_cairo_font_get_scaled_font.restype = ctypes.c_void_p


dll = _dlls.get(key, None)
        if dll is not None:
            return dll
        location = None
        for name in names:
            location = ctypes.util.find_library(name)
            if location:
                break
        if not location:
            raise RuntimeError(msg)
        dll = ctypes.CDLL(location)
        _dlls[key] = dll
        return dll

# Name:        wx_cairocffi
# Purpose:     wx.lib.wxcairo implementation functions for cairocffi
# Author:      Robin Dunn
# Tags:        phoenix-port, py3-port

"""
wx.lib.wxcairo implementation functions using cairocffi.
"""

import os
import os.path as op
import wx

# On Windows try to get cairocffi to import the Cairo DLLs included with
# wxPython instead of the first ones found on the system's PATH. Otherwise, if
# CAIRO is set in the environment then use that one instead.  This hack is
# accomplished by temporarily altering the PATH and then restoring it after
# cairocffi has initialized.
if os.name == 'nt':
    _save_path = os.environ.get('PATH')
    _cairo_path = os.environ.get('CAIRO')
    if not _cairo_path:
        _cairo_path = op.abspath(op.dirname(wx.__file__))
    os.environ['PATH'] = _cairo_path + os.pathsep + _save_path

import cairocffi
from cairocffi import cairo as cairo_c
from cairocffi import ffi

# Make it so subsequent `import cairo` statements still work as if it
# was really PyCairo
cairocffi.install_as_pycairo()

# Now restore the original PATH
if os.name == 'nt':
    os.environ['PATH'] = _save_path

# convenience functions, just to save a bit of typing below
def voidp(ptr):
    """Convert a SIP void* type to a ffi cdata"""
    if isinstance(ptr, ffi.CData):
        return ptr
    return ffi.cast('void *', int(ptr))

def ct_voidp(ptr):
    """Convert a SIP void* type to a ctypes c_void_p"""
    import ctypes
    return ctypes.c_void_p(int(ptr))

def _ContextFromDC(dc):
    if not isinstance(dc, wx.WindowDC) and not isinstance(dc, wx.MemoryDC):
        raise TypeError("Only window and memory DC's are supported at this time.")

    if 'wxMac' in wx.PlatformInfo:
        width, height = dc.GetSize()

        # use the CGContextRef of the DC to make the cairo surface
        cgc = dc.GetHandle()
        assert cgc is not None and int(cgc) != 0, "Unable to get CGContext from DC."
        ptr = voidp(cgc)
        surfaceptr = cairo_c.cairo_quartz_surface_create_for_cg_context(
            ptr, width, height)
        surface = cairocffi.Surface._from_pointer(surfaceptr, False)

        # Now create a cairo context for that surface
        ctx = cairocffi.Context(surface)

    elif 'wxMSW' in wx.PlatformInfo:
        # Similarly, get the HDC and create a surface from it
        hdc = voidp(dc.GetHandle())
        surfaceptr = cairo_c.cairo_win32_surface_create(hdc)
        surface = cairocffi.Surface._from_pointer(surfaceptr, False)

        # Now create a cairo context for that surface
        ctx = cairocffi.Context(surface)

    elif 'wxGTK' in wx.PlatformInfo:
        if 'gtk3' in wx.PlatformInfo:
            # With wxGTK3, GetHandle() returns a cairo context directly
            ctxptr = voidp( dc.GetHandle() )
            ctx = cairocffi.Context._from_pointer(ctxptr, True)

        else:
            # Get the GdkDrawable from the dc
            drawable = ct_voidp( dc.GetHandle() )

            # Call a GDK API to create a cairo context
            ctxptr = gdkLib.gdk_cairo_create(drawable)

            # Turn it into a Cairo context object
            ctx = cairocffi.Context._from_pointer(voidp(ctxptr), False)

    else:
        raise NotImplementedError("Help  me, I'm lost...")

    return ctx

def _FontFaceFromFont(font):
    if 'wxMac' in wx.PlatformInfo:
        cgFont = voidp(font.OSXGetCGFont())
        fontfaceptr = cairo_c.cairo_quartz_font_face_create_for_cgfont(cgFont)
        fontface = cairocffi.FontFace._from_pointer(fontfaceptr, False)

    elif 'wxMSW' in wx.PlatformInfo:
        hfont = voidp(font.GetHFONT())
        fontfaceptr = cairo_c.cairo_win32_font_face_create_for_hfont(hfont)
        fontface = cairocffi.FontFace._from_pointer(fontfaceptr, False)

    elif 'wxGTK' in wx.PlatformInfo:
        # wow, this is a hell of a lot of steps...
        desc = ct_voidp( font.GetPangoFontDescription() )

        pcfm = ct_voidp(pcLib.pango_cairo_font_map_get_default())

        pctx = ct_voidp(gdkLib.gdk_pango_context_get())

        pfnt = ct_voidp( pcLib.pango_font_map_load_font(pcfm, pctx, desc) )

        scaledfontptr = ct_voidp( pcLib.pango_cairo_font_get_scaled_font(pfnt) )

        fontfaceptr = cairo_c.cairo_scaled_font_get_font_face(voidp(scaledfontptr.value))
        fontface = cairocffi.FontFace._from_pointer(fontfaceptr, True)

        gdkLib.g_object_unref(pctx)

    else:
        raise NotImplementedError("Help  me, I'm lost...")

    return fontface

# GTK platforms we need to load some other shared libraries to help us get
# from wx objects to cairo objects.  This is using ctypes mainly because it
# was already working this way in wx_pycairo.py and so it required fewer brain
# cells to be sacrificed to port it to this module.
# TODO: consider moving this code to use cffi instead, for consistency.

if 'wxGTK' in wx.PlatformInfo:
    import ctypes
    _dlls = dict()

    def _findHelper(names, key, msg):

    def _findGDKLib():
        if 'gtk3' in wx.PlatformInfo:
            libname = 'gdk-3'
        else:
            libname = 'gdk-x11-2.0'
        return _findHelper([libname], 'gdk',
                           "Unable to find the GDK shared library")

    def _findPangoCairoLib():
        return _findHelper(['pangocairo-1.0'], 'pangocairo',
                           "Unable to find the pangocairo shared library")

    gdkLib = _findGDKLib()
    pcLib = _findPangoCairoLib()

    gdkLib.gdk_cairo_create.restype = ctypes.c_void_p
    gdkLib.gdk_pango_context_get.restype = ctypes.c_void_p

    pcLib.pango_cairo_font_map_get_default.restype = ctypes.c_void_p
    pcLib.pango_font_map_load_font.restype = ctypes.c_void_p
    pcLib.pango_cairo_font_get_scaled_font.restype = ctypes.c_void_p


# Name:        wx_cairocffi
# Purpose:     wx.lib.wxcairo implementation functions for cairocffi
# Author:      Robin Dunn
# Tags:        phoenix-port, py3-port

"""
wx.lib.wxcairo implementation functions using cairocffi.
"""

import os
import os.path as op
import wx

# On Windows try to get cairocffi to import the Cairo DLLs included with
# wxPython instead of the first ones found on the system's PATH. Otherwise, if
# CAIRO is set in the environment then use that one instead.  This hack is
# accomplished by temporarily altering the PATH and then restoring it after
# cairocffi has initialized.
if os.name == 'nt':
    _save_path = os.environ.get('PATH')
    _cairo_path = os.environ.get('CAIRO')
    if not _cairo_path:
        _cairo_path = op.abspath(op.dirname(wx.__file__))
    os.environ['PATH'] = _cairo_path + os.pathsep + _save_path

import cairocffi
from cairocffi import cairo as cairo_c
from cairocffi import ffi

# Make it so subsequent `import cairo` statements still work as if it
# was really PyCairo
cairocffi.install_as_pycairo()

# Now restore the original PATH
if os.name == 'nt':
    os.environ['PATH'] = _save_path

# convenience functions, just to save a bit of typing below
def voidp(ptr):
    """Convert a SIP void* type to a ffi cdata"""
    if isinstance(ptr, ffi.CData):
        return ptr
    return ffi.cast('void *', int(ptr))

def ct_voidp(ptr):
    """Convert a SIP void* type to a ctypes c_void_p"""
    import ctypes
    return ct

def _ContextFromDC(dc):
    if not isinstance(dc, wx.WindowDC) and not isinstance(dc, wx.MemoryDC):
        raise TypeError("Only window and memory DC's are supported at this time.")

    if 'wxMac' in wx.PlatformInfo:
        width, height = dc.GetSize()

        # use the CGContextRef of the DC to make the cairo surface
        cgc = dc.GetHandle()
        assert cgc is not None and int(cgc) != 0, "Unable to get CGContext from DC."
        ptr = voidp(cgc)
        surfaceptr = cairo_c.cairo_quartz_surface_create_for_cg_context(
            ptr, width, height)
        surface = cairocffi.Surface._from_pointer(surfaceptr, False)

        # Now create a cairo context for that surface
        ctx = cairocffi.Context(surface)

    elif 'wxMSW' in wx.PlatformInfo:
        # Similarly, get the HDC and create a surface from it
        hdc = voidp(dc.GetHandle())
        surfaceptr = cairo_c.cairo_win32_surface_create(hdc)
        surface = cairocffi.Surface._from_pointer(surfaceptr, False)

        # Now create a cairo context for that surface
        ctx = cairocffi.Context(surface)

    elif 'wxGTK' in wx.PlatformInfo:
        if 'gtk3' in wx.PlatformInfo:
            # With wxGTK3, GetHandle() returns a cairo context directly
            ctxptr = voidp( dc.GetHandle() )
            ctx = cairocffi.Context._from_pointer(ctxptr, True)

        else:
            # Get the GdkDrawable from the dc
            drawable = ct_voidp( dc.GetHandle() )

            # Call a GDK API to create a cairo context
            ctxptr = gdkLib.gdk_cairo_create(drawable)

            # Turn it into a Cairo context object
            ctx = cairocffi.Context._from_pointer(voidp(ctxptr), False)

    else:
        raise NotImplementedError("Help  me, I'm lost...")

    return ctx

def _FontFaceFromFont(font):
    if 'wxMac' in wx.PlatformInfo:
        cgFont = voidp(font.OSXGetCGFont())
        fontfaceptr = cairo_c.cairo_quartz_font_face_create_for_cgfont(cgFont)
        fontface = cairocffi.FontFace._from_pointer(fontfaceptr, False)

    elif 'wxMSW' in wx.PlatformInfo:
        hfont = voidp(font.GetHFONT())
        fontfaceptr = cairo_c.cairo_win32_font_face_create_for_hfont(hfont)
        fontface = cairocffi.FontFace._from_pointer(fontfaceptr, False)

    elif 'wxGTK' in wx.PlatformInfo:
        # wow, this is a hell of a lot of steps...
        desc = ct_voidp( font.GetPangoFontDescription() )

        pcfm = ct_voidp(pcLib.pango_cairo_font_map_get_default())

        pctx = ct_voidp(gdkLib.gdk_pango_context_get())

        pfnt = ct_voidp( pcLib.pango_font_map_load_font(pcfm, pctx, desc) )

        scaledfontptr = ct_voidp( pcLib.pango_cairo_font_get_scaled_font(pfnt) )

        fontfaceptr = cairo_c.cairo_scaled_font_get_font_face(voidp(scaledfontptr.value))
        fontface = cairocffi.FontFace._from_pointer(fontfaceptr, True)

        gdkLib.g_object_unref(pctx)

    else:
        raise NotImplementedError("Help  me, I'm lost...")

    return fontface

# GTK platforms we need to load some other shared libraries to help us get
# from wx objects to cairo objects.  This is using ctypes mainly because it
# was already working this way in wx_pycairo.py and so it required fewer brain
# cells to be sacrificed to port it to this module.
# TODO: consider moving this code to use cffi instead, for consistency.

if 'wxGTK' in wx.PlatformInfo:
    import ctypes
    _dlls = dict()

    def _findHelper(names, key, msg):
        dll = _dlls.get(key, None)
        if dll is not None:
            return dll
        location = None
        for name in names:
            location = ctypes.util.find_library(name)
            if location:
                break
        if not location:
            raise RuntimeError(msg)
        dll = ctypes.CDLL(location)
        _dlls[key] = dll
        return dll

    def _findGDKLib():
        if 'gtk3' in wx.PlatformInfo:
            libname = 'gdk-3'
        else:
            libname = 'gdk-x11-2.0'
        return _findHelper([libname], 'gdk',
                           "Unable to find the GDK shared library")

    def _findPangoCairoLib():
        return _findHelper(['pangocairo-1.0'], 'pangocairo',
                           "Unable to find the pangocairo shared library")

    gdkLib = _findGDKLib()
    pcLib = _findPangoCairoLib()

    gdkLib.gdk_cairo_create.restype = ctypes.c_void_p
    gdkLib.gdk_pango_context_get.restype = ctypes.c_void_p

    pcLib.pango_cairo_font_map_get_default.restype = ctypes.c_void_p
    pcLib.pango_font_map_load_font.restype = ctypes.c_void_p
    pcLib.pango_cairo_font_get_scaled_font.restype = ctypes.c_void_p


    import ctypes
    return ctypes.c_void_p(int(ptr))

# Name:        wx_cairocffi
# Purpose:     wx.lib.wxcairo implementation functions for cairocffi
# Author:      Robin Dunn
# Tags:        phoenix-port, py3-port

"""
wx.lib.wxcairo implementation functions using cairocffi.
"""

import os
import os.path as op
import wx

# On Windows try to get cairocffi to import the Cairo DLLs included with
# wxPython instead of the first ones found on the system's PATH. Otherwise, if
# CAIRO is set in the environment then use that one instead.  This hack is
# accomplished by temporarily altering the PATH and then restoring it after
# cairocffi has initialized.
if os.name == 'nt':
    _save_path = os.environ.get('PATH')
    _cairo_path = os.environ.get('CAIRO')
    if not _cairo_path:
        _cairo_path = op.abspath(op.dirname(wx.__file__))
    os.environ['PATH'] = _cairo_path + os.pathsep + _save_path

import cairocffi
from cairocffi import cairo as cairo_c
from cairocffi import ffi

# Make it so subsequent `import cairo` statements still work as if it
# was really PyCairo
cairocffi.install_as_pycairo()

# Now restore the original PATH
if os.name == 'nt':
    os.environ['PATH'] = _save_path

# convenience functions, just to save a bit of typing below
def voidp(ptr):
    """Convert a SIP void* type to a ffi cdata"""
    if isinstance(ptr, ffi.CData):
        return ptr
    return ffi.cast('void *', int(ptr))

def ct_voidp(ptr):
    """Convert a SIP void* type to a ctypes c_void_p"""


import logging
import logging.handlers

class SingletonType(type):

try:
            return cls.__instance
        except AttributeError:
            cls.__instance = super(SingletonType, cls).__call__(*args, **kwargs)
            return cls.__instance

class LogUtil(object):
    __metaclass__ = SingletonType
    _logger = None
    _filename = None

    def __init__(self, filename=None):
        self._filename = filename

        # logger
        self._logger = logging.getLogger('logger')
        # remove default handler
        self._logger.propagate = False

        stream_handler = logging.StreamHandler()
        stream_formatter = logging.Formatter('[%(levelname)8s][%(asctime)s.%(msecs)03d] %(message)s',
                                             datefmt='%Y/%m/%d %H:%M:%S')
        stream_handler.setFormatter(stream_formatter)

        if self._filename is not None:
            file_max_bytes = 10 * 1024 * 1024

            file_handler = logging.handlers.RotatingFileHandler(filename='./log/' + self._filename,
                                                               maxBytes=file_max_bytes,
                                                               backupCount=10)
            file_formatter = logging.Formatter('[%(levelname)8s][%(asctime)s.%(msecs)03d] %(message)s',
                                               datefmt='%Y/%m/%d %H:%M:%S')
            file_handler.setFormatter(file_formatter)
            self._logger.addHandler(file_handler)

        self._logger.addHandler(stream_handler)
        self._logger.setLevel(logging.DEBUG)

    def getlogger(self):
        return self._logger


try:
            return cls.__instance
        except AttributeError:
            cls.__instance = super(SingletonType, cls).__call__(*args, **kwargs)
            return cls.__instance

import logging
import logging.handlers

class SingletonType(type):
    def __call__(cls, *args, **kwargs):

class LogUtil(object):
    __metaclass__ = SingletonType
    _logger = None
    _filename = None

    def __init__(self, filename=None):
        self._filename = filename

        # logger
        self._logger = logging.getLogger('logger')
        # remove default handler
        self._logger.propagate = False

        stream_handler = logging.StreamHandler()
        stream_formatter = logging.Formatter('[%(levelname)8s][%(asctime)s.%(msecs)03d] %(message)s',
                                             datefmt='%Y/%m/%d %H:%M:%S')
        stream_handler.setFormatter(stream_formatter)

        if self._filename is not None:
            file_max_bytes = 10 * 1024 * 1024

            file_handler = logging.handlers.RotatingFileHandler(filename='./log/' + self._filename,
                                                               maxBytes=file_max_bytes,
                                                               backupCount=10)
            file_formatter = logging.Formatter('[%(levelname)8s][%(asctime)s.%(msecs)03d] %(message)s',
                                               datefmt='%Y/%m/%d %H:%M:%S')
            file_handler.setFormatter(file_formatter)
            self._logger.addHandler(file_handler)

        self._logger.addHandler(stream_handler)
        self._logger.setLevel(logging.DEBUG)

    def getlogger(self):
        return self._logger


import logging
import logging.handlers

class SingletonType(type):
    def __call__(cls, *args, **kwargs):
        try:
            return cls.__instance
        except AttributeError:
            cls.__instance = super(SingletonType, cls).__call__(*args, **kwargs)
            return cls.__instance

class LogUtil(object):
    __metaclass__ = SingletonType
    _logger = None
    _filename = None

    def __init__(self, filename=None):
        self._filename = filename

        # logger
        self._logger = logging.getLogger('logger')
        # remove default handler
        self._logger.propagate = False

        stream_

        stream_formatter = logging.Formatter('[%(levelname)8s][%(asctime)s.%(msecs)03d] %(message)s',
                                             datefmt='%Y/%m/%d %H:%M:%S')
        stream_handler.setFormatter(stream_formatter)

        if self._filename is not None:
            file_max_bytes = 10 * 1024 * 1024

            file_handler = logging.handlers.RotatingFileHandler(filename='./log/' + self._filename,
                                                               maxBytes=file_max_bytes,
                                                               backupCount=10)
            file_formatter = logging.Formatter('[%(levelname)8s][%(asctime)s.%(msecs)03d] %(message)s',
                                               datefmt='%Y/%m/%d %H:%M:%S')
            file_handler.setFormatter(file_formatter)
            self._logger.addHandler(file_handler)

        self._logger.addHandler(stream_handler)
        self._logger.setLevel(logging.DEBUG)

    def getlogger(self):
        return self._logger


from unittest import SkipTest

from oeqa.core.exception import OEQADependency

from . import OETestDiscover, registerDecorator

def _add_depends(registry, case, depends):
    module_name = case.__module__
    class_name = case.__class__.__name__

    case_id = case.id()

    for depend in depends:
        dparts = depend.split('.')

        if len(dparts) == 1:
            depend_id = ".".join((module_name, class_name, dparts[0]))
        elif len(dparts) == 2:
            depend_id = ".".join((module_name, dparts[0], dparts[1]))
        else:
            depend_id = depend

        if not case_id in registry:
            registry[case_id] = []
        if not depend_id in registry[case_id]:
            registry[case_id].append(depend_id)

def _validate_test_case_depends(cases, depends):
    for case in depends:
        if not case in cases:
            continue
        for dep in depends[case]:
            if not dep in cases:
                raise OEQADependency("TestCase %s depends on %s and isn't available"\
                       ", cases available %s." % (case, dep, str(cases.keys())))

def _order_test_case_by_depends(cases, depends):
    def _dep_resolve(graph, node, resolved, seen):
        seen.append(node)
        for edge in graph[node]:
            if edge not in resolved:
                if edge in seen:
                    raise OEQADependency("Test cases %s and %s have a circular" \
                                       " dependency." % (node, edge))
                _dep_resolve(graph, edge, resolved, seen)
        resolved.append(node)

    dep_graph = {}
    dep_graph['__root__'] = cases.keys()
    for case in cases:
        if case in depends:
            dep_graph[case] = depends[case]
        else:
            dep_graph[case] = []

    cases_ordered = []
    _dep_resolve(dep_graph, '__root__', cases_ordered, [])
    cases_ordered.remove('__root__')

    return [cases[case_id] for case_id in cases_ordered]

def _skipTestDependency(case, depends):
    for dep in depends:
        found = False
        for test, _ in case.tc.results.successes:
            if test.id() == dep:
                found = True
                break
        if not found:
            raise SkipTest("Test case %s depends on %s but it didn't pass/run." \
                        % (case.id(), dep))

@registerDecorator
class OETestDepends(OETestDiscover):
    attrs = ('depends',)

    def bind(self, registry, case):
        super(OETestDepends, self).bind(registry, case)
        if not registry.get('depends'):
            registry['depends'] = {}
        _add_depends(registry['depends'], case, self.depends)

    @staticmethod
    def discover(registry):
        if registry.get('depends'):
            _validate_test_case_depends(registry['cases'], registry['depends'])
            return _order_test_case_by_depends(registry['cases'], registry['depends'])
        else:
            return [registry['cases'][case_id] for case_id in registry['cases']]